From 4f74ff94526e9eb8594038daacde66aeb7c51b18 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sun, 2 Aug 2020 16:46:34 +0200 Subject: [PATCH] Documentation and cleanup --- flake.nix | 303 +++++++++++++++++--------------- packages/genodelabs/default.nix | 139 ++++++++------- packages/genodelabs/ports.nix | 3 + packages/genodelabs/targets.nix | 6 +- tests/hw-test-driver.exp | 5 - tests/nova-test-driver.exp | 5 - tests/test-driver.exp | 32 ---- 7 files changed, 244 insertions(+), 249 deletions(-) delete mode 100644 tests/hw-test-driver.exp delete mode 100644 tests/nova-test-driver.exp delete mode 100644 tests/test-driver.exp diff --git a/flake.nix b/flake.nix index 0da83b8..7595e7a 100644 --- a/flake.nix +++ b/flake.nix @@ -55,127 +55,101 @@ in rec { - overlay = import ./overlay; + overlay = + # Overlay of fixes applied to Nixpkgs + import ./overlay; - lib = forAllCrossSystems ({ system, localSystem, crossSystem }: - nixpkgs.lib // (import ./lib { - inherit system localSystem crossSystem; - apps = self.apps.${system}; - genodepkgs = self; - nixpkgs = nixpkgsFor.${system}; - })); + lib = + # Local utilities merged with the Nixpkgs lib + forAllCrossSystems ({ system, localSystem, crossSystem }: + nixpkgs.lib // (import ./lib { + inherit system localSystem crossSystem; + apps = self.apps.${system}; + genodepkgs = self; + nixpkgs = nixpkgsFor.${system}; + })); - legacyPackages = forAllSystems + legacyPackages = + # The nixpkgs.legacyPackages after overlaying + # and with some additional Genode packages + forAllSystems ({ system, localSystem, crossSystem }: nixpkgsFor.${system}); # pass thru Nixpkgs - packages = forAllCrossSystems ({ system, localSystem, crossSystem }: - nixpkgs.lib.filterAttrs (n: v: v != null) - nixpkgsFor.${system}.genodePackages); + packages = + # Genode native packages, not packages in the traditional + # sense in that these cannot be installed within a profile + forAllCrossSystems ({ system, localSystem, crossSystem }: + nixpkgs.lib.filterAttrs (n: v: v != null) + nixpkgsFor.${system}.genodePackages); - devShell = forAllLocalSystems (system: - let - pkgs = nixpkgsFor.${system}; - fhs = pkgs.buildFHSUserEnv { - name = "genode-env"; - targetPkgs = pkgs: - (with pkgs; [ - binutils - bison - expect - flex - git - glibc.dev - gnumake - libxml2 - qemu - tcl - which - xorriso - ]); - runScript = "bash"; - extraBuildCommands = let - toolchain = pkgs.fetchzip { - url = - "file://${packages.x86_64-linux-x86_64-genode.genodeSources.toolchain.src}"; - hash = "sha256-26rPvLUPEJm40zLSqTquwuFTJ1idTB0T4VXgaHRN+4o="; - }; - in "ln -s ${toolchain}/local usr/local"; - }; - in pkgs.stdenv.mkDerivation { - name = "genode-fhs-shell"; - nativeBuildInputs = [ fhs ]; - shellHook = "exec genode-env"; - }); - - apps = let - apps' = forAllCrossSystems ({ system, localSystem, crossSystem }: - import ./apps { - inherit system; - self = self.apps.${system}; - nixpkgs = nixpkgsFor.${system}; - nixpkgsLocal = nixpkgsFor.${localSystem}; - packages = self.packages.${system}; - } // { - nixosModule = { - type = "app"; - program = - "${self.checks.${localSystem}.nixosGuest}/bin/run-nixos-vm"; - }; - }); - in apps' // { x86_64-linux = apps'.x86_64-linux-x86_64-genode; }; - - nixosModules = import ./nixos-modules { inherit self; }; - - checks = let - - checks' = forAllCrossSystems ({ system, localSystem, crossSystem }: + devShell = + # Development shell for working with the + # upstream Genode source repositories + forAllLocalSystems (system: let - nixos = nixpkgs.lib.nixosSystem { - system = localSystem; - modules = [ - self.nixosModules.genodeHost - ({ modulesPath, ... }: { - imports = [ "${modulesPath}/virtualisation/qemu-vm.nix" ]; - boot.loader.grub.extraEntriesBeforeNixOS = true; - genode.inputFilter.extraChargen = - "${./nixos-modules/dhall/workman.chargen.dhall}"; - virtualisation = { - cores = 2; - memorySize = 1024; - useBootLoader = true; - qemu = { - options = - [ "-machine q35" "-cpu phenom" "-serial mon:stdio" ]; - networkingOptions = [ - "-net nic,netdev=user.0,model=e1000" - "-netdev user,id=user.0\${QEMU_NET_OPTS:+,$QEMU_NET_OPTS}" - ]; - }; - }; - }) - ]; + pkgs = nixpkgsFor.${system}; + fhs = pkgs.buildFHSUserEnv { + name = "genode-env"; + targetPkgs = pkgs: + (with pkgs; [ + binutils + bison + expect + flex + git + glibc.dev + gnumake + libxml2 + qemu + tcl + which + xorriso + ]); + runScript = "bash"; + extraBuildCommands = let + toolchain = pkgs.fetchzip { + url = + "file://${packages.x86_64-linux-x86_64-genode.genodeSources.toolchain.src}"; + hash = "sha256-26rPvLUPEJm40zLSqTquwuFTJ1idTB0T4VXgaHRN+4o="; + }; + in "ln -s ${toolchain}/local usr/local"; }; - in import ./tests { - inherit self; - apps = self.apps.${system}; - localPackages = nixpkgsFor.${localSystem}; - genodepkgs = self.packages.${system}; - lib = self.lib.${system}; - nixpkgs = nixpkgsFor.${system}; - legacyPackages = self.legacyPackages.${system}; - } // { + in pkgs.stdenv.mkDerivation { + name = "genode-fhs-shell"; + nativeBuildInputs = [ fhs ]; + shellHook = "exec genode-env"; + }); - ports = nixpkgsFor.${localSystem}.symlinkJoin { - name = "ports"; - paths = (builtins.attrValues - self.packages.${system}.genodeSources.ports); - }; + apps = + # Utilities accessible via "nix run" + let + apps' = forAllCrossSystems ({ system, localSystem, crossSystem }: + import ./apps { + inherit system; + self = self.apps.${system}; + nixpkgs = nixpkgsFor.${system}; + nixpkgsLocal = nixpkgsFor.${localSystem}; + packages = self.packages.${system}; + } // { + nixosModule = { + type = "app"; + program = + "${self.checks.${localSystem}.nixosGuest}/bin/run-nixos-vm"; + }; + }); + in apps' // { x86_64-linux = apps'.x86_64-linux-x86_64-genode; }; - nixosModule = nixos.config.system.build.vm; - nixosXML = nixos.config.system.build.genode.xml; + nixosModules = + # Modules for composing Genode and NixOS + import ./nixos-modules { inherit self; }; - nixosGuest = let + checks = + # Checks for continous testing + let + + checks' = forAllCrossSystems ({ system, localSystem, crossSystem }: + let nixos = nixpkgs.lib.nixosSystem { system = localSystem; modules = [ @@ -196,40 +170,89 @@ ]; }; }; - genode.guests = { - flakeCheck = { - config = { config, lib, pkgs, ... }: { - networking.firewall.allowedTCPPorts = [ 80 ]; - services.mingetty.autologinUser = - lib.mkDefault "root"; - services.nginx.enable = true; - services.openssh.permitRootLogin = - lib.mkDefault "yes"; - services.sshd.enable = true; - users.users.root.password = "nixos"; - virtualbox.memorySize = 128; - }; - }; - azimuth = { - bootFormat = "vdi"; - config = { config, lib, pkgs, ... }: { - services.cage = { - enable = true; - program = "${pkgs.azimuth}/bin/tor-azimuth"; - }; - users.users.demo.isNormalUser = true; - virtualbox.memorySize = 1024; - }; - }; - }; - }) ]; }; - in nixos.config.system.build.vm; + in import ./tests { + inherit self; + apps = self.apps.${system}; + localPackages = nixpkgsFor.${localSystem}; + genodepkgs = self.packages.${system}; + lib = self.lib.${system}; + nixpkgs = nixpkgsFor.${system}; + legacyPackages = self.legacyPackages.${system}; + } // { - }); - in { x86_64-linux = checks'.x86_64-linux-x86_64-genode; }; + ports = nixpkgsFor.${localSystem}.symlinkJoin { + name = "ports"; + paths = (builtins.attrValues + self.packages.${system}.genodeSources.ports); + }; + + nixosModule = nixos.config.system.build.vm; + nixosXML = nixos.config.system.build.genode.xml; + + nixosGuest = let + nixos = nixpkgs.lib.nixosSystem { + system = localSystem; + modules = [ + self.nixosModules.genodeHost + ({ modulesPath, ... }: { + imports = [ "${modulesPath}/virtualisation/qemu-vm.nix" ]; + boot.loader.grub.extraEntriesBeforeNixOS = true; + virtualisation = { + cores = 2; + memorySize = 1024; + useBootLoader = true; + qemu = { + options = [ + "-machine q35" + "-cpu phenom" + "-serial mon:stdio" + ]; + networkingOptions = [ + "-net nic,netdev=user.0,model=e1000" + "-netdev user,id=user.0\${QEMU_NET_OPTS:+,$QEMU_NET_OPTS}" + ]; + }; + }; + genode.guests = { + flakeCheck = { + config = { config, lib, pkgs, ... }: { + networking.firewall.allowedTCPPorts = [ 80 ]; + services.mingetty.autologinUser = + lib.mkDefault "root"; + services.nginx.enable = true; + services.openssh.permitRootLogin = + lib.mkDefault "yes"; + services.sshd.enable = true; + users.users.root.password = "nixos"; + virtualbox.memorySize = 128; + }; + }; + azimuth = { + bootFormat = "vdi"; + config = { config, lib, pkgs, ... }: { + services.cage = { + enable = true; + program = "${pkgs.azimuth}/bin/tor-azimuth"; + }; + users.users.demo.isNormalUser = true; + virtualbox.memorySize = 1024; + }; + }; + }; + + }) + ]; + }; + in nixos.config.system.build.vm; + + }); + in { + aarch64-linux = checks'.aarch64-linux-aarch64-genode; + x86_64-linux = checks'.x86_64-linux-x86_64-genode; + }; }; } diff --git a/packages/genodelabs/default.nix b/packages/genodelabs/default.nix index 18b6f64..de8f4d1 100644 --- a/packages/genodelabs/default.nix +++ b/packages/genodelabs/default.nix @@ -16,49 +16,53 @@ let else throw "unknown Genode arch for platform ${platform.system}"; - genodeSources = let + genodeSources = + # The Genode source repository + let - toolPrefix = if platform.isx86 then - "genode-x86-" - else if platform.isAarch64 then - "genode-aarch64-" - else - throw "unknown tool prefix for Genode arch ${arch}"; - in stdenvNoCC.mkDerivation { - pname = "genode-sources"; - inherit version; - src = fetchurl { - url = "https://github.com/genodelabs/genode/archive/${version}.tar.gz"; - hash = "sha256-5Fd8ywGbDtQF+9yDZ5xAQ7l1Vzh1jQAQeqVj+EtO1us="; + toolPrefix = if platform.isx86 then + "genode-x86-" + else if platform.isAarch64 then + "genode-aarch64-" + else + throw "unknown tool prefix for Genode arch ${arch}"; + in stdenvNoCC.mkDerivation { + pname = "genode-sources"; + inherit version; + src = fetchurl { + url = "https://github.com/genodelabs/genode/archive/${version}.tar.gz"; + hash = "sha256-5Fd8ywGbDtQF+9yDZ5xAQ7l1Vzh1jQAQeqVj+EtO1us="; + }; + nativeBuildInputs = [ expect gnumake tcl ]; + patches = [ ./LIB.patch ./binary-labels.patch ]; + configurePhase = '' + patchShebangs ./tool + substituteInPlace repos/base/etc/tools.conf \ + --replace "/usr/local/genode/tool/19.05/bin/" "" + substituteInPlace tool/check_abi \ + --replace "exec nm" "exec ${toolPrefix}nm" + ''; + + buildPhase = '' + echo { >> ports.nix + find repos/*/ports -name '*.hash' | while read hashFile + do + echo " $(basename --suffix=.hash $hashFile) = \"$(cut -c -6 $hashFile)\";" >> ports.nix + done + echo } >> ports.nix + ''; + + installPhase = "cp -a . $out"; }; - nativeBuildInputs = [ expect gnumake tcl ]; - patches = [ ./LIB.patch ./binary-labels.patch ]; - configurePhase = '' - patchShebangs ./tool - substituteInPlace repos/base/etc/tools.conf \ - --replace "/usr/local/genode/tool/19.05/bin/" "" - substituteInPlace tool/check_abi \ - --replace "exec nm" "exec ${toolPrefix}nm" - ''; - - buildPhase = '' - echo { >> ports.nix - find repos/*/ports -name '*.hash' | while read hashFile - do - echo " $(basename --suffix=.hash $hashFile) = \"$(cut -c -6 $hashFile)\";" >> ports.nix - done - echo } >> ports.nix - ''; - - installPhase = "cp -a . $out"; - }; toolchain = callPackage ./toolchain.nix { }; stdenv' = targetPackages.stdenvAdapters.overrideCC targetPackages.stdenv toolchain; - preparePort = name: + preparePort = + # Prepare a "port" of source code declared in the Genode sources + name: { hash ? "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" , patches ? [ ], ... }@args: let @@ -118,9 +122,10 @@ let ports = stdenv.lib.mapAttrs preparePort (import ./ports.nix { inherit buildPackages; }); - buildUpstream = { name, targets, portInputs ? [ ], nativeBuildInputs ? [ ] - , patches ? [ ], enableParallelBuilding ? true, meta ? { }, ... - }@extraAttrs: + buildUpstream = + # Build from the Genode sources + { name, targets, portInputs ? [ ], nativeBuildInputs ? [ ], patches ? [ ] + , enableParallelBuilding ? true, meta ? { }, ... }@extraAttrs: let havePatches = patches != [ ]; in stdenv'.mkDerivation (extraAttrs // { @@ -164,8 +169,10 @@ let meta = { platforms = stdenv.lib.platforms.genode; } // meta; }); - buildDepot = { name, apiOnly ? false, portInputs ? [ ] - , nativeBuildInputs ? [ ], meta ? { }, ... }@extraAttrs: + buildDepot = + # Build a Depot target from the Genode sources + { name, apiOnly ? false, portInputs ? [ ], nativeBuildInputs ? [ ] + , meta ? { }, ... }@extraAttrs: stdenv'.mkDerivation (extraAttrs // { pname = name; @@ -239,31 +246,33 @@ let ++ lib.optional isx86_64 "x86_64"; - genodeBase = buildUpstream { - name = "base"; - targets = [ "LIB=vfs" ]; - postInstall = - # The actual ld.lib.so is kernel specific - # so ship the stubbed library for linking - '' - cp $BUILD_DIR/var/libcache/ld/ld.abi.so $out/ld.lib.so - mkdir -p $out/include - cp -r --no-preserve=mode \ - $GENODE_DIR/repos/base/include/* \ - $GENODE_DIR/repos/os/include/* \ - $GENODE_DIR/repos/demo/include/* \ - $GENODE_DIR/repos/gems/include/* \ - $out/include/ - for spec in ${toString specs}; do - dir=$out/include/spec/$spec - if [ -d $dir ]; then - cp -r $dir/* $out/include/ - fi - done - rm -rf $out/include/spec - cp -r $GENODE_DIR/repos/base/src/ld $out/ld - ''; - }; + genodeBase = + # A package containing the Genode C++ headers, a stub ld.lib.so and libvfs.lib.so + buildUpstream { + name = "base"; + targets = [ "LIB=vfs" ]; + postInstall = + # The actual ld.lib.so is kernel specific + # so ship the stubbed library for linking + '' + cp $BUILD_DIR/var/libcache/ld/ld.abi.so $out/ld.lib.so + mkdir -p $out/include + cp -r --no-preserve=mode \ + $GENODE_DIR/repos/base/include/* \ + $GENODE_DIR/repos/os/include/* \ + $GENODE_DIR/repos/demo/include/* \ + $GENODE_DIR/repos/gems/include/* \ + $out/include/ + for spec in ${toString specs}; do + dir=$out/include/spec/$spec + if [ -d $dir ]; then + cp -r $dir/* $out/include/ + fi + done + rm -rf $out/include/spec + cp -r $GENODE_DIR/repos/base/src/ld $out/ld + ''; + }; in genodeSources // { inherit buildUpstream buildDepot ports specs toolchain genodeBase; diff --git a/packages/genodelabs/ports.nix b/packages/genodelabs/ports.nix index 09c78d4..f819096 100644 --- a/packages/genodelabs/ports.nix +++ b/packages/genodelabs/ports.nix @@ -1,3 +1,6 @@ +# This file specifies the output hashes of "Ports". +# Ports not listed here can still be prepared, but will result in a hash mismatch. + { buildPackages }: with buildPackages; { dde_bsd.hash = "sha256-2ilZs6JkcQAvU/sQYnHunGtGRZDt7qLTpKZxQxSOf2I="; diff --git a/packages/genodelabs/targets.nix b/packages/genodelabs/targets.nix index 433b6a0..d68816f 100644 --- a/packages/genodelabs/targets.nix +++ b/packages/genodelabs/targets.nix @@ -1,3 +1,6 @@ +# This file contains overrides necesarry to build some Make and Depot targets. +# Many targets can be built with the default attributes, and are not listed here. + { buildPackages, ports }: with ports; let @@ -58,8 +61,7 @@ in { stdcxx.portInputs = [ libc stdcxx ]; - # The following are tests that never exit - # and have no conventions on log output. + # The following are tests are patched to exit at completion "test-log".patches = [ ./test-log.patch ]; diff --git a/tests/hw-test-driver.exp b/tests/hw-test-driver.exp deleted file mode 100644 index 98c12b5..0000000 --- a/tests/hw-test-driver.exp +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env expect - -eval $env(baseSetup) - -eval $env(testScript) diff --git a/tests/nova-test-driver.exp b/tests/nova-test-driver.exp deleted file mode 100644 index 98c12b5..0000000 --- a/tests/nova-test-driver.exp +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env expect - -eval $env(baseSetup) - -eval $env(testScript) diff --git a/tests/test-driver.exp b/tests/test-driver.exp deleted file mode 100644 index be9553f..0000000 --- a/tests/test-driver.exp +++ /dev/null @@ -1,32 +0,0 @@ -#! /usr/bin/env expect - -## -# Wait for a specific output of a already running spawned process -# -proc wait_for_output { wait_for_re timeout_value running_spawn_id } { - global output - - if {$wait_for_re == "forever"} { - set timeout -1 - interact { - \003 { - send_user "Expect: 'interact' received 'strg+c' and was cancelled\n"; - exit - } - -i $running_spawn_id - } - } else { - set timeout $timeout_value - } - - expect { - -i $running_spawn_id -re $wait_for_re { } - eof { puts stderr "Error: Spawned process died unexpectedly"; exit -1 } - timeout { puts stderr "Error: Test execution timed out"; exit -1 } - } - set output $expect_out(buffer) -} - -eval $env(baseSetup) - -eval $env(testScript)