From 4c0a6e94f4171ea325e8c828af40fd453e344547 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Tue, 9 Mar 2021 21:43:23 +0100 Subject: [PATCH] Add inline documentation --- flake.nix | 99 ++++++++++-------- overlay/default.nix | 160 ++++++++++++++++++------------ packages/genodelabs/default.nix | 90 +++++++++++------ packages/genodelabs/toolchain.nix | 5 - tests/default.nix | 92 +++++++++-------- 5 files changed, 260 insertions(+), 186 deletions(-) diff --git a/flake.nix b/flake.nix index 4932771..258d979 100644 --- a/flake.nix +++ b/flake.nix @@ -6,15 +6,22 @@ outputs = { self, nixpkgs, nimble }: let systems = { - localSystem = [ "x86_64-linux" ]; - crossSystem = [ "aarch64-genode" "x86_64-genode" ]; + localSystem = [ "x86_64-linux" ]; # build platforms + crossSystem = [ "aarch64-genode" "x86_64-genode" ]; # target platforms }; - systemSpace = nixpkgs.lib.cartesianProductOfSets systems; - forAllLocalSystems = f: + systemSpace = + # All combinations of build and target systems + nixpkgs.lib.cartesianProductOfSets systems; + + forAllLocalSystems = + # Apply a function over all self-hosting (Linux) systems. + f: nixpkgs.lib.genAttrs systems.localSystem (system: f system); - forAllCrossSystems = f: + forAllCrossSystems = + # Apply a function over all cross-compiled systems (Genode). + f: with builtins; let f' = { localSystem, crossSystem }: @@ -27,7 +34,9 @@ attrSet = listToAttrs list; in attrSet; - forAllSystems = f: + forAllSystems = + # Apply a function over all Linux and Genode systems. + f: (forAllCrossSystems f) // (forAllLocalSystems (system: f { inherit system; @@ -38,7 +47,9 @@ in rec { overlay = - # Overlay of fixes applied to Nixpkgs + # Overlay of adjustments applied to Nixpkgs as well as + # the "genodePackages" set which the "packages" + # output of this flake is taken. import ./overlay { flake = self; }; lib = @@ -46,16 +57,16 @@ nixpkgs.lib.extend (final: prev: { inherit forAllSystems forAllLocalSystems forAllCrossSystems; - /* For a the name of a derivation output and a derivation, - generate a set of { cap, closure, and path } for a singular - file found within the subdirectory of the output with the - same name as that output. In the case that the derivation - does not have this named output, the subdirectory will be - taken from the default output. This subdirectory must - contain a single file, and the output must contain an - ERIS manifest file. - */ - getEris = output: pkg: + getEris = + # For a the name of a derivation output and a derivation, + # generate a set of { cap, closure, and path } for a singular + # file found within the subdirectory of the output with the + # same name as that output. In the case that the derivation + # does not have this named output, the subdirectory will be + # taken from the default output. This subdirectory must + # contain a single file, and the output must contain an + # ERIS manifest file. + output: pkg: with builtins; let pkg' = prev.getOutput output pkg; @@ -72,7 +83,10 @@ }) erisInfo); in assert length caps == 1; head caps; - nixosSystem = { modules, ... }@args: + nixosSystem = + # A derivative of the function for generating Linux NixOS systems. + # This one is not so well tested… + { modules, ... }@args: import "${nixpkgs}/nixos/lib/eval-config.nix" (args // { lib = final; @@ -147,37 +161,38 @@ }); - /* The nixpkgs.legacyPackages after overlaying - and with some additional Genode packages. - */ - legacyPackages = let f = import nixpkgs; - in forAllSystems ({ system, localSystem, crossSystem }: - if localSystem == crossSystem then - f { - inherit system; - overlays = [ self.overlay nimble.overlay ]; - } - else - f { - inherit localSystem; - crossSystem = { - system = crossSystem; - useLLVM = true; - }; - config.allowUnsupportedSystem = true; - overlays = [ self.overlay nimble.overlay ]; - }); + legacyPackages = + # The nixpkgs.legacyPackages set after overlaying. + let f = import nixpkgs; + in forAllSystems ({ system, localSystem, crossSystem }: + if localSystem == crossSystem then + f { + inherit system; + overlays = [ self.overlay nimble.overlay ]; + } + else + f { + inherit localSystem; + crossSystem = { + system = crossSystem; + useLLVM = true; + }; + config.allowUnsupportedSystem = true; + overlays = [ self.overlay nimble.overlay ]; + }); packages = # Genode native packages, not packages in the traditional - # sense in that these cannot be installed within a profile + # sense in that these cannot be installed within a profile. forAllCrossSystems ({ system, localSystem, crossSystem }: nixpkgs.lib.filterAttrs (n: v: v != null) self.legacyPackages.${system}.genodePackages); devShell = # Development shell for working with the - # upstream Genode source repositories + # upstream Genode source repositories. Some + # things are missing but everything referred + # to by way of #!/usr/bin/ should be here. forAllLocalSystems (system: let pkgs = nixpkgs.legacyPackages.${system}; @@ -217,11 +232,11 @@ }); nixosModules = - # Modules for composing Genode and NixOS + # Modules for composing Genode and NixOS. import ./nixos-modules { flake = self; }; checks = - # Checks for continous testing + # Checks for continous testing. let tests = import ./tests; in with (forAllCrossSystems ({ system, localSystem, crossSystem }: tests { diff --git a/overlay/default.nix b/overlay/default.nix index 32b0d4a..e6403a8 100644 --- a/overlay/default.nix +++ b/overlay/default.nix @@ -28,7 +28,7 @@ let (attrs: { nativeBuildInputs = [ final.autoreconfHook ]; }); nullPkgs = - # Nullify this packages to find problems early + # Nullify these packages to find problems early. if hostPlatform.isGenode then builtins.listToAttrs (map (name: { inherit name; @@ -46,23 +46,28 @@ in nullPkgs // { }) bash; binutils-unwrapped = overrideAttrsTarget (attrs: { - patches = attrs.patches ++ [ ./binutils/support-genode.patch ]; + patches = attrs.patches ++ [ + ./binutils/support-genode.patch + # Upstreamed, remove at next release. + ]; nativeBuildInputs = attrs.nativeBuildInputs ++ [ final.updateAutotoolsGnuConfigScriptsHook ]; }) binutils-unwrapped; - cmake = overrideAttrsTarget (attrs: { - postInstall = with stdenv; '' - local MODULE="$out/share/cmake-${ - lib.versions.majorMinor attrs.version - }/Modules/Platform/Genode.cmake" - if [ -e "$MODULE" ]; then - echo "Upstream provides $MODULE!" - exit 1 - fi - cp ${./cmake/Genode.cmake} $MODULE - ''; - }) cmake; + cmake = + # TODO: upstream + overrideAttrsTarget (attrs: { + postInstall = with stdenv; '' + local MODULE="$out/share/cmake-${ + lib.versions.majorMinor attrs.version + }/Modules/Platform/Genode.cmake" + if [ -e "$MODULE" ]; then + echo "Upstream provides $MODULE!" + exit 1 + fi + cp ${./cmake/Genode.cmake} $MODULE + ''; + }) cmake; coreutils = overrideHost { gmp = null; @@ -85,24 +90,26 @@ in nullPkgs // { }); }; - gccForLibs = if targetPlatform.isGenode then - final.genodePackages.genodeSources.toolchain.cc - else - prev.gccForLibs; - - gdb = addPatchesTarget [ ./gdb/genode.patch ] gdb; + gdb = addPatchesTarget [ + ./gdb/genode.patch + # Upstreamed, remove at next release. + ] gdb; genodeLibcCross = callPackage ./libc { }; - genodePackages = import ../packages { - inherit flake; - pkgs = final; - }; + genodePackages = + # The Genode-only packages. + import ../packages { + inherit flake; + pkgs = final; + }; - grub2 = if stdenv.targetPlatform.isGenode then - prev.buildPackages.grub2 - else - prev.grub2; + grub2 = + # No need for a Genode build of GRUB. + if stdenv.targetPlatform.isGenode then + prev.buildPackages.grub2 + else + prev.grub2; libcCrossChooser = name: if stdenv.targetPlatform.isGenode then @@ -111,52 +118,73 @@ in nullPkgs // { prev.libcCrossChooser name; libsodium = overrideAttrsHost (attrs: { - patches = (attrs.patches or [ ]) ++ [ ./libsodium/genode.patch ]; + patches = (attrs.patches or [ ]) ++ [ + ./libsodium/genode.patch + # https://github.com/jedisct1/libsodium/pull/1006 + ]; }) libsodium; - libkrb5 = autoreconfHost libkrb5; + libkrb5 = + # Do not want. + autoreconfHost libkrb5; - libtool = overrideAttrsTarget (attrs: { - nativeBuildInputs = with final; - attrs.nativeBuildInputs ++ [ autoconf automake115x ]; - patches = ./libtool/genode.patch; - }) libtool; + libtool = + # Autotools related nonesense. Better to compile + # everything static than to deal with this one. + overrideAttrsTarget (attrs: { + nativeBuildInputs = with final; + attrs.nativeBuildInputs ++ [ autoconf automake115x ]; + patches = ./libtool/genode.patch; + }) libtool; libtoxcore = overrideHost { libopus = null; libvpx = null; } libtoxcore; - linuxPackages = if hostPlatform.isGenode then { - extend = _: final.linuxPackages; - features = { }; - kernel.config = { - isEnabled = _: false; - isYes = _: false; - }; - } else - linuxPackages; + linuxPackages = + # Dummy package. + if hostPlatform.isGenode then { + extend = _: final.linuxPackages; + features = { }; + kernel.config = { + isEnabled = _: false; + isYes = _: false; + }; + } else + linuxPackages; - llvmPackages_11 = callPackage ./llvm-11 ({ - inherit (stdenvAdapters) overrideCC; - buildLlvmTools = buildPackages.llvmPackages_11.tools; - targetLlvmLibraries = targetPackages.llvmPackages_11.libraries; - } // lib.optionalAttrs - (stdenv.hostPlatform.isi686 && buildPackages.stdenv.cc.isGNU) { - stdenv = gcc7Stdenv; - }); + llvmPackages_11 = + # A copy of the LLVM expressions from Nixpkgs. + # Nothing here is upstreamed. + callPackage ./llvm-11 ({ + inherit (stdenvAdapters) overrideCC; + buildLlvmTools = buildPackages.llvmPackages_11.tools; + targetLlvmLibraries = targetPackages.llvmPackages_11.libraries; + } // lib.optionalAttrs + (stdenv.hostPlatform.isi686 && buildPackages.stdenv.cc.isGNU) { + stdenv = gcc7Stdenv; + }); - ncurses = addPatchesHost [ ./ncurses/genode.patch ] ncurses; + ncurses = + # https://invisible-island.net/autoconf/ + # Stay clear of upstream on this one. + addPatchesHost [ ./ncurses/genode.patch ] ncurses; - nim = overrideAttrsTarget (attrs: { - postInstall = '' - sed \ - -e '/cc = gcc/d' \ - -i $out/etc/nim/nim.cfg - ''; - }) prev.nim; + nim = + # Nim is configured to build Genode with GCC. + overrideAttrsTarget (attrs: { + postInstall = '' + sed \ + -e '/cc = gcc/d' \ + -i $out/etc/nim/nim.cfg + ''; + }) prev.nim; - nim-unwrapped = addPatchesTarget [ ./nim/genode.patch ] prev.nim-unwrapped; + nim-unwrapped = addPatchesTarget [ + ./nim/genode.patch + # Fixes to the compiler and standard libary. + ] prev.nim-unwrapped; openssl = overrideHost { static = true; } # shared library comes out stupid big @@ -173,7 +201,10 @@ in nullPkgs // { "rm $out/bin/c_rehash"; # eliminate the perl runtime dependency }) openssl); - patchelf = addPatchesTarget [ ./patchelf/dynstr.patch ] prev.patchelf; + patchelf = addPatchesTarget [ + ./patchelf/dynstr.patch + # Patch to fix a bug in rewriting the .dynstr section. + ] prev.patchelf; rsync = overrideHost { enableACLs = false; @@ -190,7 +221,10 @@ in nullPkgs // { tor = overrideAttrsHost (attrs: { configureFlags = attrs.configureFlags or [ ] ++ [ "--disable-tool-name-check" ]; - patches = attrs.patches or [ ] ++ [ ./tor/genode.patch ]; + patches = attrs.patches or [ ] ++ [ + ./tor/genode.patch + # We don't do users and groups here. + ]; postPatch = null; # Avoid torsocks patching }) tor; diff --git a/packages/genodelabs/default.nix b/packages/genodelabs/default.nix index 3e63791..178cbcb 100644 --- a/packages/genodelabs/default.nix +++ b/packages/genodelabs/default.nix @@ -13,15 +13,18 @@ let else throw "unknown Genode arch for platform ${platform.system}"; - upstreamSources = pkgs.fetchFromGitHub { - owner = "genodelabs"; - repo = "genode"; - rev = "3fac8b106d83721914797c202793ec1d8ea02d2f"; - hash = "sha256-XgN1fBUsmX8oKk4ZBvROwEWlpILRlJz+UuK4kMDSI1Y="; - }; + upstreamSources = + # This is where the Genode source tree is defined. + # Must be updated with ./patches/sources.patch. + pkgs.fetchFromGitHub { + owner = "genodelabs"; + repo = "genode"; + rev = "3fac8b106d83721914797c202793ec1d8ea02d2f"; + hash = "sha256-XgN1fBUsmX8oKk4ZBvROwEWlpILRlJz+UuK4kMDSI1Y="; + }; genodeSources = - # The Genode source repository + # The Genode source repository after patching. let toolPrefix = if platform.isx86 then "genode-x86-" @@ -56,10 +59,16 @@ let installPhase = "cp -a . $out"; }; - portVersions = import "${genodeSources}/ports.nix"; + portVersions = + # Port versions are taken from the sources to force + # updates of the port fixed-output derivations. + import "${genodeSources}/ports.nix"; preparePort = - # Prepare a "port" of source code declared in the Genode sources + # Prepare a "port" of source code declared in the Genode sources. + # This is fragile because breakage can appear when the packages + # used in preparation are updated, but previously successful + # builds will cache. name: { hash ? "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" , patches ? [ ], extraRepos ? [ ], ... }@args: @@ -122,20 +131,26 @@ let dontFixup = true; }); - ports = lib.mapAttrs preparePort (import ./ports.nix { - pkgs = flake.inputs.nixpkgs.legacyPackages.x86_64-linux // { - inherit (pkgs) genodePackages; - }; - }); - # The "ports" mechanism is hardly deterministic, so prepare with - # a pinned nixpkgs revision for a pinned platform. + ports = + # The "ports" mechanism is hardly deterministic, so prepare with + # a pinned nixpkgs revision for a pinned platform for consistency. + lib.mapAttrs preparePort (import ./ports.nix { + pkgs = flake.inputs.nixpkgs.legacyPackages.x86_64-linux // { + inherit (pkgs) genodePackages; + }; + }); - toolchain = buildPackages.buildPackages.callPackage ./toolchain.nix { }; + toolchain = + # Patched GCC build from upstream. + buildPackages.buildPackages.callPackage ./toolchain.nix { }; - stdenv' = pkgs.stdenvAdapters.overrideCC pkgs.stdenv toolchain; + stdenv' = + # Special stdenv for use within the upstream sources. + # TODO: build with Clang. + pkgs.stdenvAdapters.overrideCC pkgs.stdenv toolchain; buildUpstream = - # Build from the Genode sources + # Build from the Genode sources using the least recursive make. { name, targets, portInputs ? [ ], nativeBuildInputs ? [ ], patches ? [ ] , enableParallelBuilding ? true, meta ? { }, ... }@extraAttrs: let havePatches = patches != [ ]; @@ -185,7 +200,7 @@ let }); buildDepot = - # Build a Depot target from the Genode sources + # Build from the Genode sources using the depot build system. # WARNING: buildDepot can produce artifacts with broken linkage # to their inputs. The Genode depot mechanism links programs and # libraries to facsimilie stub libraries which are not guaranteed @@ -209,8 +224,7 @@ let enableParallelBuilding = true; nativeBuildInputs = with buildPackages.buildPackages; - [ binutils bison flex stdenv.cc tcl which ] - ++ nativeBuildInputs + [ binutils bison flex stdenv.cc tcl which ] ++ nativeBuildInputs ++ lib.optional (!stdenv.hostPlatform.isGenode) erisPatchHook; buildInputs = buildInputs ++ depotInputs'; @@ -277,15 +291,18 @@ let }); in self; - makePackages = let - overrides = import ./make-targets.nix { - inherit (pkgs) buildPackages genodePackages; - inherit ports; - }; - in lib.attrsets.mapAttrs - (name: value: (buildUpstream ({ inherit name; } // value))) overrides; + makePackages = + # Build everything in ./make-targets.nix. + let + overrides = import ./make-targets.nix { + inherit (pkgs) buildPackages genodePackages; + inherit ports; + }; + in lib.attrsets.mapAttrs + (name: value: (buildUpstream ({ inherit name; } // value))) overrides; depotPackages = lib.attrsets.mapAttrs + # Build everything in ./depot-targets.nix. (name: value: (buildDepot ({ inherit name; } // value))) (import ./depot-targets.nix { inherit (pkgs) genodePackages; @@ -313,7 +330,8 @@ let ++ lib.optional isx86_64 "x86_64"; genodeBase = - # A package containing the Genode C++ headers, a stub ld.lib.so and vfs.lib.so + # A package containing the Genode C++ headers + # and a stub ld.lib.so and vfs.lib.so. buildUpstream { name = "base"; targets = [ "LIB=vfs" ]; @@ -342,9 +360,15 @@ let in makePackages // depotPackages // { - genodeSources = genodeSources // { - inherit arch buildUpstream buildDepot genodeBase ports specs toolchain; - }; + genodeSources = + # Expose genodeSources and tuck some extras in with it. + genodeSources // { + inherit arch buildUpstream buildDepot genodeBase ports specs toolchain; + }; + + # Builds of the Genode base-systems follow. + # These contain the hardware and kernel specific core program, + # the loader and base-library, and a timer driver. base-hw-pc = buildUpstream { name = "base-hw-pc"; diff --git a/packages/genodelabs/toolchain.nix b/packages/genodelabs/toolchain.nix index b26e6c0..a450a52 100644 --- a/packages/genodelabs/toolchain.nix +++ b/packages/genodelabs/toolchain.nix @@ -1,11 +1,6 @@ # Shameless plagiarism of Blitz's toolchain expression: # https://github.com/blitz/genode-nix -# -# WARNING: these binaries are from sourceforge and -# have not been publicly verified by Genode Labs. -# - { lib, stdenv, fetchurl, ncurses5, expat, makeWrapper, wrapCC }: let diff --git a/tests/default.nix b/tests/default.nix index ba1de2d..44e4e0e 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -5,53 +5,59 @@ let nixpkgs = flake.legacyPackages.${system}; legacyPackages = flake.legacyPackages.${system}; - testingPython = import ./lib/testing-python.nix; + testingPython = + # Mostly lifted from Nixpkgs. + import ./lib/testing-python.nix; - testSpace = lib.cartesianProductOfSets { + testSpace = + # Run all tests on all defined Genode platforms + lib.cartesianProductOfSets { - test = map (p: import p) [ - ./ahci.nix - ./bash.nix - ./hello.nix - ./log.nix - ./nim.nix - ./vmm_x86.nix - ]; - - core = builtins.filter (core: - builtins.any (x: x == pkgs.stdenv.hostPlatform.system) core.platforms) [ - /* { - prefix = "hw-pc-"; - testingPython = testingPython { - inherit flake system localSystem crossSystem pkgs; - extraConfigurations = [ ../nixos-modules/base-hw-pc.nix ]; - }; - specs = [ "x86" "hw" ]; - platforms = [ "x86_64-genode" ]; - } - */ - /* { - prefix = "hw-virt_qemu-"; - testingPython = testingPython { - inherit flake system localSystem crossSystem pkgs; - extraConfigurations = [ ../nixos-modules/base-hw-virt_qemu.nix ]; - }; - specs = [ "aarch64" "hw" ]; - platforms = [ "aarch64-genode" ]; - } - */ - { - prefix = "nova-"; - testingPython = testingPython { - inherit flake system localSystem crossSystem pkgs; - extraConfigurations = [ ../nixos-modules/nova.nix ]; - }; - specs = [ "x86" "nova" ]; - platforms = [ "x86_64-genode" ]; - } + test = map (p: import p) [ + ./ahci.nix + ./bash.nix + ./hello.nix + ./log.nix + ./nim.nix + ./vmm_x86.nix ]; - }; + core = builtins.filter (core: + builtins.any (x: x == pkgs.stdenv.hostPlatform.system) core.platforms) [ + /* # Need to fix the QEMU boot parameters? + { + prefix = "hw-pc-"; + testingPython = testingPython { + inherit flake system localSystem crossSystem pkgs; + extraConfigurations = [ ../nixos-modules/base-hw-pc.nix ]; + }; + specs = [ "x86" "hw" ]; + platforms = [ "x86_64-genode" ]; + } + */ + /* # Need to fix the QEMU boot parameters? + { + prefix = "hw-virt_qemu-"; + testingPython = testingPython { + inherit flake system localSystem crossSystem pkgs; + extraConfigurations = [ ../nixos-modules/base-hw-virt_qemu.nix ]; + }; + specs = [ "aarch64" "hw" ]; + platforms = [ "aarch64-genode" ]; + } + */ + { + prefix = "nova-"; + testingPython = testingPython { + inherit flake system localSystem crossSystem pkgs; + extraConfigurations = [ ../nixos-modules/nova.nix ]; + }; + specs = [ "x86" "nova" ]; + platforms = [ "x86_64-genode" ]; + } + ]; + + }; testList = let f = { core, test }: