Add inline documentation

flake.nix

@@ -6,15 +6,22 @@
   outputs = { self, nixpkgs, nimble }:
     let
       systems = {
-        localSystem = [ "x86_64-linux" ];
-        crossSystem = [ "aarch64-genode" "x86_64-genode" ];
+        localSystem = [ "x86_64-linux" ]; # build platforms
+        crossSystem = [ "aarch64-genode" "x86_64-genode" ]; # target platforms
       };
-      systemSpace = nixpkgs.lib.cartesianProductOfSets systems;
 
-      forAllLocalSystems = f:
+      systemSpace =
+        # All combinations of build and target systems
+        nixpkgs.lib.cartesianProductOfSets systems;
+
+      forAllLocalSystems =
+        # Apply a function over all self-hosting (Linux) systems.
+        f:
         nixpkgs.lib.genAttrs systems.localSystem (system: f system);
 
-      forAllCrossSystems = f:
+      forAllCrossSystems =
+        # Apply a function over all cross-compiled systems (Genode).
+        f:
         with builtins;
         let
           f' = { localSystem, crossSystem }:
@@ -27,7 +34,9 @@
           attrSet = listToAttrs list;
         in attrSet;
 
-      forAllSystems = f:
+      forAllSystems =
+        # Apply a function over all Linux and Genode systems.
+        f:
         (forAllCrossSystems f) // (forAllLocalSystems (system:
           f {
             inherit system;
@@ -38,7 +47,9 @@
     in rec {
 
       overlay =
-        # Overlay of fixes applied to Nixpkgs
+        # Overlay of adjustments applied to Nixpkgs as well as
+        # the "genodePackages" set which the "packages"
+        # output of this flake is taken.
         import ./overlay { flake = self; };
 
       lib =
@@ -46,16 +57,16 @@
         nixpkgs.lib.extend (final: prev: {
           inherit forAllSystems forAllLocalSystems forAllCrossSystems;
 
-          /* For a the name of a derivation output and a derivation,
-             generate a set of { cap, closure, and path } for a singular
-             file found within the subdirectory of the output with the
-             same name as that output. In the case that the derivation
-             does not have this named output, the subdirectory will be
-             taken from the default output. This subdirectory must
-             contain a single file, and the output must contain an
-             ERIS manifest file.
-          */
-          getEris = output: pkg:
+          getEris =
+            # For a the name of a derivation output and a derivation,
+            # generate a set of { cap, closure, and path } for a singular
+            # file found within the subdirectory of the output with the
+            # same name as that output. In the case that the derivation
+            # does not have this named output, the subdirectory will be
+            # taken from the default output. This subdirectory must
+            # contain a single file, and the output must contain an
+            # ERIS manifest file.
+            output: pkg:
             with builtins;
             let
               pkg' = prev.getOutput output pkg;
@@ -72,7 +83,10 @@
                   }) erisInfo);
             in assert length caps == 1; head caps;
 
-          nixosSystem = { modules, ... }@args:
+          nixosSystem =
+            # A derivative of the function for generating Linux NixOS systems.
+            # This one is not so well tested…
+            { modules, ... }@args:
             import "${nixpkgs}/nixos/lib/eval-config.nix" (args // {
               lib = final;
 
@@ -147,37 +161,38 @@
 
         });
 
-      /* The nixpkgs.legacyPackages after overlaying
-         and with some additional Genode packages.
-      */
-      legacyPackages = let f = import nixpkgs;
-      in forAllSystems ({ system, localSystem, crossSystem }:
-        if localSystem == crossSystem then
-          f {
-            inherit system;
-            overlays = [ self.overlay nimble.overlay ];
-          }
-        else
-          f {
-            inherit localSystem;
-            crossSystem = {
-              system = crossSystem;
-              useLLVM = true;
-            };
-            config.allowUnsupportedSystem = true;
-            overlays = [ self.overlay nimble.overlay ];
-          });
+      legacyPackages =
+        # The nixpkgs.legacyPackages set after overlaying.
+        let f = import nixpkgs;
+        in forAllSystems ({ system, localSystem, crossSystem }:
+          if localSystem == crossSystem then
+            f {
+              inherit system;
+              overlays = [ self.overlay nimble.overlay ];
+            }
+          else
+            f {
+              inherit localSystem;
+              crossSystem = {
+                system = crossSystem;
+                useLLVM = true;
+              };
+              config.allowUnsupportedSystem = true;
+              overlays = [ self.overlay nimble.overlay ];
+            });
 
       packages =
         # Genode native packages, not packages in the traditional
-        # sense in that these cannot be installed within a profile
+        # sense in that these cannot be installed within a profile.
         forAllCrossSystems ({ system, localSystem, crossSystem }:
           nixpkgs.lib.filterAttrs (n: v: v != null)
           self.legacyPackages.${system}.genodePackages);
 
       devShell =
         # Development shell for working with the
-        # upstream Genode source repositories
+        # upstream Genode source repositories. Some
+        # things are missing but everything referred
+        # to by way of #!/usr/bin/ should be here.
         forAllLocalSystems (system:
           let
             pkgs = nixpkgs.legacyPackages.${system};
@@ -217,11 +232,11 @@
           });
 
       nixosModules =
-        # Modules for composing Genode and NixOS
+        # Modules for composing Genode and NixOS.
         import ./nixos-modules { flake = self; };
 
       checks =
-        # Checks for continous testing
+        # Checks for continous testing.
         let tests = import ./tests;
         in with (forAllCrossSystems ({ system, localSystem, crossSystem }:
           tests {

overlay/default.nix

@@ -28,7 +28,7 @@ let
     (attrs: { nativeBuildInputs = [ final.autoreconfHook ]; });
 
   nullPkgs =
-    # Nullify this packages to find problems early
+    # Nullify these packages to find problems early.
     if hostPlatform.isGenode then
       builtins.listToAttrs (map (name: {
         inherit name;
@@ -46,23 +46,28 @@ in nullPkgs // {
   }) bash;
 
   binutils-unwrapped = overrideAttrsTarget (attrs: {
-    patches = attrs.patches ++ [ ./binutils/support-genode.patch ];
+    patches = attrs.patches ++ [
+      ./binutils/support-genode.patch
+      # Upstreamed, remove at next release.
+    ];
     nativeBuildInputs = attrs.nativeBuildInputs
       ++ [ final.updateAutotoolsGnuConfigScriptsHook ];
   }) binutils-unwrapped;
 
-  cmake = overrideAttrsTarget (attrs: {
-    postInstall = with stdenv; ''
-      local MODULE="$out/share/cmake-${
-        lib.versions.majorMinor attrs.version
-      }/Modules/Platform/Genode.cmake"
-      if [ -e "$MODULE" ]; then
-          echo "Upstream provides $MODULE!"
-          exit 1
-      fi
-      cp ${./cmake/Genode.cmake} $MODULE
-    '';
-  }) cmake;
+  cmake =
+    # TODO: upstream
+    overrideAttrsTarget (attrs: {
+      postInstall = with stdenv; ''
+        local MODULE="$out/share/cmake-${
+          lib.versions.majorMinor attrs.version
+        }/Modules/Platform/Genode.cmake"
+        if [ -e "$MODULE" ]; then
+            echo "Upstream provides $MODULE!"
+            exit 1
+        fi
+        cp ${./cmake/Genode.cmake} $MODULE
+      '';
+    }) cmake;
 
   coreutils = overrideHost {
     gmp = null;
@@ -85,24 +90,26 @@ in nullPkgs // {
     });
   };
 
-  gccForLibs = if targetPlatform.isGenode then
-    final.genodePackages.genodeSources.toolchain.cc
-  else
-    prev.gccForLibs;
-
-  gdb = addPatchesTarget [ ./gdb/genode.patch ] gdb;
+  gdb = addPatchesTarget [
+    ./gdb/genode.patch
+    # Upstreamed, remove at next release.
+  ] gdb;
 
   genodeLibcCross = callPackage ./libc { };
 
-  genodePackages = import ../packages {
-    inherit flake;
-    pkgs = final;
-  };
+  genodePackages =
+    # The Genode-only packages.
+    import ../packages {
+      inherit flake;
+      pkgs = final;
+    };
 
-  grub2 = if stdenv.targetPlatform.isGenode then
-    prev.buildPackages.grub2
-  else
-    prev.grub2;
+  grub2 =
+    # No need for a Genode build of GRUB.
+    if stdenv.targetPlatform.isGenode then
+      prev.buildPackages.grub2
+    else
+      prev.grub2;
 
   libcCrossChooser = name:
     if stdenv.targetPlatform.isGenode then
@@ -111,52 +118,73 @@ in nullPkgs // {
       prev.libcCrossChooser name;
 
   libsodium = overrideAttrsHost (attrs: {
-    patches = (attrs.patches or [ ]) ++ [ ./libsodium/genode.patch ];
+    patches = (attrs.patches or [ ]) ++ [
+      ./libsodium/genode.patch
+      # https://github.com/jedisct1/libsodium/pull/1006
+    ];
   }) libsodium;
 
-  libkrb5 = autoreconfHost libkrb5;
+  libkrb5 =
+    # Do not want.
+    autoreconfHost libkrb5;
 
-  libtool = overrideAttrsTarget (attrs: {
-    nativeBuildInputs = with final;
-      attrs.nativeBuildInputs ++ [ autoconf automake115x ];
-    patches = ./libtool/genode.patch;
-  }) libtool;
+  libtool =
+    # Autotools related nonesense. Better to compile
+    # everything static than to deal with this one.
+    overrideAttrsTarget (attrs: {
+      nativeBuildInputs = with final;
+        attrs.nativeBuildInputs ++ [ autoconf automake115x ];
+      patches = ./libtool/genode.patch;
+    }) libtool;
 
   libtoxcore = overrideHost {
     libopus = null;
     libvpx = null;
   } libtoxcore;
 
-  linuxPackages = if hostPlatform.isGenode then {
-    extend = _: final.linuxPackages;
-    features = { };
-    kernel.config = {
-      isEnabled = _: false;
-      isYes = _: false;
-    };
-  } else
-    linuxPackages;
-
-  llvmPackages_11 = callPackage ./llvm-11 ({
-    inherit (stdenvAdapters) overrideCC;
-    buildLlvmTools = buildPackages.llvmPackages_11.tools;
-    targetLlvmLibraries = targetPackages.llvmPackages_11.libraries;
-  } // lib.optionalAttrs
-    (stdenv.hostPlatform.isi686 && buildPackages.stdenv.cc.isGNU) {
-      stdenv = gcc7Stdenv;
-    });
-
-  ncurses = addPatchesHost [ ./ncurses/genode.patch ] ncurses;
-
-  nim = overrideAttrsTarget (attrs: {
-    postInstall = ''
-      sed \
-        -e '/cc = gcc/d' \
-        -i $out/etc/nim/nim.cfg
-    '';
-  }) prev.nim;
-
-  nim-unwrapped = addPatchesTarget [ ./nim/genode.patch ] prev.nim-unwrapped;
+  linuxPackages =
+    # Dummy package.
+    if hostPlatform.isGenode then {
+      extend = _: final.linuxPackages;
+      features = { };
+      kernel.config = {
+        isEnabled = _: false;
+        isYes = _: false;
+      };
+    } else
+      linuxPackages;
+
+  llvmPackages_11 =
+    # A copy of the LLVM expressions from Nixpkgs.
+    # Nothing here is upstreamed.
+    callPackage ./llvm-11 ({
+      inherit (stdenvAdapters) overrideCC;
+      buildLlvmTools = buildPackages.llvmPackages_11.tools;
+      targetLlvmLibraries = targetPackages.llvmPackages_11.libraries;
+    } // lib.optionalAttrs
+      (stdenv.hostPlatform.isi686 && buildPackages.stdenv.cc.isGNU) {
+        stdenv = gcc7Stdenv;
+      });
+
+  ncurses =
+    # https://invisible-island.net/autoconf/
+    # Stay clear of upstream on this one.
+    addPatchesHost [ ./ncurses/genode.patch ] ncurses;
+
+  nim =
+    # Nim is configured to build Genode with GCC.
+    overrideAttrsTarget (attrs: {
+      postInstall = ''
+        sed \
+          -e '/cc = gcc/d' \
+          -i $out/etc/nim/nim.cfg
+      '';
+    }) prev.nim;
+
+  nim-unwrapped = addPatchesTarget [
+    ./nim/genode.patch
+    # Fixes to the compiler and standard libary.
+  ] prev.nim-unwrapped;
 
   openssl =
     overrideHost { static = true; } # shared library comes out stupid big
@@ -173,7 +201,10 @@ in nullPkgs // {
         "rm $out/bin/c_rehash"; # eliminate the perl runtime dependency
     }) openssl);
 
-  patchelf = addPatchesTarget [ ./patchelf/dynstr.patch ] prev.patchelf;
+  patchelf = addPatchesTarget [
+    ./patchelf/dynstr.patch
+    # Patch to fix a bug in rewriting the .dynstr section.
+  ] prev.patchelf;
 
   rsync = overrideHost {
     enableACLs = false;
@@ -190,7 +221,10 @@ in nullPkgs // {
   tor = overrideAttrsHost (attrs: {
     configureFlags = attrs.configureFlags or [ ]
       ++ [ "--disable-tool-name-check" ];
-    patches = attrs.patches or [ ] ++ [ ./tor/genode.patch ];
+    patches = attrs.patches or [ ] ++ [
+      ./tor/genode.patch
+      # We don't do users and groups here.
+    ];
     postPatch = null; # Avoid torsocks patching
   }) tor;
 

packages/genodelabs/default.nix

@@ -13,15 +13,18 @@ let
     else
       throw "unknown Genode arch for platform ${platform.system}";
 
-  upstreamSources = pkgs.fetchFromGitHub {
-    owner = "genodelabs";
-    repo = "genode";
-    rev = "3fac8b106d83721914797c202793ec1d8ea02d2f";
-    hash = "sha256-XgN1fBUsmX8oKk4ZBvROwEWlpILRlJz+UuK4kMDSI1Y=";
-  };
+  upstreamSources =
+    # This is where the Genode source tree is defined.
+    # Must be updated with ./patches/sources.patch.
+    pkgs.fetchFromGitHub {
+      owner = "genodelabs";
+      repo = "genode";
+      rev = "3fac8b106d83721914797c202793ec1d8ea02d2f";
+      hash = "sha256-XgN1fBUsmX8oKk4ZBvROwEWlpILRlJz+UuK4kMDSI1Y=";
+    };
 
   genodeSources =
-    # The Genode source repository
+    # The Genode source repository after patching.
     let
       toolPrefix = if platform.isx86 then
         "genode-x86-"
@@ -56,10 +59,16 @@ let
       installPhase = "cp -a . $out";
     };
 
-  portVersions = import "${genodeSources}/ports.nix";
+  portVersions =
+    # Port versions are taken from the sources to force
+    # updates of the port fixed-output derivations.
+    import "${genodeSources}/ports.nix";
 
   preparePort =
-    # Prepare a "port" of source code declared in the Genode sources
+    # Prepare a "port" of source code declared in the Genode sources.
+    # This is fragile because breakage can appear when the packages
+    # used in preparation are updated, but previously successful
+    # builds will cache.
     name:
     { hash ? "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
     , patches ? [ ], extraRepos ? [ ], ... }@args:
@@ -122,20 +131,26 @@ let
       dontFixup = true;
     });
 
-  ports = lib.mapAttrs preparePort (import ./ports.nix {
-    pkgs = flake.inputs.nixpkgs.legacyPackages.x86_64-linux // {
-      inherit (pkgs) genodePackages;
-    };
-  });
-  # The "ports" mechanism is hardly deterministic, so prepare with
-  # a pinned nixpkgs revision for a pinned platform.
+  ports =
+    # The "ports" mechanism is hardly deterministic, so prepare with
+    # a pinned nixpkgs revision for a pinned platform for consistency.
+    lib.mapAttrs preparePort (import ./ports.nix {
+      pkgs = flake.inputs.nixpkgs.legacyPackages.x86_64-linux // {
+        inherit (pkgs) genodePackages;
+      };
+    });
 
-  toolchain = buildPackages.buildPackages.callPackage ./toolchain.nix { };
+  toolchain =
+    # Patched GCC build from upstream.
+    buildPackages.buildPackages.callPackage ./toolchain.nix { };
 
-  stdenv' = pkgs.stdenvAdapters.overrideCC pkgs.stdenv toolchain;
+  stdenv' =
+    # Special stdenv for use within the upstream sources.
+    # TODO: build with Clang.
+    pkgs.stdenvAdapters.overrideCC pkgs.stdenv toolchain;
 
   buildUpstream =
-    # Build from the Genode sources
+    # Build from the Genode sources using the least recursive make.
     { name, targets, portInputs ? [ ], nativeBuildInputs ? [ ], patches ? [ ]
     , enableParallelBuilding ? true, meta ? { }, ... }@extraAttrs:
     let havePatches = patches != [ ];
@@ -185,7 +200,7 @@ let
     });
 
   buildDepot =
-    # Build a Depot target from the Genode sources
+    # Build from the Genode sources using the depot build system.
     # WARNING: buildDepot can produce artifacts with broken linkage
     # to their inputs. The Genode depot mechanism links programs and
     # libraries to facsimilie stub libraries which are not guaranteed
@@ -209,8 +224,7 @@ let
         enableParallelBuilding = true;
 
         nativeBuildInputs = with buildPackages.buildPackages;
-          [ binutils bison flex stdenv.cc tcl which ]
-          ++ nativeBuildInputs
+          [ binutils bison flex stdenv.cc tcl which ] ++ nativeBuildInputs
           ++ lib.optional (!stdenv.hostPlatform.isGenode) erisPatchHook;
 
         buildInputs = buildInputs ++ depotInputs';
@@ -277,15 +291,18 @@ let
       });
     in self;
 
-  makePackages = let
-    overrides = import ./make-targets.nix {
-      inherit (pkgs) buildPackages genodePackages;
-      inherit ports;
-    };
-  in lib.attrsets.mapAttrs
-  (name: value: (buildUpstream ({ inherit name; } // value))) overrides;
+  makePackages =
+    # Build everything in ./make-targets.nix.
+    let
+      overrides = import ./make-targets.nix {
+        inherit (pkgs) buildPackages genodePackages;
+        inherit ports;
+      };
+    in lib.attrsets.mapAttrs
+    (name: value: (buildUpstream ({ inherit name; } // value))) overrides;
 
   depotPackages = lib.attrsets.mapAttrs
+    # Build everything in ./depot-targets.nix.
     (name: value: (buildDepot ({ inherit name; } // value)))
     (import ./depot-targets.nix {
       inherit (pkgs) genodePackages;
@@ -313,7 +330,8 @@ let
     ++ lib.optional isx86_64 "x86_64";
 
   genodeBase =
-    # A package containing the Genode C++ headers, a stub ld.lib.so and vfs.lib.so
+    # A package containing the Genode C++ headers
+    # and a stub ld.lib.so and vfs.lib.so.
     buildUpstream {
       name = "base";
       targets = [ "LIB=vfs" ];
@@ -342,9 +360,15 @@ let
 
 in makePackages // depotPackages // {
 
-  genodeSources = genodeSources // {
-    inherit arch buildUpstream buildDepot genodeBase ports specs toolchain;
-  };
+  genodeSources =
+    # Expose genodeSources and tuck some extras in with it.
+    genodeSources // {
+      inherit arch buildUpstream buildDepot genodeBase ports specs toolchain;
+    };
+
+  # Builds of the Genode base-systems follow.
+  # These contain the hardware and kernel specific core program,
+  # the loader and base-library, and a timer driver.
 
   base-hw-pc = buildUpstream {
     name = "base-hw-pc";

packages/genodelabs/toolchain.nix

@@ -1,11 +1,6 @@
 # Shameless plagiarism of Blitz's toolchain expression:
 # https://github.com/blitz/genode-nix
 
-#
-# WARNING: these binaries are from sourceforge and
-# have not been publicly verified by Genode Labs.
-#
-
 { lib, stdenv, fetchurl, ncurses5, expat, makeWrapper, wrapCC }:
 
 let

tests/default.nix

@@ -5,53 +5,59 @@ let
   nixpkgs = flake.legacyPackages.${system};
   legacyPackages = flake.legacyPackages.${system};
 
-  testingPython = import ./lib/testing-python.nix;
+  testingPython =
+    # Mostly lifted from Nixpkgs.
+    import ./lib/testing-python.nix;
 
-  testSpace = lib.cartesianProductOfSets {
+  testSpace =
+    # Run all tests on all defined Genode platforms
+    lib.cartesianProductOfSets {
 
-    test = map (p: import p) [
-      ./ahci.nix
-      ./bash.nix
-      ./hello.nix
-      ./log.nix
-      ./nim.nix
-      ./vmm_x86.nix
-    ];
-
-    core = builtins.filter (core:
-      builtins.any (x: x == pkgs.stdenv.hostPlatform.system) core.platforms) [
-        /* {
-             prefix = "hw-pc-";
-             testingPython = testingPython {
-               inherit flake system localSystem crossSystem pkgs;
-               extraConfigurations = [ ../nixos-modules/base-hw-pc.nix ];
-             };
-             specs = [ "x86" "hw" ];
-             platforms = [ "x86_64-genode" ];
-           }
-        */
-        /* {
-             prefix = "hw-virt_qemu-";
-             testingPython = testingPython {
-               inherit flake system localSystem crossSystem pkgs;
-               extraConfigurations = [ ../nixos-modules/base-hw-virt_qemu.nix ];
-             };
-             specs = [ "aarch64" "hw" ];
-             platforms = [ "aarch64-genode" ];
-           }
-        */
-        {
-          prefix = "nova-";
-          testingPython = testingPython {
-            inherit flake system localSystem crossSystem pkgs;
-            extraConfigurations = [ ../nixos-modules/nova.nix ];
-          };
-          specs = [ "x86" "nova" ];
-          platforms = [ "x86_64-genode" ];
-        }
+      test = map (p: import p) [
+        ./ahci.nix
+        ./bash.nix
+        ./hello.nix
+        ./log.nix
+        ./nim.nix
+        ./vmm_x86.nix
       ];
 
-  };
+      core = builtins.filter (core:
+        builtins.any (x: x == pkgs.stdenv.hostPlatform.system) core.platforms) [
+          /* # Need to fix the QEMU boot parameters?
+             {
+               prefix = "hw-pc-";
+               testingPython = testingPython {
+                 inherit flake system localSystem crossSystem pkgs;
+                 extraConfigurations = [ ../nixos-modules/base-hw-pc.nix ];
+               };
+               specs = [ "x86" "hw" ];
+               platforms = [ "x86_64-genode" ];
+             }
+          */
+          /* # Need to fix the QEMU boot parameters?
+             {
+               prefix = "hw-virt_qemu-";
+               testingPython = testingPython {
+                 inherit flake system localSystem crossSystem pkgs;
+                 extraConfigurations = [ ../nixos-modules/base-hw-virt_qemu.nix ];
+               };
+               specs = [ "aarch64" "hw" ];
+               platforms = [ "aarch64-genode" ];
+             }
+          */
+          {
+            prefix = "nova-";
+            testingPython = testingPython {
+              inherit flake system localSystem crossSystem pkgs;
+              extraConfigurations = [ ../nixos-modules/nova.nix ];
+            };
+            specs = [ "x86" "nova" ];
+            platforms = [ "x86_64-genode" ];
+          }
+        ];
+
+    };
 
   testList = let
     f = { core, test }: