diff --git a/README.md b/README.md index 3ac2570..1aeda0f 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,66 @@ nix build .#checks.x86_64-linux-x86_64-genode.nova-solo5-net.iso nix build .#checks.x86_64-linux-x86_64-genode.nova-solo5-net.sotest ``` +# FAQ + +## How to build a Genode make target? + +```nix +let flake = getFlake "genodepkgs"; +in flake.packages.x86_64-linux-x86_64-genode.genodeSources.make "app/ping" +``` + +## How to build a Genode depot package? + +```nix +let flake = getFlake "genodepkgs"; +in flake.packages.x86_64-linux-x86_64-genode.genodeSources.depot "wifi_drv" +``` + +## How to add missing ports to a depot package? + +The missing ports as indicated by a depot build failure must be added to +[targets.nix](./overlay/genode/targets.nix). To continue from the previous +example, this would be sufficient: + +```nix +{ + wifi_drv = { portInputs = [ dde_linux libc openssl ]; }; +} +``` + +## How to add undefined ports? + +Should a required port not be defined, it must be added to +[ports.nix](./overlay/genode/ports.nix). Initially it should be sufficient to +define the port as an empty attribute set, run the build again, and then add the +hash found in the successive build error. + +To continue from the previous example: + +```nix +{ + dde_linux = { }; +} +``` + +And after the reattempting the build a fixed-output hash should be produced, +this must now be added to the port definitions. + +```nix +{ + dde_linux = { + hash = "sha256-3G5Mprv9ZBNhDJRuVF6Nrkp3KbFaN10IKFq0OdyAY9M="; }; +} +``` + +Should the port preparation fail do to missing utilities, these must be added to +the port definition using a `nativeBuildInputs` attribute. + +Note that the hash checked into `ports.nix` is an *output hashe*. The port hash +within the Genode source tree is an *explicit input hash* and the output of +the port preparation is non-deterministic. + # Tips and tricks ## Render a Dhall Init configuration to XML diff --git a/overlay/default.nix b/overlay/default.nix index 07a11a2..46b0056 100644 --- a/overlay/default.nix +++ b/overlay/default.nix @@ -29,7 +29,7 @@ with prev; { genodePackages = import ../packages { nixpkgs = final; }; - genodeSources = buildPackages.callPackage ./genode { }; + genodeSources = buildPackages.callPackage ../packages/genodelabs { }; libcCrossChooser = name: if stdenv.targetPlatform.isGenode then diff --git a/overlay/genode/part_block.patch b/overlay/genode/part_block.patch deleted file mode 100644 index efa5ff6..0000000 --- a/overlay/genode/part_block.patch +++ /dev/null @@ -1,23 +0,0 @@ -commit 1b8f44e80801bbb8d3dba813b72eed4c6a5a5ca6 -Author: Emery Hemingway -Date: Fri Mar 20 20:10:28 2020 +0530 - - part_block: support for up to 128 session - - Allow sessions for each of the 128 sessions supported by GPT. - - Fix #3705 - -diff --git a/repos/os/src/server/part_block/main.cc b/repos/os/src/server/part_block/main.cc -index a446be31db..56bade6086 100644 ---- a/repos/os/src/server/part_block/main.cc -+++ b/repos/os/src/server/part_block/main.cc -@@ -226,7 +226,7 @@ class Block::Main : Rpc_object>, - Gpt _gpt { _env, _block, _heap, _reporter }; - Partition_table &_partition_table { _table() }; - -- enum { MAX_SESSIONS = 32 }; -+ enum { MAX_SESSIONS = 128 }; - Session_component *_sessions[MAX_SESSIONS] { }; - Job_queue<128> _job_queue { }; - Registry _job_registry { }; diff --git a/overlay/genode/targets.nix b/overlay/genode/targets.nix deleted file mode 100644 index c6887d7..0000000 --- a/overlay/genode/targets.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ buildPackages, ports }: -with ports; { - "app/gpt_write".portInputs = [ jitterentropy ]; - - "app/lighttpd".portInputs = [ libc lighttpd openssl zlib ]; - - "drivers/framebuffer/intel" = { - BOARD = "pc"; - portInputs = [ dde_linux ]; - }; - - "drivers/framebuffer/vesa" = { - BOARD = "pc"; - portInputs = [ x86emu ]; - }; - - "drivers/usb".portInputs = [ dde_linux ]; - - "lib/vfs/lwip".portInputs = [ lwip ]; - "lib/vfs/ttf".portInputs = [ libc stb ]; - "lib/vfs/rump" = { - portInputs = [ dde_rump ]; - buildInputs = with buildPackages; [ zlib ]; - }; - - noux = { - targets = [ "noux" "lib/libc_noux" ]; - portInputs = [ libc ]; - }; - - virtualbox5 = { - KERNEL = "nova"; - portInputs = [ libc libiconv qemu-usb stdcxx virtualbox5 ]; - nativeBuildInputs = with buildPackages; [ iasl yasm ]; - }; - -} diff --git a/packages/default.nix b/packages/default.nix index 5553c2a..a57ccb0 100644 --- a/packages/default.nix +++ b/packages/default.nix @@ -9,7 +9,7 @@ let dhallPackages = buildPackages.callPackage ./dhall { }; - buildWorld = let + buildDepotWorld = let genodeWorld = fetchFromGitHub { owner = "genodelabs"; repo = "genode-world"; @@ -17,14 +17,27 @@ let hash = "sha256-sirmUtLmZ5YnfLKrOvOBafnZW3UW+1LtkiGu85Ma820="; }; in attrs: - genodeSources.buildUpstream (attrs // { - postConfigure = - "echo REPOSITORIES += ${genodeWorld} >> build/etc/build.conf"; + genodeSources.buildDepot (attrs // { + postConfigure = '' + cp -r --no-preserve=mode ${genodeWorld} $GENODE_DIR/repos + ''; }); genodeTupRules = ./Tuprules.tup; -in rec { + depotPkgs = with builtins; + let + names = attrNames (import ./genodelabs/targets.nix { + buildPackages = null; + ports = null; + }); + f = name: { + inherit name; + value = genodeSources.depot name; + }; + in listToAttrs (map f names); + +in depotPkgs // { inherit stdenv genodeSources; base-hw-pc = buildUpstream { @@ -65,14 +78,14 @@ in rec { ''; }; - init = genodeSources.make "init"; - bender = buildPackages.callPackage ./bender { }; block_router = callPackage ./block_router { }; dhallGenode = dhallPackages.genode; + init = genodeSources.depot "init"; + nic_bus = callPackage ./nic_bus { }; NOVA = callPackage ./NOVA { }; @@ -82,9 +95,8 @@ in rec { sotest-producer = callPackage ./sotest-producer { }; - ssh_client = buildWorld { + ssh_client = buildDepotWorld { name = "ssh_client"; - targets = [ "app/ssh_client" ]; portInputs = with genodeSources.ports; [ libc libssh openssl zlib ]; }; diff --git a/packages/genodelabs/cxx-align.patch b/packages/genodelabs/cxx-align.patch new file mode 100644 index 0000000..c80dea9 --- /dev/null +++ b/packages/genodelabs/cxx-align.patch @@ -0,0 +1,57 @@ +commit 0df2ea0956e46f3914be01aa9d7b20a06d805e53 +Author: Emery Hemingway +Date: Wed Apr 8 12:39:44 2020 +0530 + + cxx: align exception allocator to sixteen bytes + +diff --git a/repos/base/src/lib/cxx/malloc_free.cc b/repos/base/src/lib/cxx/malloc_free.cc +index b551dfe627..870dbb6c6d 100644 +--- a/repos/base/src/lib/cxx/malloc_free.cc ++++ b/repos/base/src/lib/cxx/malloc_free.cc +@@ -61,27 +61,23 @@ void Genode::init_cxx_heap(Env &env) + } + + +-typedef unsigned long Block_header; ++typedef size_t Block_header; + + + extern "C" void *malloc(size_t size) + { +- /* enforce size to be a multiple of 4 bytes */ +- size = (size + 3) & ~3; +- + /* +- * We store the size of the allocation at the very +- * beginning of the allocated block and return +- * the subsequent address. This way, we can retrieve +- * the size information when freeing the block. ++ * We pad each allocation with 16 leading bytes for ++ * storing the size of the allocation. This way, we can ++ * retrieve the size information when freeing the block. + */ +- unsigned long real_size = size + sizeof(Block_header); +- void *addr = 0; +- if (!cxx_heap().alloc(real_size, &addr)) +- return 0; ++ size_t real_size = size + 16; ++ addr_t real_addr = 0; ++ if (!cxx_heap().alloc(real_size, (void**)&real_addr)) ++ return nullptr; + +- *(Block_header *)addr = real_size; +- return (Block_header *)addr + 1; ++ *(Block_header *)real_addr = real_size; ++ return (void*)(real_addr + 16); + } + + +@@ -101,7 +97,7 @@ extern "C" void free(void *ptr) + { + if (!ptr) return; + +- unsigned long *addr = ((unsigned long *)ptr) - 1; ++ unsigned long *addr = (unsigned long *)(addr_t(ptr) - 16); + cxx_heap().free(addr, *addr); + } + diff --git a/overlay/genode/default.nix b/packages/genodelabs/default.nix similarity index 61% rename from overlay/genode/default.nix rename to packages/genodelabs/default.nix index b9f7ea6..1166e85 100644 --- a/overlay/genode/default.nix +++ b/packages/genodelabs/default.nix @@ -87,7 +87,7 @@ let ''; installPhase = - # this trash is non-deterministic + # strip non-deterministic and extra artifacts '' runHook preInstall chmod -R +w $CONTRIB_DIR/* @@ -106,46 +106,89 @@ let (import ./ports.nix { inherit buildPackages; }); buildUpstream = { name, targets, portInputs ? [ ], nativeBuildInputs ? [ ] - , ... }@extraAttrs: + , enableParallelBuilding ? true, ... }@extraAttrs: - let - drv = stdenv'.mkDerivation (extraAttrs // { - pname = name; - inherit (genodeSources) version; - inherit targets; + stdenv'.mkDerivation (extraAttrs // { + pname = name; + inherit (genodeSources) version; + inherit targets enableParallelBuilding; - dontUnpack = true; - GENODE_DIR = genodeSources; + dontUnpack = true; + GENODE_DIR = genodeSources; - nativeBuildInputs = [ binutils bison flex stdenv.cc tcl which ] - ++ nativeBuildInputs; + nativeBuildInputs = [ binutils bison flex stdenv.cc tcl which ] + ++ nativeBuildInputs; - enableParallelBuilding = true; + configurePhase = let + linkPorts = toString + (builtins.map (drv: " ln -sv ${drv}/* $CONTRIB_DIR/;") portInputs); + in '' + runHook preConfigure + export CONTRIB_DIR=$NIX_BUILD_TOP/contrib + export BUILD_DIR=$NIX_BUILD_TOP/build + $GENODE_DIR/tool/create_builddir ${arch} + substituteInPlace build/etc/build.conf \ + --replace "#REPOSITORIES" "REPOSITORIES" + mkdir $CONTRIB_DIR; ${linkPorts} + runHook postConfigure + ''; - configurePhase = let - linkPorts = toString - (builtins.map (drv: " ln -sv ${drv}/* contrib/;") portInputs); - in '' - runHook preConfigure - export CONTRIB_DIR=$NIX_BUILD_TOP/contrib - export BUILD_DIR=$NIX_BUILD_TOP/build - $GENODE_DIR/tool/create_builddir ${arch} - substituteInPlace build/etc/build.conf \ - --replace "#REPOSITORIES" "REPOSITORIES" - mkdir $CONTRIB_DIR; ${linkPorts} - runHook postConfigure - ''; + makeFlags = [ "-C build" ] ++ targets; - makeFlags = [ "-C build" ] ++ targets; + installPhase = '' + runHook preInstall + find build/bin -name '*.xsd' -delete + find build/bin -follow -type f -exec install -Dt $out '{}' \; + runHook postInstall + ''; + }); - installPhase = '' - runHook preInstall - find build/bin -name '*.xsd' -delete - find build/bin -follow -type f -exec install -Dt $out '{}' \; - runHook postInstall - ''; - }); - in drv; + buildDepot = { name, portInputs ? [ ], nativeBuildInputs ? [ ] + , enableParallelBuilding ? true, ... }@extraAttrs: + + stdenv'.mkDerivation (extraAttrs // { + pname = name; + inherit (genodeSources) version; + inherit enableParallelBuilding; + + nativeBuildInputs = [ binutils bison flex stdenv.cc tcl which ] + ++ nativeBuildInputs; + + src = genodeSources; + # The genode source tree must be copied to the build directory + # because the depot tool must modify the source tree as it runs. + + configurePhase = let + copyPorts = # wasteful copy + toString + (builtins.map (drv: " cp -r ${drv}/* $CONTRIB_DIR/;") portInputs); + in '' + runHook preConfigure + export GENODE_DIR=$(pwd) + export CONTRIB_DIR=$GENODE_DIR/contrib + export DEPOT_DIR=$GENODE_DIR/depot + mkdir -p $CONTRIB_DIR; ${copyPorts} + chmod +rwX -R . + runHook postConfigure + ''; + + makefile = "tool/depot/create"; + makeFlags = [ + "genodelabs/bin/${arch}/${name}" + + # by default the build system will refuse to be useful + "FORCE=1" + "UPDATE_VERSIONS=1" + "VERBOSE=" + ]; + + installPhase = '' + runHook preInstall + find depot/genodelabs/bin -type f -exec install -Dt $out {} \; + runHook postInstall + ''; + + }); buildOverrides = callPackage ./targets.nix { inherit ports; }; @@ -195,18 +238,16 @@ let }; in genodeSources // { - inherit buildUpstream ports specs toolchain genodeBase; + inherit buildUpstream buildDepot ports specs toolchain genodeBase; make = target: - with builtins; - let - attrs = if hasAttr target buildOverrides then - getAttr target buildOverrides - else - { }; - in buildUpstream ({ - name = replaceStrings [ "/" ] [ "-" ] target; + buildUpstream { + name = builtins.replaceStrings [ "/" ] [ "-" ] target; targets = [ target ]; - } // attrs); + }; + + depot = name: + let attrs = buildOverrides.${name} or { }; + in buildDepot ({ inherit name; } // attrs); } diff --git a/overlay/genode/init.xsd.patch b/packages/genodelabs/init.xsd.patch similarity index 100% rename from overlay/genode/init.xsd.patch rename to packages/genodelabs/init.xsd.patch diff --git a/overlay/genode/openssl.patch b/packages/genodelabs/openssl.patch similarity index 100% rename from overlay/genode/openssl.patch rename to packages/genodelabs/openssl.patch diff --git a/overlay/genode/ports.nix b/packages/genodelabs/ports.nix similarity index 100% rename from overlay/genode/ports.nix rename to packages/genodelabs/ports.nix diff --git a/overlay/genode/svn-trust-server-cert.patch b/packages/genodelabs/svn-trust-server-cert.patch similarity index 100% rename from overlay/genode/svn-trust-server-cert.patch rename to packages/genodelabs/svn-trust-server-cert.patch diff --git a/packages/genodelabs/targets.nix b/packages/genodelabs/targets.nix new file mode 100644 index 0000000..2a15e53 --- /dev/null +++ b/packages/genodelabs/targets.nix @@ -0,0 +1,41 @@ +{ buildPackages, ports }: +with ports; { + + gpt_write.portInputs = [ jitterentropy ]; + + intel_fb_drv = { + BOARD = "pc"; + portInputs = [ dde_linux ]; + }; + + libc.portInputs = [ libc ]; + + lighttpd.portInputs = [ libc lighttpd openssl zlib ]; + + noux.portInputs = [ libc ]; + + posix.portInputs = [ libc ]; + + rump = { + portInputs = [ dde_rump ]; + buildInputs = with buildPackages; [ zlib ]; + }; + + usb_drv.portInputs = [ dde_linux ]; + + vbox5 = { + KERNEL = "nova"; + portInputs = [ libc libiconv qemu-usb stdcxx virtualbox5 ]; + nativeBuildInputs = with buildPackages; [ iasl yasm ]; + enableParallelBuilding = false; + }; + + vesa_drv.portInputs = [ libc x86emu ]; + + vfs_jitterentropy.portInputs = [ jitterentropy libc ]; + vfs_lwip.portInputs = [ lwip ]; + vfs_ttf.portInputs = [ libc stb ]; + + wifi_drv.portInputs = [ dde_linux libc openssl ]; + +} diff --git a/overlay/genode/toolchain.nix b/packages/genodelabs/toolchain.nix similarity index 100% rename from overlay/genode/toolchain.nix rename to packages/genodelabs/toolchain.nix diff --git a/tests/block_router.nix b/tests/block_router.nix index 3971ada..513a2bd 100644 --- a/tests/block_router.nix +++ b/tests/block_router.nix @@ -8,11 +8,9 @@ testEnv.mkTest { meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; testConfig = ./block_router.dhall; - testInputs = map pkgs.genodeSources.make [ - "server/part_block" - "server/ram_block" - "test/block/client" - ] ++ [ pkgs.block_router ]; + testInputs = + map pkgs.genodeSources.depot [ "part_block" "ram_block" "test-block" ] + ++ [ pkgs.block_router ]; testScript = '' catch { exec dd if=/dev/zero of=gpt.raw bs=512 count=7168 } exec ${buildPackages.gptfdisk}/bin/sgdisk -o -n 3:2048:4095 -c 3:first-test-partition -n 31:4096:6143 -c 31:second-test-partition gpt.raw diff --git a/tests/driver-hw.nix b/tests/driver-hw.nix index 2e4249a..8596203 100644 --- a/tests/driver-hw.nix +++ b/tests/driver-hw.nix @@ -40,9 +40,8 @@ let mkTest = { name ? "unamed", testScript ? defaultScript, testConfig , testInputs ? [ ], testEnv ? { }, qemuArgs ? [ ], ... }@t: let - manifest = lib.mergeManifests (map addManifest (with testPkgs; - [ base-hw-pc (genodeSources.make "init") sotest-producer ] - ++ testInputs)); + manifest = lib.mergeManifests (map addManifest + (with testPkgs; [ base-hw-pc init sotest-producer ] ++ testInputs)); testConfig' = "${./test-wrapper.dhall} ${testConfig} (toMap ${manifest})"; testEnv' = { DHALL_GENODE = "${testPkgs.dhallGenode}/source.dhall"; diff --git a/tests/driver_manager.nix b/tests/driver_manager.nix index 73be7bc..422bf49 100644 --- a/tests/driver_manager.nix +++ b/tests/driver_manager.nix @@ -7,23 +7,22 @@ testEnv.mkTest { name = "driver_manager"; meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; testEnv = { drivers = ./../compositions/pc-drivers.dhall; }; - testInputs = (map pkgs.genodeSources.make [ - "app/driver_manager" - "app/rom_reporter" - "drivers/acpi" - "drivers/ahci" - "drivers/framebuffer/boot" - "drivers/framebuffer/intel" - "drivers/framebuffer/vesa" - "drivers/input/spec/ps2" - "drivers/platform" - "drivers/rtc" - "drivers/usb" - "server/dynamic_rom" - "server/input_filter" - "server/report_rom" - "test/driver_manager" - ]); + testInputs = (map pkgs.genodeSources.depot [ + "acpi_drv" + "ahci_drv" + "boot_fb_drv" + "driver_manager" + "dynamic_rom" + "input_filter" + "intel_fb_drv" + "platform_drv" + "ps2_drv" + "report_rom" + "rom_reporter" + "rtc_drv" + "usb_drv" + "vesa_drv" + ]) ++ (map pkgs.genodeSources.make [ "test/driver_manager" ]); testScript = '' catch { exec dd if=/dev/zero of=hdd_disk.raw bs=1M count=32 } diff --git a/tests/log.nix b/tests/log.nix index 94fdd2c..ff3b835 100644 --- a/tests/log.nix +++ b/tests/log.nix @@ -8,6 +8,6 @@ testEnv.mkTest rec { meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; testConfig = ./log.dhall; - testInputs = [ (pkgs.genodeSources.make "test/log") ]; + testInputs = [ (pkgs.genodeSources.depot "test-log") ]; testScript = "run_genode_until {Test done.} 120"; } diff --git a/tests/noux.nix b/tests/noux.nix index e0e8205..2246c99 100644 --- a/tests/noux.nix +++ b/tests/noux.nix @@ -6,7 +6,7 @@ with pkgs; testEnv.mkTest rec { name = "noux"; testConfig = ./noux.dhall; - testInputs = [ (pkgs.genodeSources.make "noux") ] + testInputs = map pkgs.genodeSources.depot [ "noux" "vfs" "libc" "posix" ] ++ (with depot; [ bash-minimal ncurses ]); meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; testScript = "run_genode_until {SOTEST SUCCESS} 40"; diff --git a/tests/pci.nix b/tests/pci.nix index 4341d0a..73c62a0 100644 --- a/tests/pci.nix +++ b/tests/pci.nix @@ -8,12 +8,9 @@ testEnv.mkTest { meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; testConfig = ./pci.dhall; - testInputs = map pkgs.genodeSources.make [ - "drivers/acpi" - "drivers/platform" - "server/report_rom" - "test/pci" - ]; + testInputs = + (map pkgs.genodeSources.depot [ "acpi_drv" "platform_drv" "report_rom" ]) + ++ (map pkgs.genodeSources.make [ "test/pci" ]); testScript = '' run_genode_until ".*--- Platform test finished ---.*\n" 60 ''; diff --git a/tests/rtc.nix b/tests/rtc.nix index 5f4e1e5..fea9572 100644 --- a/tests/rtc.nix +++ b/tests/rtc.nix @@ -8,6 +8,6 @@ testEnv.mkTest { meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; testConfig = ./rtc.dhall; - testInputs = map pkgs.genodeSources.make [ "drivers/rtc" "test/rtc" ]; + testInputs = map pkgs.genodeSources.depot [ "rtc_drv" "test-rtc" ]; testScript = "run_genode_until {--- RTC test finished ---} 40"; } diff --git a/tests/signal.nix b/tests/signal.nix index 1cc4816..39d5584 100644 --- a/tests/signal.nix +++ b/tests/signal.nix @@ -8,6 +8,6 @@ testEnv.mkTest rec { meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; testConfig = ./signal.dhall; - testInputs = [ (pkgs.genodeSources.make "test/signal") ]; + testInputs = [ (pkgs.genodeSources.depot "test-signal") ]; testScript = "run_genode_until {--- Signalling test finished ---} 120"; } diff --git a/tests/solo5/default.nix b/tests/solo5/default.nix index c2b42ed..c1bb6be 100644 --- a/tests/solo5/default.nix +++ b/tests/solo5/default.nix @@ -25,13 +25,14 @@ let toSimple = name: ''(${./simple.dhall} \"solo5-test_${name}\")''; - makeGenode = pkgs.genodeSources.make; + genodeDepot = pkgs.genodeSources.depot; + genodeMake = pkgs.genodeSources.make; tests = [ { name = "blk"; testConfig = ./blk.dhall; - testInputs = [ (makeGenode "server/ram_block") ]; + testInputs = [ (genodeDepot "ram_block") ]; } { @@ -52,19 +53,15 @@ let { name = "net"; testConfig = ./net.dhall; - testInputs = - map makeGenode [ "app/ping" "server/nic_bridge" "server/nic_loopback" ]; + testInputs = (map genodeDepot [ "nic_bridge" "nic_loopback" ]) + ++ (map genodeMake [ "app/ping" ]); } { name = "net_2if"; testConfig = ./net_2if.dhall; - testInputs = map makeGenode [ - "app/ping" - "app/sequence" - "server/nic_bridge" - "server/nic_loopback" - ]; + testInputs = (map genodeDepot [ "sequence" "nic_bridge" "nic_loopback" ]) + ++ (map genodeMake [ "app/ping" ]); } { @@ -83,7 +80,7 @@ let ] ++ (testEnv.lib.optional (!testEnv.isLinux) { name = "time"; testConfig = ./time.dhall; - testInputs = [ (makeGenode "drivers/rtc") ]; + testInputs = [ (genodeDepot "rtc_drv") ]; } );