Browse Source

Port NixOS module system

Convert the tests to use the module system from NixOS.
posix
Emery Hemingway 5 months ago
parent
commit
ebf3606705

+ 7
- 12
flake.nix View File

@@ -65,9 +65,7 @@
65 65
         forAllCrossSystems ({ system, localSystem, crossSystem }:
66 66
           nixpkgs.lib // (import ./lib {
67 67
             inherit system localSystem crossSystem;
68
-            localPackages = nixpkgs.legacyPackages.${localSystem};
69
-            genodepkgs = self;
70
-            nixpkgs = nixpkgsFor.${system};
68
+            pkgs = self.legacyPackages.${system};
71 69
           }));
72 70
 
73 71
       legacyPackages =
@@ -139,15 +137,12 @@
139 137
 
140 138
       checks =
141 139
         # Checks for continous testing
142
-        with (forAllCrossSystems ({ system, localSystem, crossSystem }:
143
-          import ./tests {
144
-            inherit self;
145
-            apps = self.apps.${system};
146
-            localPackages = nixpkgsFor.${localSystem};
147
-            genodepkgs = self.packages.${system};
148
-            lib = self.lib.${system};
149
-            nixpkgs = nixpkgsFor.${system};
150
-            legacyPackages = self.legacyPackages.${system};
140
+        let tests = import ./tests;
141
+        in with (forAllCrossSystems ({ system, localSystem, crossSystem }:
142
+          tests {
143
+            flake = self;
144
+            inherit system localSystem crossSystem;
145
+            pkgs = self.legacyPackages.${system};
151 146
           } // {
152 147
             ports = nixpkgsFor.${localSystem}.symlinkJoin {
153 148
               name = "ports";

+ 17
- 17
lib/default.nix View File

@@ -1,22 +1,22 @@
1
-{ system, localSystem, crossSystem, genodepkgs, nixpkgs, localPackages }:
1
+{ system, localSystem, crossSystem, pkgs }:
2 2
 
3 3
 let
4
-  thisSystem = builtins.getAttr system;
5
-  inherit (nixpkgs) buildPackages;
6
-  testPkgs = thisSystem genodepkgs.packages;
4
+  inherit (pkgs) buildPackages;
5
+  localPackages = pkgs.buildPackages.buildPackages;
6
+  inherit (pkgs.genodePackages) dhallGenode genodeSources;
7 7
 
8 8
   dhallCachePrelude = ''
9 9
     export XDG_CACHE_HOME=$NIX_BUILD_TOP
10
-    export DHALL_GENODE="${testPkgs.dhallGenode}/binary.dhall";
10
+    export DHALL_GENODE="${dhallGenode}/binary.dhall";
11 11
     ${buildPackages.xorg.lndir}/bin/lndir -silent \
12
-      ${testPkgs.dhallGenode}/.cache \
12
+      ${dhallGenode}/.cache \
13 13
       $XDG_CACHE_HOME
14 14
   '';
15 15
 
16 16
 in rec {
17 17
 
18 18
   runDhallCommand = name: env: script:
19
-    nixpkgs.runCommand name (env // {
19
+    pkgs.runCommand name (env // {
20 20
       nativeBuildInputs = [ localPackages.dhall ]
21 21
         ++ env.nativeBuildInputs or [ ];
22 22
     }) ''
@@ -42,7 +42,7 @@ in rec {
42 42
   hwImage = coreLinkAddr: bootstrapLinkAddr: basePkg: name:
43 43
     { gzip ? false, ... }@env:
44 44
     boot:
45
-    nixpkgs.stdenv.mkDerivation {
45
+    pkgs.stdenv.mkDerivation {
46 46
       name = name + "-hw-image";
47 47
       build = compileBoot name env boot;
48 48
       nativeBuildInputs = [ localPackages.dhall ];
@@ -74,7 +74,7 @@ in rec {
74 74
         LD="${buildPackages.binutils}/bin/${buildPackages.binutils.targetPrefix}ld"
75 75
           $LD \
76 76
             --strip-all \
77
-            -T${testPkgs.genodeSources}/repos/base/src/ld/genode.ld \
77
+            -T${genodeSources}/repos/base/src/ld/genode.ld \
78 78
             -z max-page-size=0x1000 \
79 79
             -Ttext=$link_address -gc-sections \
80 80
             "$lib" "boot_modules.o" \
@@ -96,13 +96,13 @@ in rec {
96 96
           bootstrap/modules_asm \
97 97
           ${bootstrapLinkAddr} \
98 98
           $out/image.elf
99
-      '' + nixpkgs.lib.optionalString gzip "gzip $out/image.elf";
99
+      '' + pkgs.lib.optionalString gzip "gzip $out/image.elf";
100 100
     };
101 101
 
102 102
   novaImage = name:
103 103
     { gzip ? false, ... }@env:
104 104
     boot:
105
-    nixpkgs.stdenv.mkDerivation {
105
+    pkgs.stdenv.mkDerivation {
106 106
       name = name + "-nova-image";
107 107
       build = compileBoot name env boot;
108 108
 
@@ -115,17 +115,17 @@ in rec {
115 115
         # link final image
116 116
         LD="${buildPackages.binutils}/bin/${buildPackages.binutils.targetPrefix}ld"
117 117
         $LD --strip-all -nostdlib \
118
-        	-T${testPkgs.genodeSources}/repos/base/src/ld/genode.ld \
119
-        	-T${testPkgs.genodeSources}/repos/base-nova/src/core/core-bss.ld \
118
+        	-T${genodeSources}/repos/base/src/ld/genode.ld \
119
+        	-T${genodeSources}/repos/base-nova/src/core/core-bss.ld \
120 120
         	-z max-page-size=0x1000 \
121 121
         	-Ttext=0x100000 -gc-sections \
122
-        	"${testPkgs.base-nova.coreObj}" boot_modules.o \
122
+        	"${pkgs.genodePackages.base-nova.coreObj}" boot_modules.o \
123 123
         	-o $out/image.elf
124
-      '' + nixpkgs.lib.optionalString gzip "gzip $out/image.elf";
124
+      '' + pkgs.lib.optionalString gzip "gzip $out/image.elf";
125 125
     };
126 126
 
127 127
   mergeManifests = inputs:
128
-    nixpkgs.writeTextFile {
128
+    pkgs.writeTextFile {
129 129
       name = "manifest.dhall";
130 130
       text = with builtins;
131 131
         let
@@ -133,7 +133,7 @@ in rec {
133 133
             if hasAttr "manifest" input then
134 134
               ''
135 135
                 ${head}, { mapKey = "${
136
-                  nixpkgs.lib.getName input
136
+                  pkgs.lib.getName input
137 137
                 }", mapValue = ${input.manifest} }''
138 138
             else
139 139
               abort "${input.pname} does not have a manifest";

+ 27
- 0
nixos-modules/base-hw-pc.nix View File

@@ -0,0 +1,27 @@
1
+{ config, pkgs, lib, ... }:
2
+
3
+with lib;
4
+let
5
+  localPackages = pkgs.buildPackages;
6
+  utils = import ../lib {
7
+    inherit (config.nixpkgs) system localSystem crossSystem;
8
+    inherit pkgs;
9
+  };
10
+in {
11
+  genode.core = {
12
+    prefix = "hw-pc-";
13
+    supportedSystems = [ "x86_64-genode" ];
14
+    basePackages = with pkgs.genodePackages; [ base-hw-pc rtc_drv ];
15
+  };
16
+
17
+  genode.boot = {
18
+
19
+    initrd = "${config.genode.boot.image}/image.elf";
20
+
21
+    image = utils.hwImage "0xffffffc000000000" "0x00200000"
22
+      pkgs.genodePackages.base-hw-pc config.system.name { }
23
+      config.genode.boot.configFile;
24
+
25
+  };
26
+
27
+}

+ 27
- 0
nixos-modules/base-hw-virt_qemu.nix View File

@@ -0,0 +1,27 @@
1
+{ config, pkgs, lib, ... }:
2
+
3
+with lib;
4
+let
5
+  localPackages = pkgs.buildPackages;
6
+  utils = import ../lib {
7
+    inherit (config.nixpkgs) system localSystem crossSystem;
8
+    inherit pkgs;
9
+  };
10
+in {
11
+  genode.core = {
12
+    prefix = "hw-virt_qemu";
13
+    supportedSystems = [ "aarch64-genode" ];
14
+    basePackages = with pkgs.genodePackages; [ base-hw-virt_qemu rtc-dummy ];
15
+  };
16
+
17
+  genode.boot = {
18
+
19
+    initrd = "${config.genode.boot.image}/image.elf";
20
+
21
+    image = utils.hwImage "0xffffffc000000000" "0x40000000"
22
+      pkgs.genodePackages.base-hw-virt_qemu config.system.name { }
23
+      config.genode.boot.configFile;
24
+
25
+  };
26
+
27
+}

+ 157
- 0
nixos-modules/genode-core.nix View File

@@ -0,0 +1,157 @@
1
+{ config, pkgs, lib, modulesPath, ... }:
2
+
3
+with lib;
4
+let localPackages = pkgs.buildPackages;
5
+in {
6
+  options.genode = {
7
+    core = {
8
+
9
+      prefix = mkOption {
10
+        type = types.str;
11
+        example = "hw-pc-";
12
+      };
13
+
14
+      supportedSystems = mkOption {
15
+        type = types.listOf types.str;
16
+        example = [ "i686-genode" "x86_64-genode" ];
17
+      };
18
+
19
+      basePackages = mkOption { type = types.listOf types.package; };
20
+
21
+    };
22
+
23
+    boot = {
24
+
25
+      kernel = mkOption {
26
+        type = types.path;
27
+        default = "${pkgs.genodePackages.bender}/bender";
28
+      };
29
+
30
+      initrd = mkOption {
31
+        type = types.str;
32
+        default = "${pkgs.genodePackages.bender}/bender";
33
+        description = "Path to an image or a command-line arguments";
34
+      };
35
+
36
+      configFile = mkOption {
37
+        type = types.path;
38
+        description = ''
39
+          Dhall boot configuration. See
40
+          https://git.sr.ht/~ehmry/dhall-genode/tree/master/Boot/package.dhall
41
+        '';
42
+      };
43
+
44
+      image = mkOption {
45
+        type = types.path;
46
+        description =
47
+          "Boot image containing the base component binaries and configuration.";
48
+      };
49
+
50
+      romModules = mkOption {
51
+        type = types.attrsOf types.path;
52
+        description = "Attr set of initial ROM modules";
53
+      };
54
+
55
+    };
56
+
57
+  };
58
+
59
+  config = let
60
+    initInputs = unique config.genode.init.inputs;
61
+
62
+    addManifest = drv:
63
+      drv // {
64
+        manifest =
65
+          localPackages.runCommand "${drv.name}.dhall" { inherit drv; } ''
66
+            set -eu
67
+            echo -n '[' >> $out
68
+            find $drv/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out
69
+            ${if builtins.elem "lib" drv.outputs then
70
+              ''
71
+                find ${drv.lib}/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out''
72
+            else
73
+              ""}
74
+            echo -n ']' >> $out
75
+          '';
76
+      };
77
+
78
+    mergeManifests = inputs:
79
+      localPackages.writeTextFile {
80
+        name = "manifest.dhall";
81
+        text = with builtins;
82
+          let
83
+            f = head: input:
84
+              if hasAttr "manifest" input then
85
+                ''
86
+                  ${head}, { mapKey = "${
87
+                    lib.getName input
88
+                  }", mapValue = ${input.manifest} }''
89
+              else
90
+                abort "${input.pname} does not have a manifest";
91
+          in (foldl' f "[" inputs) + "]";
92
+      };
93
+
94
+  in {
95
+
96
+    assertions = [{
97
+      assertion = builtins.any (s: s == config.nixpkgs.system)
98
+        config.genode.core.supportedSystems;
99
+      message = "invalid Genode core for this system";
100
+    }];
101
+
102
+    genode.boot.configFile = let
103
+      tarball =
104
+        "${config.system.build.tarball}/tarball/${config.system.build.tarball.fileName}.tar";
105
+      manifest = mergeManifests (map addManifest
106
+        (config.genode.core.basePackages ++ [ config.system.build.tarball ]
107
+          ++ (with pkgs.genodePackages; [ init cached_fs_rom vfs ])));
108
+    in localPackages.runCommand "boot.dhall" { } ''
109
+      cat > $out << EOF
110
+      ${./store-wrapper.dhall}
111
+      (${config.genode.init.configFile})
112
+      "${config.system.build.tarball.fileName}.tar"
113
+      $(stat --format '%s' ${tarball})
114
+      ${config.system.build.storeManifest} ${manifest}
115
+      EOF
116
+    '';
117
+
118
+    system.build.storeManifest = mergeManifests (map addManifest initInputs);
119
+
120
+    # Create the tarball of the store to live in core ROM
121
+    system.build.tarball =
122
+      pkgs.callPackage "${modulesPath}/../lib/make-system-tarball.nix" {
123
+        contents = [ ];
124
+        storeContents = [
125
+          {
126
+            # assume that the init config will depend
127
+            # on every store path needed to boot
128
+            object = config.genode.init.configFile;
129
+            symlink = "/config.dhall";
130
+          }
131
+          {
132
+            object = pkgs.buildPackages.symlinkJoin {
133
+              name = config.system.name + ".rom";
134
+              paths = config.genode.init.inputs;
135
+            };
136
+            symlink = "/rom";
137
+          }
138
+        ];
139
+        compressCommand = "cat";
140
+        compressionExtension = "";
141
+      };
142
+
143
+    system.build.initXml = pkgs.buildPackages.runCommand "init.xml" {
144
+      nativeBuildInputs = with pkgs.buildPackages; [ dhall xorg.lndir ];
145
+      DHALL_GENODE = "${pkgs.genodePackages.dhallGenode}/binary.dhall";
146
+      BOOT_CONFIG = config.genode.boot.configFile;
147
+    } ''
148
+      export XDG_CACHE_HOME=$NIX_BUILD_TOP
149
+      lndir -silent \
150
+        ${pkgs.genodePackages.dhallGenode}/.cache \
151
+        $XDG_CACHE_HOME
152
+      dhall text <<< "(env:DHALL_GENODE).Init.render (env:BOOT_CONFIG).config" > $out
153
+    '';
154
+
155
+  };
156
+
157
+}

+ 111
- 0
nixos-modules/genode-init.nix View File

@@ -0,0 +1,111 @@
1
+{ config, pkgs, lib, ... }:
2
+
3
+with lib;
4
+
5
+let
6
+  inputs = mkOption {
7
+    description = "List of packages to build a ROM store with.";
8
+    type = types.listOf types.package;
9
+  };
10
+in {
11
+
12
+  options.genode.init = {
13
+    inherit inputs;
14
+
15
+    configFile = mkOption {
16
+      description = ''
17
+        Dhall configuration of this init instance after children have been merged.
18
+      '';
19
+      type = types.path;
20
+    };
21
+
22
+    baseConfig = mkOption {
23
+      description =
24
+        "Dhall configuration of this init instance before merging children.";
25
+      type = types.str;
26
+      default = ''
27
+        let Genode = env:DHALL_GENODE
28
+
29
+        in  Genode.Init::{
30
+            , routes =
31
+              [ Genode.Init.ServiceRoute.parent "File_system"
32
+              , Genode.Init.ServiceRoute.parent "Rtc"
33
+              , Genode.Init.ServiceRoute.parent "Timer"
34
+              , Genode.Init.ServiceRoute.parent "IRQ"
35
+              , Genode.Init.ServiceRoute.parent "IO_MEM"
36
+              , Genode.Init.ServiceRoute.parent "IO_PORT"
37
+              ]
38
+            }
39
+      '';
40
+    };
41
+
42
+    children = mkOption {
43
+      default = { };
44
+      type = with types;
45
+        attrsOf (submodule {
46
+          options = {
47
+            inherit inputs;
48
+            configFile = mkOption {
49
+              type = types.path;
50
+              description = ''
51
+                Dhall configuration of child.
52
+                See https://git.sr.ht/~ehmry/dhall-genode/tree/master/Init/Child/Type
53
+              '';
54
+            };
55
+          };
56
+        });
57
+    };
58
+
59
+    subinits = mkOption {
60
+      default = { };
61
+      type = with types;
62
+        attrsOf (submodule {
63
+          options = {
64
+            inherit inputs;
65
+            configFile = mkOption {
66
+              type = types.path;
67
+              description = ''
68
+                Dhall configuration of child init.
69
+                See https://git.sr.ht/~ehmry/dhall-genode/tree/master/Init/Type
70
+              '';
71
+            };
72
+          };
73
+        });
74
+    };
75
+
76
+  };
77
+
78
+  config = {
79
+
80
+    genode.init.inputs = with builtins;
81
+      [ pkgs.genodePackages.report_rom ] ++ concatLists (catAttrs "inputs"
82
+        ((attrValues config.genode.init.children)
83
+          ++ (attrValues config.genode.init.subinits)));
84
+
85
+    # TODO: convert the subinits to children
86
+
87
+    genode.init.configFile = pkgs.writeText "init.dhall" ''
88
+      let Genode = env:DHALL_GENODE
89
+      let baseConfig = ${config.genode.init.baseConfig}
90
+
91
+      in baseConfig with children = baseConfig.children # toMap {${
92
+        concatMapStrings (name:
93
+          ", `${name}` = (${
94
+            config.genode.init.children.${name}.configFile
95
+          } : Genode.Init.Child.Type)")
96
+        (builtins.attrNames config.genode.init.children)
97
+      } ${
98
+        concatMapStrings (name: ''
99
+          , `${name}` =
100
+            Genode.Init.toChild
101
+              (${
102
+                config.genode.init.subinits.${name}.configFile
103
+              } : Genode.Init.Type)
104
+              Genode.Init.Attributes.default
105
+        '') (builtins.attrNames config.genode.init.subinits)
106
+      } }
107
+    '';
108
+
109
+  };
110
+
111
+}

+ 195
- 0
nixos-modules/hardware.nix View File

@@ -0,0 +1,195 @@
1
+{ config, pkgs, lib, ... }:
2
+
3
+with lib;
4
+
5
+{
6
+  options.networking.interfaces = lib.mkOption {
7
+    type = with types;
8
+      attrsOf (submodule ({ ... }: {
9
+        options.genode = {
10
+
11
+          driver = mkOption {
12
+            type = types.enum [ "ipxe" "virtio" ];
13
+            default = "ipxe";
14
+          };
15
+
16
+          stack = mkOption {
17
+            type = types.enum [ "lwip" "lxip" ];
18
+            default = "lwip";
19
+          };
20
+
21
+        };
22
+      }));
23
+  };
24
+
25
+  config.genode.init.children = let
26
+    inherit (builtins) toFile;
27
+
28
+    nics = mapAttrs' (name: interface:
29
+      let name' = "nic." + name;
30
+      in {
31
+        name = name';
32
+        value = {
33
+          inputs = with pkgs.genodePackages;
34
+            {
35
+              ipxe = [ ipxe_nic_drv ];
36
+              virtio = [ virtio_nic_drv ];
37
+            }.${interface.genode.driver};
38
+          configFile = toFile "${name'}.dhall" ''
39
+            let Genode = env:DHALL_GENODE
40
+
41
+            let Init = Genode.Init
42
+
43
+            in  Init.Child.flat
44
+                  Init.Child.Attributes::{
45
+                  , binary = "virtio_pci_nic"
46
+                  , provides = [ "Nic" ]
47
+                  , resources = Init.Resources::{
48
+                    , caps = 128
49
+                    , ram = Genode.units.MiB 4
50
+                    }
51
+                  , routes = [ Init.ServiceRoute.parent "IO_MEM" ]
52
+                  , config = Init.Config::{
53
+                    , policies =
54
+                      [ Init.Config.Policy::{
55
+                        , service = "Nic"
56
+                        , label =
57
+                            Init.LabelSelector.prefix "sockets.${name}"
58
+                        }
59
+                      ]
60
+                    }
61
+                  }
62
+          '';
63
+        };
64
+      }) config.networking.interfaces;
65
+
66
+    sockets = mapAttrs' (name: interface:
67
+      let name' = "sockets." + name;
68
+      in {
69
+        name = name';
70
+        value = {
71
+          inputs = with pkgs.genodePackages;
72
+            {
73
+              lwip = [ vfs_lwip ];
74
+              lxip = [ vfs_lixp ];
75
+            }.${interface.genode.stack};
76
+          configFile = let ipv4 = builtins.head interface.ipv4.addresses;
77
+          in toFile "${name'}.dhall" ''
78
+            let Genode = env:DHALL_GENODE
79
+
80
+            let Init = Genode.Init
81
+
82
+            in  Init.Child.flat
83
+                  Init.Child.Attributes::{
84
+                  , binary = "vfs"
85
+                  , provides = [ "File_system" ]
86
+                  , resources = Init.Resources::{ caps = 128, ram = Genode.units.MiB 16 }
87
+                  , config = Init.Config::{
88
+                    , policies =
89
+                      [ Init.Config.Policy::{
90
+                        , service = "File_system"
91
+                        , label = Init.LabelSelector.suffix "sockets"
92
+                        , attributes = toMap { root = "/" }
93
+                        }
94
+                      ]
95
+                   , content =
96
+                        let XML = Genode.Prelude.XML
97
+
98
+                        in  [ XML.element
99
+                                { name = "vfs"
100
+                                , attributes = XML.emptyAttributes
101
+                                , content =
102
+                                  [ XML.leaf
103
+                                      { name = "lwip"
104
+                                      , attributes = toMap
105
+                                          { ip_addr = "${ipv4.address}", netmask = "${
106
+                                            if ipv4.prefixLength == 24 then
107
+                                              "255.255.255.0"
108
+                                            else
109
+                                              throw
110
+                                              "missing prefix to netmask conversion"
111
+                                          }" }
112
+                                      }
113
+                                  ]
114
+                                }
115
+                            ]
116
+                    }
117
+                  }
118
+          '';
119
+        };
120
+      }) config.networking.interfaces;
121
+
122
+  in nics // sockets // {
123
+
124
+    platform_drv = {
125
+      inputs = [ pkgs.genodePackages.platform_drv ];
126
+      configFile = let
127
+        policies = concatMapStrings (name: ''
128
+          Init.Config.Policy::{
129
+          , service = "Platform"
130
+          , label = Init.LabelSelector.prefix "nic.${name}"
131
+          , content =
132
+            [ Genode.Prelude.XML.leaf
133
+                { name = "pci", attributes = toMap { class = "ETHERNET" } }
134
+            ]
135
+          }
136
+        '') (builtins.attrNames config.networking.interfaces);
137
+      in toFile "platform_drv.dhall" ''
138
+        let Genode = env:DHALL_GENODE
139
+
140
+        let Init = Genode.Init
141
+
142
+        let label = \(_ : Text) -> { local = _, route = _ }
143
+
144
+        in  Init.Child.flat
145
+              Init.Child.Attributes::{
146
+              , binary = "platform_drv"
147
+              , resources = Init.Resources::{
148
+                , caps = 800
149
+                , ram = Genode.units.MiB 4
150
+                , constrainPhys = True
151
+                }
152
+              , reportRoms = [ label "acpi" ]
153
+              , provides = [ "Platform" ]
154
+              , routes =
155
+                [ Init.ServiceRoute.parent "IRQ"
156
+                , Init.ServiceRoute.parent "IO_MEM"
157
+                , Init.ServiceRoute.parent "IO_PORT"
158
+                ]
159
+              , config = Init.Config::{
160
+                , policies = [ ${policies} ]
161
+                }
162
+              }
163
+      '';
164
+    };
165
+
166
+    acpi_drv = {
167
+      inputs = [ pkgs.genodePackages.acpi_drv ];
168
+      configFile = toFile "acpi_drv.dhall" ''
169
+        let Genode = env:DHALL_GENODE
170
+
171
+        let Init = Genode.Init
172
+
173
+        let label = \(_ : Text) -> { local = _, route = _ }
174
+
175
+        in  Init.Child.flat
176
+              Init.Child.Attributes::{
177
+              , binary = "acpi_drv"
178
+              , resources = Init.Resources::{
179
+                , caps = 400
180
+                , ram = Genode.units.MiB 4
181
+                , constrainPhys = True
182
+                }
183
+              , romReports = [ label "acpi" ]
184
+              , routes =
185
+                [ Init.ServiceRoute.parent "IRQ"
186
+                , Init.ServiceRoute.parent "IO_MEM"
187
+                , Init.ServiceRoute.parent "IO_PORT"
188
+                ]
189
+              }
190
+      '';
191
+    };
192
+
193
+  };
194
+
195
+}

+ 27
- 0
nixos-modules/nova.nix View File

@@ -0,0 +1,27 @@
1
+{ config, pkgs, lib, ... }:
2
+
3
+with lib;
4
+let
5
+  localPackages = pkgs.buildPackages;
6
+  utils = import ../lib {
7
+    inherit (config.nixpkgs) system localSystem crossSystem;
8
+    inherit pkgs;
9
+  };
10
+in {
11
+  genode.core = {
12
+    prefix = "nova-";
13
+    supportedSystems = [ "x86_64-genode" ];
14
+    basePackages = with pkgs.genodePackages; [ base-nova rtc_drv ];
15
+  };
16
+
17
+  genode.boot = {
18
+
19
+    initrd =
20
+      "'${pkgs.genodePackages.NOVA}/hypervisor-x86_64 arg=iommu novpid serial,${config.genode.boot.image}/image.elf'";
21
+
22
+    image =
23
+      utils.novaImage config.system.name { } config.genode.boot.configFile;
24
+
25
+  };
26
+
27
+}

+ 364
- 553
nixos-modules/qemu-vm.nix
File diff suppressed because it is too large
View File


+ 166
- 0
nixos-modules/store-wrapper.dhall View File

@@ -0,0 +1,166 @@
1
+let Genode =
2
+      env:DHALL_GENODE sha256:e90438be23b5100003cf018b783986df67bc6d0e3d35e800677d0d9109ff6aa9
3
+
4
+let Prelude = Genode.Prelude
5
+
6
+let XML = Prelude.XML
7
+
8
+let Init = Genode.Init
9
+
10
+let Child = Init.Child
11
+
12
+let TextMapType = Prelude.Map.Type Text
13
+
14
+let Manifest/Type = TextMapType (TextMapType Text)
15
+
16
+let Manifest/toRoutes =
17
+      λ(manifest : Manifest/Type) →
18
+        Prelude.List.map
19
+          (Prelude.Map.Entry Text Text)
20
+          Init.ServiceRoute.Type
21
+          ( λ(entry : Prelude.Map.Entry Text Text) →
22
+              { service =
23
+                { name = "ROM"
24
+                , label = Init.LabelSelector.Type.Last entry.mapKey
25
+                }
26
+              , route =
27
+                  Init.Route.Type.Child
28
+                    { name = "store_rom"
29
+                    , label = Some entry.mapValue
30
+                    , diag = Some True
31
+                    }
32
+              }
33
+          )
34
+          ( Prelude.List.concat
35
+              (Prelude.Map.Entry Text Text)
36
+              (Prelude.Map.values Text (Prelude.Map.Type Text Text) manifest)
37
+          )
38
+
39
+let parentROMs =
40
+      Prelude.List.map
41
+        Text
42
+        Init.ServiceRoute.Type
43
+        ( λ(label : Text) →
44
+            { service = { name = "ROM", label = Init.LabelSelector.last label }
45
+            , route =
46
+                Init.Route.Type.Parent { label = Some label, diag = None Bool }
47
+            }
48
+        )
49
+
50
+let wrapStore
51
+    : Init.Type → Manifest/Type → Child.Type
52
+    = λ(init : Init.Type) →
53
+      λ(manifest : Manifest/Type) →
54
+        Init.toChild
55
+          init
56
+          Init.Attributes::{
57
+          , exitPropagate = True
58
+          , resources = Init.Resources::{ ram = Genode.units.MiB 4 }
59
+          , routes =
60
+                [ Init.ServiceRoute.parent "IO_MEM"
61
+                , Init.ServiceRoute.parent "IO_PORT"
62
+                , Init.ServiceRoute.parent "IRQ"
63
+                , Init.ServiceRoute.parent "VM"
64
+                , Init.ServiceRoute.child "Timer" "timer"
65
+                , Init.ServiceRoute.child "Rtc" "rtc"
66
+                ]
67
+              # parentROMs
68
+                  [ "ld.lib.so"
69
+                  , "init"
70
+                  , "platform_info"
71
+                  , "core_log"
72
+                  , "kernel_log"
73
+                  , "vfs"
74
+                  , "vfs.lib.so"
75
+                  , "cached_fs_rom"
76
+                  ]
77
+              # Manifest/toRoutes manifest
78
+              # [ Init.ServiceRoute.child "ROM" "store_rom" ]
79
+          }
80
+
81
+in  λ(subinit : Init.Type) →
82
+    λ(storeName : Text) →
83
+    λ(storeSize : Natural) →
84
+    λ(storeManifest : Manifest/Type) →
85
+    λ(bootManifest : Manifest/Type) →
86
+      Genode.Boot::{
87
+      , config = Init::{
88
+        , children =
89
+            let child = Prelude.Map.keyValue Child.Type
90
+
91
+            in  [ child
92
+                    "timer"
93
+                    ( Child.flat
94
+                        Child.Attributes::{
95
+                        , binary = "timer_drv"
96
+                        , provides = [ "Timer" ]
97
+                        }
98
+                    )
99
+                , child
100
+                    "rtc"
101
+                    ( Child.flat
102
+                        Child.Attributes::{
103
+                        , binary = "rtc_drv"
104
+                        , provides = [ "Rtc" ]
105
+                        , routes = [ Init.ServiceRoute.parent "IO_PORT" ]
106
+                        }
107
+                    )
108
+                , child
109
+                    "store_fs"
110
+                    ( Child.flat
111
+                        Child.Attributes::{
112
+                        , binary = "vfs"
113
+                        , config = Init.Config::{
114
+                          , content =
115
+                            [ XML.element
116
+                                { name = "vfs"
117
+                                , attributes = XML.emptyAttributes
118
+                                , content =
119
+                                  [ XML.leaf
120
+                                      { name = "tar"
121
+                                      , attributes = toMap { name = storeName }
122
+                                      }
123
+                                  ]
124
+                                }
125
+                            ]
126
+                          , policies =
127
+                            [ Init.Config.Policy::{
128
+                              , service = "File_system"
129
+                              , label = Init.LabelSelector.suffix "nix-store"
130
+                              , attributes = toMap { root = "/nix/store" }
131
+                              }
132
+                            , Init.Config.Policy::{
133
+                              , service = "File_system"
134
+                              , label = Init.LabelSelector.prefix "store_rom"
135
+                              , attributes = toMap { root = "/" }
136
+                              }
137
+                            ]
138
+                          }
139
+                        , provides = [ "File_system" ]
140
+                        }
141
+                    )
142
+                , child
143
+                    "store_rom"
144
+                    ( Child.flat
145
+                        Child.Attributes::{
146
+                        , binary = "cached_fs_rom"
147
+                        , provides = [ "ROM" ]
148
+                        , resources = Init.Resources::{
149
+                          , ram = storeSize + Genode.units.MiB 1
150
+                          }
151
+                        }
152
+                    )
153
+                , child "init" (wrapStore subinit storeManifest)
154
+                ]
155
+        }
156
+      , rom =
157
+          Genode.BootModules.toRomPaths
158
+            ( Prelude.List.concat
159
+                (Prelude.Map.Entry Text Text)
160
+                ( Prelude.Map.values
161
+                    Text
162
+                    (Prelude.Map.Type Text Text)
163
+                    bootManifest
164
+                )
165
+            )
166
+      }

+ 159
- 0
nixos-modules/systemd-runner.dhall View File

@@ -0,0 +1,159 @@
1
+let Genode = env:DHALL_GENODE
2
+
3
+let Prelude = Genode.Prelude
4
+
5
+let XML = Prelude.XML
6
+
7
+let Init = Genode.Init
8
+
9
+let Child = Init.Child
10
+
11
+let parentRoutes =
12
+      Prelude.List.map Text Init.ServiceRoute.Type Init.ServiceRoute.parent
13
+
14
+in  λ(params : { coreutils : Text, execStart : Text }) →
15
+      Init::{
16
+      , verbose = True
17
+      , routes = parentRoutes [ "Timer", "Rtc", "File_system" ]
18
+      , children = toMap
19
+          { vfs =
20
+              Child.flat
21
+                Child.Attributes::{
22
+                , binary = "vfs"
23
+                , exitPropagate = True
24
+                , provides = [ "File_system" ]
25
+                , resources = Genode.Init.Resources::{
26
+                  , caps = 256
27
+                  , ram = Genode.units.MiB 8
28
+                  }
29
+                , config = Init.Config::{
30
+                  , content =
31
+                    [ XML.element
32
+                        { name = "vfs"
33
+                        , attributes = XML.emptyAttributes
34
+                        , content =
35
+                            let dir =
36
+                                  λ(name : Text) →
37
+                                  λ(content : List XML.Type) →
38
+                                    XML.element
39
+                                      { name = "dir"
40
+                                      , content
41
+                                      , attributes = toMap { name }
42
+                                      }
43
+
44
+                            let leaf =
45
+                                  λ(name : Text) →
46
+                                    XML.leaf
47
+                                      { name, attributes = XML.emptyAttributes }
48
+
49
+                            in  [ dir
50
+                                    "dev"
51
+                                    [ dir "pipes" [ leaf "pipe" ]
52
+                                    , dir
53
+                                        "sockets"
54
+                                        [ XML.leaf
55
+                                            { name = "fs"
56
+                                            , attributes = toMap
57
+                                                { label = "sockets" }
58
+                                            }
59
+                                        ]
60
+                                    , leaf "log"
61
+                                    , leaf "null"
62
+                                    , leaf "rtc"
63
+                                    , leaf "zero"
64
+                                    ]
65
+                                , dir
66
+                                    "etc"
67
+                                    [ XML.element
68
+                                        { name = "inline"
69
+                                        , attributes = toMap
70
+                                            { name = "ExecStart" }
71
+                                        , content =
72
+                                          [ XML.text params.execStart ]
73
+                                        }
74
+                                    ]
75
+                                , dir
76
+                                    "usr"
77
+                                    [ dir
78
+                                        "bin"
79
+                                        [ XML.leaf
80
+                                            { name = "symlink"
81
+                                            , attributes = toMap
82
+                                                { name = "env"
83
+                                                , target =
84
+                                                    "${params.coreutils}/bin/env"
85
+                                                }
86
+                                            }
87
+                                        ]
88
+                                    ]
89
+                                , dir "tmp" [ leaf "ram" ]
90
+                                , dir
91
+                                    "nix"
92
+                                    [ dir
93
+                                        "store"
94
+                                        [ XML.leaf
95
+                                            { name = "fs"
96
+                                            , attributes = toMap
97
+                                                { label = "nix-store" }
98
+                                            }
99
+                                        ]
100
+                                    ]
101
+                                ]
102
+                        }
103
+                    ]
104
+                  , policies =
105
+                    [ Init.Config.Policy::{
106
+                      , service = "File_system"
107
+                      , label = Init.LabelSelector.prefix "shell"
108
+                      , attributes = toMap { root = "/", writeable = "yes" }
109
+                      }
110
+                    ]
111
+                  }
112
+                }
113
+          , shell =
114
+              Child.flat
115
+                Child.Attributes::{
116
+                , binary = "bash"
117
+                , exitPropagate = True
118
+                , resources = Genode.Init.Resources::{
119
+                  , caps = 256
120
+                  , ram = Genode.units.MiB 8
121
+                  }
122
+                , config = Genode.Init.Config::{
123
+                  , content =
124
+                        [ XML.leaf
125
+                            { name = "libc"
126
+                            , attributes = toMap
127
+                                { stdin = "/dev/null"
128
+                                , stdout = "/dev/log"
129
+                                , stderr = "/dev/log"
130
+                                , pipe = "/dev/pipes"
131
+                                , rtc = "/dev/rtc"
132
+                                , socket = "/dev/sockets"
133
+                                }
134
+                            }
135
+                        , XML.element
136
+                            { name = "vfs"
137
+                            , attributes = XML.emptyAttributes
138
+                            , content =
139
+                              [ XML.leaf
140
+                                  { name = "fs"
141
+                                  , attributes = XML.emptyAttributes
142
+                                  }
143
+                              ]
144
+                            }
145
+                        ]
146
+                      # Prelude.List.map
147
+                          Text
148
+                          XML.Type
149
+                          ( λ(x : Text) →
150
+                              XML.leaf
151
+                                { name = "arg"
152
+                                , attributes = toMap { value = x }
153
+                                }
154
+                          )
155
+                          [ "bash", "/etc/ExecStart" ]
156
+                  }
157
+                }
158
+          }
159
+      }

+ 33
- 0
nixos-modules/systemd.nix View File

@@ -0,0 +1,33 @@
1
+{ config, pkgs, lib, ... }:
2
+with lib; {
3
+
4
+  options.systemd.services = lib.mkOption {
5
+    type = types.attrsOf (types.submodule ({ name, config, ... }: {
6
+      options.genode.enable = lib.mkOption {
7
+        type = types.bool;
8
+        default = false;
9
+        description = "Translate this systemd unit to a Genode subsystem.";
10
+      };
11
+    }));
12
+  };
13
+
14
+  config.services.klogd.enable = false;
15
+  # The default is determined by checking the Linux version
16
+  # which cannot be evaluated here.
17
+
18
+  config.genode.init.subinits = mapAttrs' (name: service:
19
+    let name' = "services." + name;
20
+    in {
21
+      name = name';
22
+      value = {
23
+        inputs = with pkgs; with genodePackages; [ bash libc posix vfs_pipe ];
24
+        configFile = pkgs.writeText "${name'}.dhall" ''
25
+          ${./systemd-runner.dhall} {
26
+          , coreutils = "${pkgs.coreutils}"
27
+          , execStart = "${toString service.serviceConfig.ExecStart}"
28
+          }
29
+        '';
30
+      };
31
+    }) (filterAttrs (name: service: service.genode.enable)
32
+      config.systemd.services);
33
+}

+ 22
- 1
packages/genodelabs/targets.nix View File

@@ -1,5 +1,7 @@
1
-# This file contains overrides necesarry to build some Make and Depot targets.
1
+# This file contains overrides necessary to build some Make and Depot targets.
2 2
 # Many targets can be built with the default attributes, and are not listed here.
3
+# However, any package listed here with empty overrides ({ }) will be added to
4
+# the package attributes of this flake.
3 5
 
4 6
 { buildPackages, ports }:
5 7
 with ports;
@@ -15,6 +17,8 @@ let
15 17
   };
16 18
 in {
17 19
 
20
+  acpi_drv = { };
21
+
18 22
   cached_fs_rom = { };
19 23
 
20 24
   fb_sdl = with buildPackages; {
@@ -47,10 +51,20 @@ in {
47 51
 
48 52
   lx_block.HOST_INC_DIR = [ hostLibcInc ];
49 53
 
54
+  nic_bridge = { };
55
+
56
+  nic_loopback = { };
57
+
50 58
   noux.portInputs = [ libc ];
51 59
 
60
+  platform_drv = { };
61
+
52 62
   posix.portInputs = [ libc ];
53 63
 
64
+  report_rom = { };
65
+
66
+  rom_logger = { };
67
+
54 68
   rtc_drv.meta.platforms = [ "x86_64-genode" ];
55 69
 
56 70
   rump = {
@@ -58,6 +72,8 @@ in {
58 72
     buildInputs = with buildPackages; [ zlib ];
59 73
   };
60 74
 
75
+  sequence = { };
76
+
61 77
   stdcxx.portInputs = [ libc stdcxx ];
62 78
 
63 79
   # The following are tests are patched to exit at completion
@@ -79,12 +95,17 @@ in {
79 95
   vesa_drv.portInputs = [ libc x86emu ];
80 96
 
81 97
   vfs.outputs = [ "out" "lib" ];
98
+  vfs_audit = {};
82 99
   vfs_block = { };
83 100
   vfs_import.patches = [ ./vfs_import.patch ];
84 101
   vfs_jitterentropy.portInputs = [ jitterentropy libc ];
85 102
   vfs_lwip.portInputs = [ lwip ];
103
+  vfs_pipe = { };
86 104
   vfs_ttf.portInputs = [ libc stb ];
87 105
 
106
+  virtdev_rom = { };
107
+  virtio_nic_drv = { };
108
+
88 109
   wifi_drv.portInputs = [ dde_linux libc openssl ];
89 110
 
90 111
 }

+ 33
- 0
packages/genodelabs/test-pci.patch View File

@@ -8,3 +8,36 @@ index c6d9e2012b..050de6136c 100644
8 8
  	log("--- Platform test finished ---");
9 9
 +	env.parent().exit(0);
10 10
  }
11
+commit 03a5f469313e9fdc9ee1135ebf0b167e4d3d3266
12
+Author: Emery Hemingway <ehmry@posteo.net>
13
+Date:   Wed Oct 21 15:16:34 2020 +0200
14
+
15
+    test-pci: recognize VirtIO vendor IDs
16
+
17
+diff --git a/repos/os/src/test/pci/test.cc b/repos/os/src/test/pci/test.cc
18
+index c6d9e2012b..9cc2a2ac4b 100644
19
+--- a/repos/os/src/test/pci/test.cc
20
++++ b/repos/os/src/test/pci/test.cc
21
+@@ -19,7 +19,10 @@
22
+ 
23
+ using namespace Genode;
24
+ 
25
+-enum { INTEL_VENDOR_ID = 0x8086 };
26
++enum {
27
++	INTEL_VENDOR_ID = 0x8086,
28
++	VIRTIO_VENDOR_ID = 0x1af4,
29
++};
30
+ 
31
+ 
32
+ /**
33
+@@ -45,7 +48,9 @@ static void print_device_info(Platform::Device_capability device_cap)
34
+ 	    Hex(fun, Hex::OMIT_PREFIX), " "
35
+ 	    "class=", Hex(class_code), " "
36
+ 	    "vendor=", Hex(vendor_id), " ",
37
+-	               (vendor_id == INTEL_VENDOR_ID ? "(Intel)" : "(unknown)"),
38
++	               (vendor_id == INTEL_VENDOR_ID ? "(Intel)" :
39
++	                vendor_id == VIRTIO_VENDOR_ID ? "(VirtIO)" :
40
++	                "(unknown)"),
41
+ 	    " device=", Hex(device_id));
42
+ 
43
+ 	for (int resource_id = 0; resource_id < 6; resource_id++) {

+ 36
- 297
tests/default.nix View File

@@ -1,324 +1,63 @@
1
-{ self, apps, localPackages, genodepkgs, lib, nixpkgs, legacyPackages }:
1
+{ flake, system, localSystem, crossSystem, pkgs }:
2 2
 
3 3
 let
4
-
5
-  callTest = path:
6
-    import path {
7
-      pkgs = testPkgs;
8
-      inherit nixpkgs localPackages legacyPackages;
9
-    };
10
-
11
-  testFiles =
12
-    map callTest [ ./log.nix ./posix.nix ./vmm_arm.nix ./vmm_x86.nix ./x86.nix ]
13
-    ++ (callTest ./solo5);
14
-
15
-  testPkgs = genodepkgs;
16
-
17
-  qemu' = localPackages.qemu;
18
-
19
-  qemuBinary = qemuPkg:
20
-    {
21
-      aarch64-genode = "${qemuPkg}/bin/qemu-system-aarch64";
22
-      x86_64-genode = "${qemuPkg}/bin/qemu-system-x86_64";
23
-    }.${genodepkgs.stdenv.hostPlatform.system};
4
+  lib = flake.lib.${system};
5
+  nixpkgs = flake.legacyPackages.${system};
6
+  legacyPackages = flake.legacyPackages.${system};
7
+
8
+  testingPython = import ./lib/testing-python.nix;
9
+
10
+  testSpecs = map (p: import p) [
11
+    ./hello.nix
12
+    ./log.nix
13
+    ./solo5/multi.nix
14
+    ./vmm_x86.nix
15
+    ./x86.nix
16
+  ];
24 17
 
25 18
   cores = [
26 19
     {
27 20
       prefix = "hw-pc-";
21
+      testingPython = testingPython {
22
+        inherit flake system localSystem crossSystem pkgs;
23
+        extraConfigurations = [ ../nixos-modules/base-hw-pc.nix ];
24
+      };
28 25
       specs = [ "x86" "hw" ];
29 26
       platforms = [ "x86_64-genode" ];
30
-      basePackages = [ testPkgs.base-hw-pc ]
31
-        ++ map testPkgs.genodeSources.depot [ "rtc_drv" ];
32
-      makeImage =
33
-        lib.hwImage "0xffffffc000000000" "0x00200000" testPkgs.base-hw-pc;
34
-      startVM = vmName: image: ''
35
-        #! ${localPackages.runtimeShell}
36
-        exec ${qemuBinary qemu'} \
37
-          -name ${vmName} \
38
-          -machine q35 \
39
-          -m 384 \
40
-          -netdev user,id=net0 \
41
-          -device virtio-net-pci,netdev=net0 \
42
-          -kernel "${testPkgs.bender}/bender" \
43
-          -initrd "${image}/image.elf" \
44
-          $QEMU_OPTS \
45
-          "$@"
46
-          '';
47
-    }
48
-    {
49
-      prefix = "hw-virt_qemu-";
50
-      specs = [ "aarch64" "hw" ];
51
-      platforms = [ "aarch64-genode" ];
52
-      basePackages = with testPkgs; [ base-hw-virt_qemu rtc-dummy ];
53
-      makeImage = lib.hwImage "0xffffffc000000000" "0x40000000"
54
-        testPkgs.base-hw-virt_qemu;
55
-      startVM = vmName: image: ''
56
-        #! ${localPackages.runtimeShell}
57
-        exec ${qemuBinary qemu'} \
58
-          -name ${vmName} \
59
-          -M virt,virtualization=true,gic_version=3 \
60
-          -cpu cortex-a53 \
61
-          -smp 4 \
62
-          -m 384 \
63
-          -kernel "${image}/image.elf" \
64
-          $QEMU_OPTS \
65
-          "$@"
66
-          '';
67 27
     }
28
+    /* {
29
+         prefix = "hw-virt_qemu-";
30
+         testingPython = testingPython {
31
+           inherit flake system localSystem crossSystem pkgs;
32
+           extraConfigurations = [ ../nixos-modules/base-hw-virt_qemu.nix ];
33
+         };
34
+         specs = [ "aarch64" "hw" ];
35
+         platforms = [ "aarch64-genode" ];
36
+       }
37
+    */
68 38
     {
69 39
       prefix = "nova-";
40
+      testingPython = testingPython {
41
+        inherit flake system localSystem crossSystem pkgs;
42
+        extraConfigurations = [ ../nixos-modules/nova.nix ];
43
+      };
70 44
       specs = [ "x86" "nova" ];
71 45
       platforms = [ "x86_64-genode" ];
72
-      basePackages = [ testPkgs.base-nova ]
73
-        ++ map testPkgs.genodeSources.depot [ "rtc_drv" ];
74
-      makeImage = lib.novaImage;
75
-      startVM = vmName: image: ''
76
-        #! ${localPackages.runtimeShell}
77
-        exec ${qemuBinary qemu'} \
78
-          -name ${vmName} \
79
-          -machine q35 \
80
-          -m 384 \
81
-          -kernel "${testPkgs.bender}/bender" \
82
-          -initrd "${testPkgs.NOVA}/hypervisor-x86_64 arg=iommu novpid serial,${image}/image.elf" \
83
-          $QEMU_OPTS \
84
-          "$@"
85
-          '';
86 46
     }
87 47
   ];
88 48
 
89 49
   cores' = builtins.filter (core:
90
-    builtins.any (x: x == genodepkgs.stdenv.hostPlatform.system) core.platforms)
50
+    builtins.any (x: x == pkgs.stdenv.hostPlatform.system) core.platforms)
91 51
     cores;
92 52
 
93
-  testDriver = with localPackages;
94
-    let testDriverScript = ./test-driver/test-driver.py;
95
-    in stdenv.mkDerivation {
96
-      name = "nixos-test-driver";
97
-
98
-      nativeBuildInputs = [ makeWrapper ];
99
-      buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ];
100
-      checkInputs = with python3Packages; [ pylint mypy ];
101
-
102
-      dontUnpack = true;
103
-
104
-      preferLocalBuild = true;
105
-
106
-      doCheck = true;
107
-      checkPhase = ''
108
-        mypy --disallow-untyped-defs \
109
-             --no-implicit-optional \
110
-             --ignore-missing-imports ${testDriverScript}
111
-        pylint --errors-only ${testDriverScript}
112
-      '';
113
-
114
-      installPhase = ''
115
-        mkdir -p $out/bin
116
-        cp ${testDriverScript} $out/bin/nixos-test-driver
117
-        chmod u+x $out/bin/nixos-test-driver
118
-        # TODO: copy user script part into this file (append)
119
-
120
-        wrapProgram $out/bin/nixos-test-driver \
121
-          --prefix PATH : "${lib.makeBinPath [ qemu' coreutils ]}" \
122
-      '';
123
-    };
124
-
125
-  defaultTestScript = ''
126
-    start_all()
127
-    machine.wait_until_serial_output('child "init" exited with exit value 0')
128
-  '';
129
-
130
-  makeTest = with localPackages;
131
-    { prefix, specs, platforms, basePackages, makeImage, startVM }:
132
-    { name ? "unnamed", testScript ? defaultTestScript,
133
-    # Skip linting (mainly intended for faster dev cycles)
134
-    skipLint ? false, ... }@t:
135
-
136
-    let
137
-      testDriverName = "genode-test-driver-${name}";
138
-
139
-      buildVM = vmName:
140
-        { config, inputs, env ? { }, extraPaths ? [ ] }:
141
-        let
142
-          storeTarball = localPackages.runCommand "store" { } ''
143
-            mkdir -p $out
144
-            tar cf "$out/store.tar" --absolute-names ${toString inputs} ${
145
-              toString extraPaths
146
-            }
147
-          '';
148
-          addManifest = drv:
149
-            drv // {
150
-              manifest =
151
-                nixpkgs.runCommand "${drv.name}.dhall" { inherit drv; } ''
152
-                  set -eu
153
-                  echo -n '[' >> $out
154
-                  find $drv/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out
155
-                  ${if builtins.elem "lib" drv.outputs then
156
-                    ''
157
-                      find ${drv.lib}/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out''
158
-                  else
159
-                    ""}
160
-                  echo -n ']' >> $out
161
-                '';
162
-            };
163
-
164
-          storeManifest = lib.mergeManifests (map addManifest inputs);
165
-          manifest = lib.mergeManifests (map addManifest (basePackages
166
-            ++ [ testPkgs.sotest-producer storeTarball ]
167
-            ++ map testPkgs.genodeSources.depot [
168
-              "init"
169
-              "vfs"
170
-              "cached_fs_rom"
171
-            ]));
172
-          config' = "${
173
-              ./test-wrapper.dhall
174
-            } (${config}) $(stat --format '%s' ${storeTarball}/store.tar) ${storeManifest} ${manifest}";
175
-          env' = {
176
-            DHALL_GENODE = "${testPkgs.dhallGenode}/source.dhall";
177
-            DHALL_GENODE_TEST = "${./test.dhall}";
178
-          } // env;
179
-
180
-          image = makeImage vmName env' config';
181
-          startVM' = startVM vmName image;
182
-        in {
183
-          script = localPackages.writeScriptBin "run-${vmName}-vm" startVM';
184
-
185
-          config = lib.runDhallCommand (name + ".dhall") env' ''
186
-            ${apps.dhall.program} <<< "${config'}" > $out
187
-          '';
188
-
189
-          store = storeTarball;
190
-
191
-          xml = lib.runDhallCommand (name + ".config") env'
192
-            ''${apps.render-init.program} <<< "(${config'}).config" > $out'';
193
-        };
194
-
195
-      nodes = lib.mapAttrs buildVM
196
-        (t.nodes or (if t ? machine then { machine = t.machine; } else { }));
197
-
198
-      testScript' =
199
-        # Call the test script with the computed nodes.
200
-        if lib.isFunction testScript then
201
-          testScript { inherit nodes; }
202
-        else
203
-          testScript;
204
-
205
-      vms = map (node: node.script) (lib.attrValues nodes);
206
-
207
-      # Generate onvenience wrappers for running the test driver
208
-      # interactively with the specified network, and for starting the
209
-      # VMs from the command line.
210
-      driver =
211
-        let warn = if skipLint then lib.warn "Linting is disabled!" else lib.id;
212
-        in warn (runCommand testDriverName {
213
-          buildInputs = [ makeWrapper ];
214
-          testScript = testScript';
215
-          preferLocalBuild = true;
216
-          testName = name;
217
-        } ''
218
-          mkdir -p $out/bin
219
-
220
-          echo -n "$testScript" > $out/test-script
221
-          ${lib.optionalString (!skipLint) ''
222
-            ${python3Packages.black}/bin/black --check --quiet --diff $out/test-script
223
-          ''}
224
-
225
-          ln -s ${testDriver}/bin/nixos-test-driver $out/bin/
226
-          vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
227
-          wrapProgram $out/bin/nixos-test-driver \
228
-            --add-flags "''${vms[*]}" \
229
-            --run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\""
230
-          ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
231
-          wrapProgram $out/bin/nixos-run-vms \
232
-            --add-flags "''${vms[*]}" \
233
-            --set tests 'start_all(); join_all();'
234
-        ''); # "
235
-
236
-      passMeta = drv:
237
-        drv
238
-        // lib.optionalAttrs (t ? meta) { meta = (drv.meta or { }) // t.meta; };
239
-
240
-      # Run an automated test suite in the given virtual network.
241
-      # `driver' is the script that runs the network.
242
-      runTests = driver:
243
-        stdenv.mkDerivation {
244
-          name = "test-run-${driver.testName}";
245
-
246
-          buildCommand = ''
247
-            mkdir -p $out
248
-
249
-            LOGFILE=/dev/null tests='exec(os.environ["testScript"])' ${driver}/bin/nixos-test-driver
250
-          '';
251
-        };
252
-
253
-      test = passMeta (runTests driver);
254
-
255
-      nodeNames = builtins.attrNames nodes;
256
-      invalidNodeNames =
257
-        lib.filter (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
258
-        nodeNames;
259
-
260
-    in if lib.length invalidNodeNames > 0 then
261
-      throw ''
262
-        Cannot create machines out of (${
263
-          lib.concatStringsSep ", " invalidNodeNames
264
-        })!
265
-        All machines are referenced as python variables in the testing framework which will break the
266
-        script when special characters are used.
267
-
268
-        Please stick to alphanumeric chars and underscores as separation.
269
-      ''
270
-    else
271
-      test // { inherit nodes driver test; };
272
-
273 53
   testList = let
274
-    f = core:
275
-      let makeTest' = makeTest core;
276
-      in test:
54
+    f = core: test:
277 55
       if (test.constraints or (_: true)) core.specs then {
278 56
         name = core.prefix + test.name;
279
-        value = makeTest' test;
57
+        value = core.testingPython.makeTest test;
280 58
       } else
281 59
         null;
282 60
 
283
-  in lib.lists.crossLists f [ cores' testFiles ];
61
+  in lib.lists.crossLists f [ cores' testSpecs ];
284 62
 
285 63
 in builtins.listToAttrs (builtins.filter (_: _ != null) testList)
286
-
287
-/* sotest = let
288
-          hwTests = with hw; [ multi posix x86 ];
289
-          novaTests = with nova; [ multi posix x86 vmm ];
290
-          allTests = hwTests ++ novaTests;
291
-
292
-          projectCfg.boot_items =
293
-
294
-            (map (test: {
295
-              inherit (test) name;
296
-              exec = "bender";
297
-              load = [ "${test.name}.image.elf" ];
298
-            }) hwTests)
299
-
300
-            ++ (map (test: {
301
-              inherit (test) name;
302
-              exec = "bender";
303
-              load = [ "hypervisor serial novga iommu" test.image.name ];
304
-            }) novaTests);
305
-
306
-        in localPackages.stdenv.mkDerivation {
307
-          name = "sotest";
308
-          buildCommand = ''
309
-            mkdir zip; cd zip
310
-            cp "${testPkgs.bender}/bender" bender
311
-            cp "${testPkgs.NOVA}/hypervisor-x86_64" hypervisor
312
-            ${concatStringsSep "\n"
313
-            (map (test: "cp ${test.image}/image.elf ${test.name}.image.elf")
314
-              allTests)}
315
-            mkdir -p $out/nix-support
316
-            ${localPackages.zip}/bin/zip "$out/binaries.zip" *
317
-            cat << EOF > "$out/project.json"
318
-            ${builtins.toJSON projectCfg}
319
-            EOF
320
-            echo file sotest-binaries $out/binaries.zip >> "$out/nix-support/hydra-build-products"
321
-            echo file sotest-config $out/project.json >> "$out/nix-support/hydra-build-products"
322
-          '';
323
-        };
324
-*/

+ 51
- 0
tests/hello.dhall View File

@@ -0,0 +1,51 @@
1
+let Genode =
2
+        env:DHALL_GENODE
3
+      ? https://git.sr.ht/~ehmry/dhall-genode/blob/master/package.dhall
4
+
5
+let Init = Genode.Init
6
+
7
+let Child = Init.Child
8
+
9
+in  Child.flat
10
+      Child.Attributes::{
11
+      , binary = "hello"
12
+      , exitPropagate = True
13
+      , resources = Genode.Init.Resources::{
14
+        , caps = 500
15
+        , ram = Genode.units.MiB 10
16
+        }
17
+      , config = Init.Config::{
18
+        , content =
19
+            let XML = Genode.Prelude.XML
20
+
21
+            in  [ XML.leaf
22
+                    { name = "libc"
23
+                    , attributes = toMap
24
+                        { stdin = "/dev/null"
25
+                        , stdout = "/dev/log"
26
+                        , stderr = "/dev/log"
27
+                        }
28
+                    }
29
+                , XML.element
30
+                    { name = "vfs"
31
+                    , attributes = XML.emptyAttributes
32
+                    , content =
33
+                        let dir =
34
+                              λ(name : Text) →
35
+                              λ(content : List XML.Type) →
36
+                                XML.element
37
+                                  { name = "dir"
38
+                                  , content
39
+                                  , attributes = toMap { name }
40
+                                  }
41
+
42
+                        let leaf =
43
+                              λ(name : Text) →
44
+                                XML.leaf
45
+                                  { name, attributes = XML.emptyAttributes }
46
+
47
+                        in  [ dir "dev" [ leaf "log", leaf "null" ] ]
48
+                    }
49
+                ]
50
+        }
51
+      }

+ 28
- 0
tests/hello.nix View File

@@ -0,0 +1,28 @@
1
+{
2
+  name = "hello";
3
+  machine = { pkgs, ... }:
4
+    let
5
+      hello = pkgs.stdenv.mkDerivation {
6
+        name = "hello";
7
+        dontUnpack = true;
8
+        buildPhase = ''
9
+          cat > hello.c << EOF
10
+          #include <stdio.h>
11
+          int main(int argc, char **argv) { printf("hello world!\n"); return 0; }
12
+          EOF
13
+
14
+          $CC hello.c -o hello
15
+        '';
16
+        installPhase = "install -Dt $out/bin hello";
17
+      };
18
+    in {
19
+      genode.init.children.hello = {
20
+        configFile = ./hello.dhall;
21
+        inputs = [ hello ];
22
+      };
23
+    };
24
+  testScript = ''
25
+    start_all()
26
+    machine.wait_until_serial_output("child \"init\" exited with exit value 0")
27
+  '';
28
+}

+ 110
- 0
tests/lib/build-vms.nix View File

@@ -0,0 +1,110 @@
1
+{ system, localSystem, crossSystem
2
+# Nixpkgs, for qemu, lib and more
3
+, pkgs, modulesPath
4
+# NixOS configuration to add to the VMs
5
+, extraConfigurations ? [ ] }:
6
+
7
+with pkgs.lib;
8
+with import ./qemu-flags.nix { inherit pkgs; };
9
+
10
+rec {
11
+
12
+  inherit pkgs;
13
+
14
+  qemu = pkgs.buildPackages.buildPackages.qemu_test;
15
+
16
+  # Build a virtual network from an attribute set `{ machine1 =
17
+  # config1; ... machineN = configN; }', where `machineX' is the
18
+  # hostname and `configX' is a NixOS system configuration.  Each
19
+  # machine is given an arbitrary IP address in the virtual network.
20
+  buildVirtualNetwork = nodes:
21
+    let nodesOut = mapAttrs (n: buildVM nodesOut) (assignIPAddresses nodes);
22
+    in nodesOut;
23
+
24
+  buildVM = nodes: configurations:
25
+
26
+    import "${modulesPath}/../lib/eval-config.nix" {
27
+      inherit system;
28
+      modules = configurations ++ extraConfigurations;
29
+      baseModules = (import "${modulesPath}/module-list.nix") ++ [
30
+        ../../nixos-modules/genode-core.nix
31
+        ../../nixos-modules/genode-init.nix
32
+        ../../nixos-modules/qemu-vm.nix
33
+        {
34
+          key = "no-manual";
35
+          documentation.nixos.enable = false;
36
+        }
37
+        {
38
+          key = "qemu";
39
+          system.build.qemu = qemu;
40
+        }
41
+        {
42
+          key = "nodes";
43
+          _module.args.nodes = nodes;
44
+        }
45
+        {
46
+          system.build.qemu = qemu;
47
+          nixpkgs = { inherit system crossSystem localSystem pkgs; };
48
+        }
49
+      ];
50
+    };
51
+
52
+  # Given an attribute set { machine1 = config1; ... machineN =
53
+  # configN; }, sequentially assign IP addresses in the 192.168.1.0/24
54
+  # range to each machine, and set the hostname to the attribute name.
55
+  assignIPAddresses = nodes:
56
+
57
+    let
58
+
59
+      machines = attrNames nodes;
60
+
61
+      machinesNumbered = zipLists machines (range 1 254);
62
+
63
+      nodes_ = forEach machinesNumbered (m:
64
+        nameValuePair m.fst [
65
+          ({ config, nodes, ... }:
66
+            let
67
+              interfacesNumbered =
68
+                zipLists config.virtualisation.vlans (range 1 255);
69
+              interfaces = forEach interfacesNumbered ({ fst, snd }:
70
+                nameValuePair "eth${toString snd}" {
71
+                  ipv4.addresses = [{
72
+                    address = "192.168.${toString fst}.${toString m.snd}";
73
+                    prefixLength = 24;
74
+                  }];
75
+                });
76
+            in {
77
+              key = "ip-address";
78
+              config = {
79
+                networking.hostName = mkDefault m.fst;
80
+
81
+                networking.interfaces = listToAttrs interfaces;
82
+
83
+                networking.primaryIPAddress = optionalString (interfaces != [ ])
84
+                  (head (head interfaces).value.ipv4.addresses).address;
85
+
86
+                # Put the IP addresses of all VMs in this machine's
87
+                # /etc/hosts file.  If a machine has multiple
88
+                # interfaces, use the IP address corresponding to
89
+                # the first interface (i.e. the first network in its
90
+                # virtualisation.vlans option).
91
+                networking.extraHosts = flip concatMapStrings machines (m':
92
+                  let config = (getAttr m' nodes).config;
93
+                  in optionalString (config.networking.primaryIPAddress != "")
94
+                  ("${config.networking.primaryIPAddress} "
95
+                    + optionalString (config.networking.domain != null)
96
+                    "${config.networking.hostName}.${config.networking.domain} "
97
+                    + ''
98
+                      ${config.networking.hostName}
99
+                    ''));
100
+
101
+                virtualisation.qemu.options = forEach interfacesNumbered
102
+                  ({ fst, snd }: qemuNICFlags snd fst m.snd);
103
+              };
104
+            })
105
+          (getAttr m.fst nodes)
106
+        ]);
107
+
108
+    in listToAttrs nodes_;
109
+
110
+}

+ 41
- 0
tests/lib/qemu-flags.nix View File

@@ -0,0 +1,41 @@
1
+# QEMU flags shared between various Nix expressions.
2
+{ pkgs }:
3
+
4
+let
5
+  zeroPad = n:
6
+    pkgs.lib.optionalString (n < 16) "0" + (if n > 255 then
7
+      throw "Can't have more than 255 nets or nodes!"
8
+    else
9
+      pkgs.lib.toHexString n);
10
+
11
+in rec {
12
+  qemuNicMac = net: machine: "52:54:00:12:${zeroPad net}:${zeroPad machine}";
13
+
14
+  qemuNICFlags = nic: net: machine: [
15
+    "-device virtio-net-pci,netdev=vlan${toString nic},mac=${
16
+      qemuNicMac net machine
17
+    }"
18
+    "-netdev vde,id=vlan${toString nic},sock=$QEMU_VDE_SOCKET_${toString net}"
19
+  ];
20
+
21
+  qemuSerialDevice = if pkgs.stdenv.isi686 || pkgs.stdenv.isx86_64 then
22
+    "ttyS0"
23
+  else if pkgs.stdenv.isAarch32 || pkgs.stdenv.isAarch64 then
24
+    "ttyAMA0"
25
+  else
26
+    throw
27
+    "Unknown QEMU serial device for system '${pkgs.stdenv.hostPlatform.system}'";
28
+
29
+  qemuBinary = qemuPkg:
30
+    {
31
+      x86_64-linux = "${qemuPkg}/bin/qemu-kvm -cpu max";
32
+      armv7l-linux =
33
+        "${qemuPkg}/bin/qemu-system-arm -enable-kvm -machine virt -cpu host";
34
+      aarch64-linux =
35
+        "${qemuPkg}/bin/qemu-system-aarch64 -enable-kvm -machine virt,gic-version=host -cpu host";
36
+      x86_64-darwin = "${qemuPkg}/bin/qemu-kvm -cpu max";
37
+      aarch64-genode =
38
+        "${qemuPkg}/bin/qemu-system-aarch64 -M virt,virtualization=true,gic_version=3 -cpu cortex-a53";
39
+      x86_64-genode = "${qemuPkg}/bin/qemu-system-x86_64 -machine q35";
40
+    }.${pkgs.stdenv.hostPlatform.system} or "${qemuPkg}/bin/qemu-kvm";
41
+}

+ 967
- 0
tests/lib/test-driver.py View File

@@ -0,0 +1,967 @@
1
+#! /somewhere/python3
2
+# Copyright (c) 2003-2020 Nixpkgs/NixOS contributors
3
+
4
+from contextlib import contextmanager, _GeneratorContextManager
5
+from queue import Queue, Empty
6
+from typing import Tuple, Any, Callable, Dict, Iterator, Optional, List
7
+from xml.sax.saxutils import XMLGenerator
8
+import queue
9
+import io
10
+import _thread
11
+import argparse
12
+import atexit
13
+import base64
14
+import codecs
15
+import os
16
+import pathlib
17
+import ptpython.repl
18
+import pty
19
+import re
20
+import shlex
21
+import shutil
22
+import socket
23
+import subprocess
24
+import sys
25
+import tempfile
26
+import time
27
+import traceback
28
+import unicodedata
29
+
30
+CHAR_TO_KEY = {
31
+    "A": "shift-a",
32
+    "N": "shift-n",
33
+    "-": "0x0C",
34
+    "_": "shift-0x0C",
35
+    "B": "shift-b",
36
+    "O": "shift-o",
37
+    "=": "0x0D",
38
+    "+": "shift-0x0D",
39
+    "C": "shift-c",
40
+    "P": "shift-p",
41
+    "[": "0x1A",
42
+    "{": "shift-0x1A",
43
+    "D": "shift-d",
44
+    "Q": "shift-q",
45
+    "]": "0x1B",
46
+    "}": "shift-0x1B",
47
+    "E": "shift-e",
48
+    "R": "shift-r",
49
+    ";": "0x27",
50
+    ":": "shift-0x27",
51
+    "F": "shift-f",
52
+    "S": "shift-s",
53
+    "'": "0x28",
54
+    '"': "shift-0x28",
55
+    "G": "shift-g",
56
+    "T": "shift-t",
57
+    "`": "0x29",
58
+    "~": "shift-0x29",
59
+    "H": "shift-h",
60
+    "U": "shift-u",
61
+    "\\": "0x2B",
62
+    "|": "shift-0x2B",
63
+    "I": "shift-i",
64
+    "V": "shift-v",
65
+    ",": "0x33",
66
+    "<": "shift-0x33",
67
+    "J": "shift-j",
68
+    "W": "shift-w",
69
+    ".": "0x34",
70
+    ">": "shift-0x34",
71
+    "K": "shift-k",
72
+    "X": "shift-x",
73
+    "/": "0x35",
74
+    "?": "shift-0x35",
75
+    "L": "shift-l",
76
+    "Y": "shift-y",
77
+    " ": "spc",
78
+    "M": "shift-m",
79
+    "Z": "shift-z",
80
+    "\n": "ret",
81
+    "!": "shift-0x02",
82
+    "@": "shift-0x03",
83
+    "#": "shift-0x04",
84
+    "$": "shift-0x05",
85
+    "%": "shift-0x06",
86
+    "^": "shift-0x07",
87
+    "&": "shift-0x08",
88
+    "*": "shift-0x09",
89
+    "(": "shift-0x0A",
90
+    ")": "shift-0x0B",
91
+}
92
+
93
+# Forward references
94
+log: "Logger"
95
+machines: "List[Machine]"
96
+
97
+
98
+def eprint(*args: object, **kwargs: Any) -> None:
99
+    print(*args, file=sys.stderr, **kwargs)
100
+
101
+
102
+def make_command(args: list) -> str:
103
+    return " ".join(map(shlex.quote, (map(str, args))))
104
+
105
+
106
+def create_vlan(vlan_nr: str) -> Tuple[str, str, "subprocess.Popen[bytes]", Any]:
107
+    global log
108
+    log.log("starting VDE switch for network {}".format(vlan_nr))
109
+    vde_socket = tempfile.mkdtemp(
110
+        prefix="nixos-test-vde-", suffix="-vde{}.ctl".format(vlan_nr)
111
+    )
112
+    pty_master, pty_slave = pty.openpty()
113
+    vde_process = subprocess.Popen(
114
+        ["vde_switch", "-s", vde_socket, "--dirmode", "0700"],
115
+        bufsize=1,
116
+        stdin=pty_slave,
117
+        stdout=subprocess.PIPE,
118
+        stderr=subprocess.PIPE,
119
+        shell=False,
120
+    )
121
+    fd = os.fdopen(pty_master, "w")
122
+    fd.write("version\n")
123
+    # TODO: perl version checks if this can be read from
124
+    # an if not, dies. we could hang here forever. Fix it.
125
+    assert vde_process.stdout is not None
126
+    vde_process.stdout.readline()
127
+    if not os.path.exists(os.path.join(vde_socket, "ctl")):
128
+        raise Exception("cannot start vde_switch")
129
+
130
+    return (vlan_nr, vde_socket, vde_process, fd)
131
+
132
+
133
+def retry(fn: Callable) -> None:
134
+    """Call the given function repeatedly, with 1 second intervals,
135
+    until it returns True or a timeout is reached.
136
+    """
137
+
138
+    for _ in range(900):
139
+        if fn(False):
140
+            return
141
+        time.sleep(1)
142
+
143
+    if not fn(True):
144
+        raise Exception("action timed out")
145
+
146
+
147
+class Logger:
148
+    def __init__(self) -> None:
149
+        self.logfile = os.environ.get("LOGFILE", "/dev/null")
150
+        self.logfile_handle = codecs.open(self.logfile, "wb")
151
+        self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8")
152
+        self.queue: "Queue[Dict[str, str]]" = Queue()
153
+
154
+        self.xml.startDocument()
155
+        self.xml.startElement("logfile", attrs={})
156
+
157
+    def close(self) -> None:
158
+        self.xml.endElement("logfile")
159
+        self.xml.endDocument()
160
+        self.logfile_handle.close()
161
+
162
+    def sanitise(self, message: str) -> str:
163
+        return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C")
164
+
165
+    def maybe_prefix(self, message: str, attributes: Dict[str, str]) -> str:
166
+        if "machine" in attributes:
167
+            return "{}: {}".format(attributes["machine"], message)
168
+        return message
169
+
170
+    def log_line(self, message: str, attributes: Dict[str, str]) -> None:
171
+        self.xml.startElement("line", attributes)
172
+        self.xml.characters(message)
173
+        self.xml.endElement("line")
174
+
175
+    def log(self, message: str, attributes: Dict[str, str] = {}) -> None:
176
+        eprint(self.maybe_prefix(message, attributes))
177
+        self.drain_log_queue()
178
+        self.log_line(message, attributes)
179
+
180
+    def enqueue(self, message: Dict[str, str]) -> None:
181
+        self.queue.put(message)
182
+
183
+    def drain_log_queue(self) -> None:
184
+        try:
185
+            while True:
186
+                item = self.queue.get_nowait()
187
+                attributes = {"machine": item["machine"], "type": "serial"}
188
+                self.log_line(self.sanitise(item["msg"]), attributes)
189
+        except Empty:
190
+            pass
191
+
192
+    @contextmanager
193
+    def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]:
194
+        eprint(self.maybe_prefix(message, attributes))
195
+
196
+        self.xml.startElement("nest", attrs={})
197
+        self.xml.startElement("head", attributes)
198
+        self.xml.characters(message)
199
+        self.xml.endElement("head")
200
+
201
+        tic = time.time()
202
+        self.drain_log_queue()
203
+        yield
204
+        self.drain_log_queue()
205
+        toc = time.time()
206
+        self.log("({:.2f} seconds)".format(toc - tic))
207
+
208
+        self.xml.endElement("nest")
209
+
210
+
211
+class Machine:
212
+    def __init__(self, args: Dict[str, Any]) -> None:
213
+        if "name" in args:
214
+            self.name = args["name"]
215
+        else:
216
+            self.name = "machine"
217
+            cmd = args.get("startCommand", None)
218
+            if cmd:
219
+                match = re.search("bin/run-(.+)-vm$", cmd)
220
+                if match:
221
+                    self.name = match.group(1)
222
+
223
+        self.script = args.get("startCommand", self.create_startcommand(args))
224
+
225
+        tmp_dir = os.environ.get("TMPDIR", tempfile.gettempdir())
226
+
227
+        def create_dir(name: str) -> str:
228
+            path = os.path.join(tmp_dir, name)
229
+            os.makedirs(path, mode=0o700, exist_ok=True)
230
+            return path
231
+
232
+        self.state_dir = create_dir("vm-state-{}".format(self.name))
233
+        self.shared_dir = create_dir("shared-xchg")
234
+
235
+        self.booted = False
236
+        self.connected = False
237
+        self.pid: Optional[int] = None
238
+        self.socket = None
239
+        self.monitor: Optional[socket.socket] = None
240
+        self.logger: Logger = args["log"]
241
+        self.serialQueue: "Queue[str]" = Queue()
242
+
243
+        self.allow_reboot = args.get("allowReboot", False)
244
+
245
+    @staticmethod
246
+    def create_startcommand(args: Dict[str, str]) -> str:
247
+        net_backend = "-netdev user,id=net0"
248
+        net_frontend = "-device virtio-net-pci,netdev=net0"
249
+
250
+        if "netBackendArgs" in args:
251
+            net_backend += "," + args["netBackendArgs"]
252
+
253
+        if "netFrontendArgs" in args:
254
+            net_frontend += "," + args["netFrontendArgs"]
255
+
256
+        start_command = (
257
+            "qemu-kvm -m 384 " + net_backend + " " + net_frontend + " $QEMU_OPTS "
258
+        )
259
+
260
+        if "hda" in args:
261
+            hda_path = os.path.abspath(args["hda"])
262
+            if args.get("hdaInterface", "") == "scsi":
263
+                start_command += (
264
+                    "-drive id=hda,file="
265
+                    + hda_path
266
+                    + ",werror=report,if=none "
267
+                    + "-device scsi-hd,drive=hda "
268
+                )
269
+            else:
270
+                start_command += (
271
+                    "-drive file="
272
+                    + hda_path
273
+                    + ",if="
274
+                    + args["hdaInterface"]
275
+                    + ",werror=report "
276
+                )
277
+
278
+        if "cdrom" in args:
279
+            start_command += "-cdrom " + args["cdrom"] + " "
280
+
281
+        if "usb" in args:
282
+            start_command += (
283
+                "-device piix3-usb-uhci -drive "
284
+                + "id=usbdisk,file="
285
+                + args["usb"]
286
+                + ",if=none,readonly "
287
+                + "-device usb-storage,drive=usbdisk "
288
+            )
289
+        if "bios" in args:
290
+            start_command += "-bios " + args["bios"] + " "
291
+
292
+        start_command += args.get("qemuFlags", "")
293
+
294
+        return start_command
295
+
296
+    def is_up(self) -> bool:
297
+        return self.booted and self.connected
298
+
299
+    def log(self, msg: str) -> None:
300
+        self.logger.log(msg, {"machine": self.name})
301
+
302
+    def nested(self, msg: str, attrs: Dict[str, str] = {}) -> _GeneratorContextManager:
303
+        my_attrs = {"machine": self.name}
304
+        my_attrs.update(attrs)
305
+        return self.logger.nested(msg, my_attrs)
306
+
307
+    def wait_for_monitor_prompt(self) -> str:
308
+        assert self.monitor is not None
309
+        answer = ""
310
+        while True:
311
+            undecoded_answer = self.monitor.recv(1024)
312
+            if not undecoded_answer:
313
+                break
314
+            answer += undecoded_answer.decode()
315
+            if answer.endswith("(qemu) "):
316
+                break
317
+        return answer
318
+
319
+    def send_monitor_command(self, command: str) -> str:
320
+        message = ("{}\n".format(command)).encode()
321
+        self.log("sending monitor command: {}".format(command))
322
+        assert self.monitor is not None
323
+        self.monitor.send(message)
324
+        return self.wait_for_monitor_prompt()
325
+
326
+    def wait_for_unit(self, unit: str, user: Optional[str] = None) -> None:
327
+        """Wait for a systemd unit to get into "active" state.
328
+        Throws exceptions on "failed" and "inactive" states as well as
329
+        after timing out.
330
+        """
331
+
332
+        def check_active(_: Any) -> bool:
333
+            info = self.get_unit_info(unit, user)
334
+            state = info["ActiveState"]
335
+            if state == "failed":
336
+                raise Exception('unit "{}" reached state "{}"'.format(unit, state))
337
+
338
+            if state == "inactive":
339
+                status, jobs = self.systemctl("list-jobs --full 2>&1", user)
340
+                if "No jobs" in jobs:
341
+                    info = self.get_unit_info(unit, user)
342
+                    if info["ActiveState"] == state:
343
+                        raise Exception(
344
+                            (
345
+                                'unit "{}" is inactive and there ' "are no pending jobs"
346
+                            ).format(unit)
347
+                        )
348
+
349
+            return state == "active"
350
+
351
+        retry(check_active)
352
+
353
+    def get_unit_info(self, unit: str, user: Optional[str] = None) -> Dict[str, str]:
354
+        status, lines = self.systemctl('--no-pager show "{}"'.format(unit), user)
355
+        if status != 0:
356
+            raise Exception(
357
+                'retrieving systemctl info for unit "{}" {} failed with exit code {}'.format(
358
+                    unit, "" if user is None else 'under user "{}"'.format(user), status
359
+                )
360
+            )
361
+
362
+        line_pattern = re.compile(r"^([^=]+)=(.*)$")
363
+
364
+        def tuple_from_line(line: str) -> Tuple[str, str]:
365
+            match = line_pattern.match(line)
366
+            assert match is not None
367
+            return match[1], match[2]
368
+
369
+        return dict(
370
+            tuple_from_line(line)
371
+            for line in lines.split("\n")
372
+            if line_pattern.match(line)
373
+        )
374
+
375
+    def systemctl(self, q: str, user: Optional[str] = None) -> Tuple[int, str]:
376
+        if user is not None:
377
+            q = q.replace("'", "\\'")
378
+            return self.execute(
379
+                (
380
+                    "su -l {} --shell /bin/sh -c "
381
+                    "$'XDG_RUNTIME_DIR=/run/user/`id -u` "
382
+                    "systemctl --user {}'"
383
+                ).format(user, q)
384
+            )
385
+        return self.execute("systemctl {}".format(q))
386
+
387
+    def require_unit_state(self, unit: str, require_state: str = "active") -> None:
388
+        with self.nested(
389
+            "checking if unit ‘{}’ has reached state '{}'".format(unit, require_state)
390
+        ):
391
+            info = self.get_unit_info(unit)
392
+            state = info["ActiveState"]
393
+            if state != require_state:
394
+                raise Exception(
395
+                    "Expected unit ‘{}’ to to be in state ".format(unit)
396
+                    + "'{}' but it is in state ‘{}’".format(require_state, state)
397
+                )
398
+
399
+    def execute(self, command: str) -> Tuple[int, str]:
400
+        self.connect()
401
+
402
+        out_command = "( {} ); echo '|!=EOF' $?\n".format(command)
403
+        self.shell.send(out_command.encode())
404
+
405
+        output = ""
406
+        status_code_pattern = re.compile(r"(.*)\|\!=EOF\s+(\d+)")
407
+
408
+        while True:
409
+            chunk = self.shell.recv(4096).decode(errors="ignore")
410
+            match = status_code_pattern.match(chunk)
411
+            if match:
412
+                output += match[1]
413
+                status_code = int(match[2])
414
+                return (status_code, output)
415
+            output += chunk
416
+
417
+    def succeed(self, *commands: str) -> str:
418
+        """Execute each command and check that it succeeds."""
419
+        output = ""
420
+        for command in commands:
421
+            with self.nested("must succeed: {}".format(command)):
422
+                (status, out) = self.execute(command)
423
+                if status != 0:
424
+                    self.log("output: {}".format(out))
425
+                    raise Exception(
426
+                        "command `{}` failed (exit code {})".format(command, status)
427
+                    )
428
+                output += out
429
+        return output
430
+
431
+    def fail(self, *commands: str) -> None:
432
+        """Execute each command and check that it fails."""
433
+        for command in commands:
434
+            with self.nested("must fail: {}".format(command)):
435
+                status, output = self.execute(command)
436
+                if status == 0:
437
+                    raise Exception(
438
+                        "command `{}` unexpectedly succeeded".format(command)
439
+                    )
440
+
441
+    def wait_until_succeeds(self, command: str) -> str:
442
+        """Wait until a command returns success and return its output.
443
+        Throws an exception on timeout.
444
+        """
445
+        output = ""
446
+
447
+        def check_success(_: Any) -> bool:
448
+            nonlocal output
449
+            status, output = self.execute(command)
450
+            return status == 0
451
+
452
+        with self.nested("waiting for success: {}".format(command)):
453
+            retry(check_success)
454
+            return output
455
+
456
+    def wait_until_fails(self, command: str) -> str:
457
+        """Wait until a command returns failure.
458
+        Throws an exception on timeout.
459
+        """
460
+        output = ""
461
+
462
+        def check_failure(_: Any) -> bool:
463
+            nonlocal output
464
+            status, output = self.execute(command)
465
+            return status != 0
466
+
467
+        with self.nested("waiting for failure: {}".format(command)):
468
+            retry(check_failure)
469
+            return output
470
+
471
+    def wait_for_shutdown(self) -> None:
472
+        if not self.booted:
473
+            return
474
+
475
+        with self.nested("waiting for the VM to power off"):
476
+            sys.stdout.flush()
477
+            self.process.wait()
478
+
479
+            self.pid = None
480
+            self.booted = False
481
+            self.connected = False
482
+
483
+    def get_tty_text(self, tty: str) -> str:
484
+        status, output = self.execute(
485
+            "fold -w$(stty -F /dev/tty{0} size | "
486
+            "awk '{{print $2}}') /dev/vcs{0}".format(tty)
487
+        )
488
+        return output
489
+
490
+    def wait_until_tty_matches(self, tty: str, regexp: str) -> None:
491
+        """Wait until the visible output on the chosen TTY matches regular
492
+        expression. Throws an exception on timeout.
493
+        """
494
+        matcher = re.compile(regexp)
495
+
496
+        def tty_matches(last: bool) -> bool:
497
+            text = self.get_tty_text(tty)
498
+            if last:
499
+                self.log(
500
+                    f"Last chance to match /{regexp}/ on TTY{tty}, "
501
+                    f"which currently contains: {text}"
502
+                )
503
+            return len(matcher.findall(text)) > 0
504
+
505
+        with self.nested("waiting for {} to appear on tty {}".format(regexp, tty)):
506
+            retry(tty_matches)
507
+
508
+    def wait_until_serial_output(self, regexp: str) -> None:
509
+        """Wait until the serial output matches regular expression.
510
+        Throws an exception on timeout.
511
+        """
512
+        matcher = re.compile(regexp)
513
+
514
+        def serial_matches(last: bool) -> bool:
515
+            while not self.serialQueue.empty():
516
+                text = self.serialQueue.get()
517
+                if last:
518
+                    self.log(
519
+                        f"Last chance to match /{regexp}/ on serial, "
520
+                        f"which currently contains: {text}"
521
+                    )
522
+                if len(matcher.findall(text)) > 0:
523
+                    return True
524
+            return False
525
+
526
+        with self.nested("waiting for {} to appear on serial output".format(regexp)):
527
+            retry(serial_matches)
528
+
529
+    def send_chars(self, chars: List[str]) -> None:
530
+        with self.nested("sending keys ‘{}‘".format(chars)):
531
+            for char in chars:
532
+                self.send_key(char)
533
+
534
+    def wait_for_file(self, filename: str) -> None:
535
+        """Waits until the file exists in machine's file system."""
536
+
537
+        def check_file(_: Any) -> bool:
538
+            status, _ = self.execute("test -e {}".format(filename))
539
+            return status == 0
540
+
541
+        with self.nested("waiting for file ‘{}‘".format(filename)):
542
+            retry(check_file)
543
+
544
+    def wait_for_open_port(self, port: int) -> None:
545
+        def port_is_open(_: Any) -> bool:
546
+            status, _ = self.execute("nc -z localhost {}".format(port))
547
+            return status == 0
548
+
549
+        with self.nested("waiting for TCP port {}".format(port)):
550
+            retry(port_is_open)
551
+
552
+    def wait_for_closed_port(self, port: int) -> None:
553
+        def port_is_closed(_: Any) -> bool:
554
+            status, _ = self.execute("nc -z localhost {}".format(port))
555
+            return status != 0
556
+
557
+        retry(port_is_closed)
558
+
559
+    def start_job(self, jobname: str, user: Optional[str] = None) -> Tuple[int, str]:
560
+        return self.systemctl("start {}".format(jobname), user)
561
+
562
+    def stop_job(self, jobname: str, user: Optional[str] = None) -> Tuple[int, str]:
563
+        return self.systemctl("stop {}".format(jobname), user)
564
+
565
+    def wait_for_job(self, jobname: str) -> None:
566
+        self.wait_for_unit(jobname)
567
+
568
+    def connect(self) -> None:
569
+        if self.connected:
570
+            return
571
+
572
+        with self.nested("waiting for the VM to finish booting"):
573
+            self.start()
574
+
575
+            tic = time.time()
576
+            self.shell.recv(1024)
577
+            # TODO: Timeout
578
+            toc = time.time()
579
+
580
+            self.log("connected to guest root shell")
581
+            self.log("(connecting took {:.2f} seconds)".format(toc - tic))
582
+            self.connected = True
583
+
584
+    def screenshot(self, filename: str) -> None:
585
+        out_dir = os.environ.get("out", os.getcwd())
586
+        word_pattern = re.compile(r"^\w+$")
587
+        if word_pattern.match(filename):
588
+            filename = os.path.join(out_dir, "{}.png".format(filename))
589
+        tmp = "{}.ppm".format(filename)
590
+
591
+        with self.nested(
592
+            "making screenshot {}".format(filename),
593
+            {"image": os.path.basename(filename)},
594
+        ):
595
+            self.send_monitor_command("screendump {}".format(tmp))
596
+            ret = subprocess.run("pnmtopng {} > {}".format(tmp, filename), shell=True)
597
+            os.unlink(tmp)
598
+            if ret.returncode != 0:
599
+                raise Exception("Cannot convert screenshot")
600
+
601
+    def copy_from_host_via_shell(self, source: str, target: str) -> None:
602
+        """Copy a file from the host into the guest by piping it over the
603
+        shell into the destination file. Works without host-guest shared folder.
604
+        Prefer copy_from_host for whenever possible.
605
+        """
606
+        with open(source, "rb") as fh:
607
+            content_b64 = base64.b64encode(fh.read()).decode()
608
+            self.succeed(
609
+                f"mkdir -p $(dirname {target})",
610
+                f"echo -n {content_b64} | base64 -d > {target}",
611
+            )
612
+
613
+    def copy_from_host(self, source: str, target: str) -> None:
614
+        """Copy a file from the host into the guest via the `shared_dir` shared
615
+        among all the VMs (using a temporary directory).
616
+        """
617
+        host_src = pathlib.Path(source)
618
+        vm_target = pathlib.Path(target)
619
+        with tempfile.TemporaryDirectory(dir=self.shared_dir) as shared_td:
620
+            shared_temp = pathlib.Path(shared_td)
621
+            host_intermediate = shared_temp / host_src.name
622
+            vm_shared_temp = pathlib.Path("/tmp/shared") / shared_temp.name
623
+            vm_intermediate = vm_shared_temp / host_src.name
624
+
625
+            self.succeed(make_command(["mkdir", "-p", vm_shared_temp]))
626
+            if host_src.is_dir():
627
+                shutil.copytree(host_src, host_intermediate)
628
+            else:
629
+                shutil.copy(host_src, host_intermediate)
630
+            self.succeed("sync")
631
+            self.succeed(make_command(["mkdir", "-p", vm_target.parent]))
632
+            self.succeed(make_command(["cp", "-r", vm_intermediate, vm_target]))
633
+        # Make sure the cleanup is synced into VM
634
+        self.succeed("sync")
635
+
636
+    def copy_from_vm(self, source: str, target_dir: str = "") -> None:
637
+        """Copy a file from the VM (specified by an in-VM source path) to a path
638
+        relative to `$out`. The file is copied via the `shared_dir` shared among
639
+        all the VMs (using a temporary directory).
640
+        """
641
+        # Compute the source, target, and intermediate shared file names
642
+        out_dir = pathlib.Path(os.environ.get("out", os.getcwd()))
643
+        vm_src = pathlib.Path(source)
644
+        with tempfile.TemporaryDirectory(dir=self.shared_dir) as shared_td:
645
+            shared_temp = pathlib.Path(shared_td)
646
+            vm_shared_temp = pathlib.Path("/tmp/shared") / shared_temp.name
647
+            vm_intermediate = vm_shared_temp / vm_src.name
648
+            intermediate = shared_temp / vm_src.name
649
+            # Copy the file to the shared directory inside VM
650
+            self.succeed(make_command(["mkdir", "-p", vm_shared_temp]))
651
+            self.succeed(make_command(["cp", "-r", vm_src, vm_intermediate]))
652
+            self.succeed("sync")
653
+            abs_target = out_dir / target_dir / vm_src.name
654
+            abs_target.parent.mkdir(exist_ok=True, parents=True)
655
+            # Copy the file from the shared directory outside VM
656
+            if intermediate.is_dir():
657
+                shutil.copytree(intermediate, abs_target)
658
+            else:
659
+                shutil.copy(intermediate, abs_target)
660
+        # Make sure the cleanup is synced into VM
661
+        self.succeed("sync")
662
+
663
+    def dump_tty_contents(self, tty: str) -> None:
664
+        """Debugging: Dump the contents of the TTY<n>
665
+        """
666
+        self.execute("fold -w 80 /dev/vcs{} | systemd-cat".format(tty))
667
+
668
+    def get_screen_text(self) -> str:
669
+        if shutil.which("tesseract") is None:
670
+            raise Exception("get_screen_text used but enableOCR is false")
671
+
672
+        magick_args = (
673
+            "-filter Catrom -density 72 -resample 300 "
674
+            + "-contrast -normalize -despeckle -type grayscale "
675
+            + "-sharpen 1 -posterize 3 -negate -gamma 100 "
676
+            + "-blur 1x65535"
677
+        )
678
+
679
+        tess_args = "-c debug_file=/dev/null --psm 11 --oem 2"
680
+
681
+        with self.nested("performing optical character recognition"):
682
+            with tempfile.NamedTemporaryFile() as tmpin:
683
+                self.send_monitor_command("screendump {}".format(tmpin.name))
684
+
685
+                cmd = "convert {} {} tiff:- | tesseract - - {}".format(
686
+                    magick_args, tmpin.name, tess_args
687
+                )
688
+                ret = subprocess.run(cmd, shell=True, capture_output=True)
689
+                if ret.returncode != 0:
690
+                    raise Exception(
691
+                        "OCR failed with exit code {}".format(ret.returncode)
692
+                    )
693
+
694
+                return ret.stdout.decode("utf-8")
695
+
696
+    def wait_for_text(self, regex: str) -> None:
697
+        def screen_matches(last: bool) -> bool:
698
+            text = self.get_screen_text()
699
+            matches = re.search(regex, text) is not None
700
+
701
+            if last and not matches:
702
+                self.log("Last OCR attempt failed. Text was: {}".format(text))
703
+
704
+            return matches
705
+
706
+        with self.nested("waiting for {} to appear on screen".format(regex)):
707
+            retry(screen_matches)
708
+
709
+    def send_key(self, key: str) -> None:
710
+        key = CHAR_TO_KEY.get(key, key)
711
+        self.send_monitor_command("sendkey {}".format(key))
712
+
713
+    def start(self) -> None:
714
+        if self.booted:
715
+            return
716
+
717
+        self.log("starting vm")
718
+
719
+        def create_socket(path: str) -> socket.socket:
720
+            if os.path.exists(path):
721
+                os.unlink(path)
722
+            s = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)
723
+            s.bind(path)
724
+            s.listen(1)
725
+            return s
726
+
727
+        monitor_path = os.path.join(self.state_dir, "monitor")
728
+        self.monitor_socket = create_socket(monitor_path)
729
+
730
+        shell_path = os.path.join(self.state_dir, "shell")
731
+        self.shell_socket = create_socket(shell_path)
732
+
733
+        qemu_options = (
734
+            " ".join(
735
+                [
736
+                    "" if self.allow_reboot else "-no-reboot",
737
+                    "-monitor unix:{}".format(monitor_path),
738
+                    "-chardev socket,id=shell,path={}".format(shell_path),
739
+                    "-device virtio-serial",
740
+                    "-device virtconsole,chardev=shell",
741
+                    "-device virtio-rng-pci",
742
+                    "-serial stdio" if "DISPLAY" in os.environ else "-nographic",
743
+                ]
744
+            )
745
+            + " "
746
+            + os.environ.get("QEMU_OPTS", "")
747
+        )
748
+
749
+        environment = dict(os.environ)
750
+        environment.update(
751
+            {
752
+                "TMPDIR": self.state_dir,
753
+                "SHARED_DIR": self.shared_dir,
754
+                "USE_TMPDIR": "1",
755
+                "QEMU_OPTS": qemu_options,
756
+            }
757
+        )
758
+
759
+        self.process = subprocess.Popen(
760
+            self.script,
761
+            bufsize=1,
762
+            stdin=subprocess.DEVNULL,
763
+            stdout=subprocess.PIPE,
764
+            stderr=subprocess.STDOUT,
765
+            shell=True,
766
+            cwd=self.state_dir,
767
+            env=environment,
768
+        )
769
+        self.monitor, _ = self.monitor_socket.accept()
770
+        self.shell, _ = self.shell_socket.accept()
771
+
772
+        def process_serial_output() -> None:
773
+            assert self.process.stdout is not None
774
+            for _line in self.process.stdout:
775
+                # Ignore undecodable bytes that may occur in boot menus
776
+                line = _line.decode(errors="ignore").replace("\r", "").rstrip()
777
+                eprint("{} # {}".format(self.name, line))
778
+                self.logger.enqueue({"msg": line, "machine": self.name})
779
+                self.serialQueue.put(line)
780
+
781
+        _thread.start_new_thread(process_serial_output, ())
782
+
783
+        self.wait_for_monitor_prompt()
784
+
785
+        self.pid = self.process.pid
786
+        self.booted = True
787
+
788
+        self.log("QEMU running (pid {})".format(self.pid))
789
+
790
+    def shutdown(self) -> None:
791
+        if not self.booted:
792
+            return
793
+
794
+        self.shell.send("poweroff\n".encode())
795
+        self.wait_for_shutdown()
796
+
797
+    def crash(self) -> None:
798
+        if not self.booted:
799
+            return
800
+
801
+        self.log("forced crash")
802
+        self.send_monitor_command("quit")
803
+        self.wait_for_shutdown()
804
+
805
+    def wait_for_x(self) -> None:
806
+        """Wait until it is possible to connect to the X server.  Note that
807
+        testing the existence of /tmp/.X11-unix/X0 is insufficient.
808
+        """
809
+
810
+        def check_x(_: Any) -> bool:
811
+            cmd = (
812
+                "journalctl -b SYSLOG_IDENTIFIER=systemd | "
813
+                + 'grep "Reached target Current graphical"'
814
+            )
815
+            status, _ = self.execute(cmd)
816
+            if status != 0:
817
+                return False
818
+            status, _ = self.execute("[ -e /tmp/.X11-unix/X0 ]")
819
+            return status == 0
820
+
821
+        with self.nested("waiting for the X11 server"):
822
+            retry(check_x)