From 40c1977779700bb58381b3649c22db6f6af14f98 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Sun, 31 May 2020 15:05:24 +0530 Subject: [PATCH] Replace test framework Adopt the Python test driver from NixOS. Temporarily drop Sotest runs. --- README.md | 10 +- flake.nix | 2 + lib/default.nix | 6 +- tests/default.nix | 356 +++++++++--- tests/driver-hw.nix | 173 ------ tests/driver-nova.nix | 179 ------ tests/driver_manager.nix | 37 -- tests/log.nix | 17 +- tests/posix.nix | 21 +- tests/solo5/default.nix | 56 +- tests/test-driver/test-driver.py | 912 +++++++++++++++++++++++++++++++ tests/vmm_x86.nix | 16 +- tests/x86.nix | 24 +- 13 files changed, 1273 insertions(+), 536 deletions(-) delete mode 100644 tests/driver-hw.nix delete mode 100644 tests/driver-nova.nix delete mode 100644 tests/driver_manager.nix create mode 100644 tests/test-driver/test-driver.py diff --git a/README.md b/README.md index 1373b18..df82491 100644 --- a/README.md +++ b/README.md @@ -135,14 +135,14 @@ Tests are performed using QEMU, the test artifacts are built as follows: # Build a test log from a QEMU test run: nix build .#checks.x86_64-linux.nova-x86 -# Build an ISO of the test run: -nix build .#checks.x86_64-linux.nova-x86.iso +# Build a tarball of the Nix store internal to a test VM: +nix build .#checks.x86_64-linux.nova-x86.nodes.machine.store -# Build a tarball of the Nix store internal to a test: -nix build .#checks.x86_64-linux.nova-x86.store +# Build the XML configuration of the test VM: +nix build .#checks.x86_64-linux.nova-x86.nodes.machine.xml # Build the Dhall boot description of the test run: -nix build .#checks.x86_64-linux.nova-x86.config +nix build .#checks.x86_64-linux.nova-x86.nodes.machine.config ``` # System description format diff --git a/flake.nix b/flake.nix index 261bcd8..4fba9f7 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,8 @@ # SPDX-License-Identifier: CC0-1.0 { + edition = 201909; + description = "Genode packages"; inputs.nixpkgs.url = "github:ehmry/nixpkgs?ref=genode"; diff --git a/lib/default.nix b/lib/default.nix index e01b114..4f2d4bc 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -31,7 +31,7 @@ in rec { ''; compileBoot = name: env: bootDhall: - runDhallCommand name env '' + runDhallCommand "${name}-boot" env '' dhall to-directory-tree --output $out \ <<< "${./compile-boot.dhall} (${bootDhall}) \"$out\"" dhall <<< "(${bootDhall}).config" \ @@ -41,7 +41,7 @@ in rec { hwImage = name: env: boot: nixpkgs.stdenv.mkDerivation { - name = name + ".image.elf"; + name = name + "-hw-image"; build = compileBoot name env boot; nativeBuildInputs = [ buildPackages.dhall ]; buildCommand = let @@ -99,7 +99,7 @@ in rec { novaImage = name: env: boot: nixpkgs.stdenv.mkDerivation { - name = name + ".image.elf"; + name = name + "-nova-image"; build = compileBoot name env boot; buildCommand = '' diff --git a/tests/default.nix b/tests/default.nix index d2608a0..27e7ed7 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -1,89 +1,299 @@ # SPDX-License-Identifier: CC0-1.0 -let tests = call: { log = call ./log.nix { }; }; - -in { self, apps, buildPackages, genodepkgs, lib, nixpkgs, legacyPackages }: +{ self, apps, buildPackages, genodepkgs, lib, nixpkgs, legacyPackages }: let + + callTest = path: + import path { + pkgs = testPkgs; + inherit nixpkgs buildPackages legacyPackages; + }; + + testFiles = map callTest [ + ./log.nix + ./posix.nix + ./vmm_x86.nix + ./x86.nix + ] ++ (callTest ./solo5); + testPkgs = genodepkgs; - addManifest = drv: - drv // { - manifest = nixpkgs.runCommand "${drv.name}.dhall" { inherit drv; } '' - set -eu - echo -n '[' >> $out - find $drv/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out - ${if builtins.elem "lib" drv.outputs then - '' - find ${drv.lib}/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out'' - else - ""} - echo -n ']' >> $out + qemu' = buildPackages.qemu_test; + + qemuBinary = qemuPkg: + { + aarch64-genode = "${qemuPkg}/bin/qemu-system-aarch64"; + x86_64-genode = "${qemuPkg}/bin/qemu-system-x86_64"; + }.${genodepkgs.stdenv.hostPlatform.system}; + + platforms = [ + { + prefix = "hw-pc-"; + specs = [ "x86" "hw" ]; + basePkg = testPkgs.base-hw-pc; + makeImage = lib.hwImage; + startVM = vmName: image: '' + #! ${buildPackages.runtimeShell} + exec ${qemuBinary qemu'} \ + -name ${vmName} \ + -machine q35 \ + -m 384 \ + -kernel "${testPkgs.bender}/bender" \ + -initrd "${image}/image.elf" \ + $QEMU_OPTS \ + "$@" + ''; + } + { + prefix = "nova-"; + specs = [ "x86" "nova" ]; + basePkg = testPkgs.base-nova; + makeImage = lib.novaImage; + startVM = vmName: image: '' + #! ${buildPackages.runtimeShell} + exec ${qemuBinary qemu'} \ + -name ${vmName} \ + -machine q35 \ + -m 384 \ + -kernel "${testPkgs.bender}/bender" \ + -initrd "${testPkgs.NOVA}/hypervisor-x86_64 arg=iommu novpid serial,${image}/image.elf" \ + $QEMU_OPTS \ + "$@" + ''; + } + ]; + + testDriver = with buildPackages; + let testDriverScript = ./test-driver/test-driver.py; + in stdenv.mkDerivation { + name = "nixos-test-driver"; + + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ]; + checkInputs = with python3Packages; [ pylint mypy ]; + + dontUnpack = true; + + preferLocalBuild = true; + + doCheck = true; + checkPhase = '' + mypy --disallow-untyped-defs \ + --no-implicit-optional \ + --ignore-missing-imports ${testDriverScript} + pylint --errors-only ${testDriverScript} + ''; + + installPhase = '' + mkdir -p $out/bin + cp ${testDriverScript} $out/bin/nixos-test-driver + chmod u+x $out/bin/nixos-test-driver + # TODO: copy user script part into this file (append) + + wrapProgram $out/bin/nixos-test-driver \ + --prefix PATH : "${lib.makeBinPath [ qemu' coreutils ]}" \ ''; }; - nova = (call: - ((tests call) // { - driver_manager = call ./driver_manager.nix { }; - posix = call ./posix.nix { }; - vmm = call ./vmm_x86.nix { }; - x86 = call ./x86.nix { }; - } // call ./solo5 { })) (import ./driver-nova.nix { - inherit apps addManifest buildPackages lib nixpkgs testPkgs - legacyPackages; - }).callTest; + defaultTestScript = '' + start_all() + machine.wait_until_serial_output('child "init" exited with exit value 0') + ''; - hw = (call: - ((tests call) // { - posix = call ./posix.nix { }; - x86 = call ./x86.nix { }; - } // call ./solo5 { })) (import ./driver-hw.nix { - inherit apps addManifest buildPackages lib nixpkgs testPkgs - legacyPackages; - }).callTest; + makeTest = with buildPackages; + { prefix, specs, basePkg, makeImage, startVM }: + { name ? "unnamed", testScript ? defaultTestScript, + # Skip linting (mainly intended for faster dev cycles) + skipLint ? false, ... }@t: - testsToList = tests: - map (test: { - inherit (test) name; - value = test; - }) (builtins.attrValues tests); + let + testDriverName = "genode-test-driver-${name}"; -in with builtins; -listToAttrs ((concatLists (map (testsToList) [ hw nova ]))) // { - sotest = let - hwTests = with hw; [ multi posix x86 ]; - novaTests = with nova; [ multi posix x86 vmm ]; - allTests = hwTests ++ novaTests; + buildVM = vmName: + { config, inputs, env ? { }, extraPaths ? [ ] }: + let + storeTarball = buildPackages.runCommand "store" { } '' + mkdir -p $out + tar cf "$out/store.tar" --absolute-names ${toString inputs} ${ + toString extraPaths + } + ''; + addManifest = drv: + drv // { + manifest = + nixpkgs.runCommand "${drv.name}.dhall" { inherit drv; } '' + set -eu + echo -n '[' >> $out + find $drv/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out + ${if builtins.elem "lib" drv.outputs then + '' + find ${drv.lib}/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out'' + else + ""} + echo -n ']' >> $out + ''; + }; - projectCfg.boot_items = + storeManifest = lib.mergeManifests (map addManifest inputs); + manifest = lib.mergeManifests (map addManifest + ([ basePkg testPkgs.sotest-producer storeTarball ] + ++ map testPkgs.genodeSources.depot [ + "init" + "rtc_drv" + "vfs" + "cached_fs_rom" + ])); + config' = "${ + ./test-wrapper.dhall + } (${config}) $(stat --format '%s' ${storeTarball}/store.tar) ${storeManifest} ${manifest}"; + env' = { + DHALL_GENODE = "${testPkgs.dhallGenode}/source.dhall"; + DHALL_GENODE_TEST = "${./test.dhall}"; + } // env; - (map (test: { - inherit (test) name; - exec = "bender"; - load = [ test.image.name ]; - }) hwTests) + image = makeImage vmName env' config'; + startVM' = startVM vmName image; + in { + script = buildPackages.writeScriptBin "run-${vmName}-vm" startVM'; - ++ (map (test: { - inherit (test) name; - exec = "bender"; - load = [ "hypervisor serial novga iommu" test.image.name ]; - }) novaTests); + config = lib.runDhallCommand (name + ".dhall") env' '' + ${apps.dhall.program} <<< "${config'}" > $out + ''; - in buildPackages.stdenv.mkDerivation { - name = "sotest"; - buildCommand = '' - mkdir zip; cd zip - cp "${testPkgs.bender}/bender" bender - cp "${testPkgs.NOVA}/hypervisor-x86_64" hypervisor - ${concatStringsSep "\n" - (map (test: "cp ${test.image}/image.elf ${test.image.name}") allTests)} - mkdir -p $out/nix-support - ${buildPackages.zip}/bin/zip "$out/binaries.zip" * - cat << EOF > "$out/project.json" - ${builtins.toJSON projectCfg} - EOF - echo file sotest-binaries $out/binaries.zip >> "$out/nix-support/hydra-build-products" - echo file sotest-config $out/project.json >> "$out/nix-support/hydra-build-products" - ''; - }; -} + store = storeTarball; + + xml = lib.runDhallCommand (name + ".config") env' + ''${apps.render-init.program} <<< "(${config'}).config" > $out''; + }; + + nodes = lib.mapAttrs buildVM + (t.nodes or (if t ? machine then { machine = t.machine; } else { })); + + testScript' = + # Call the test script with the computed nodes. + if lib.isFunction testScript then + testScript { inherit nodes; } + else + testScript; + + vms = map (node: node.script) (lib.attrValues nodes); + + # Generate onvenience wrappers for running the test driver + # interactively with the specified network, and for starting the + # VMs from the command line. + driver = + let warn = if skipLint then lib.warn "Linting is disabled!" else lib.id; + in warn (runCommand testDriverName { + buildInputs = [ makeWrapper ]; + testScript = testScript'; + preferLocalBuild = true; + testName = name; + } '' + mkdir -p $out/bin + + echo -n "$testScript" > $out/test-script + ${lib.optionalString (!skipLint) '' + ${python3Packages.black}/bin/black --check --quiet --diff $out/test-script + ''} + + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/ + vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done)) + wrapProgram $out/bin/nixos-test-driver \ + --add-flags "''${vms[*]}" \ + --run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms + wrapProgram $out/bin/nixos-run-vms \ + --add-flags "''${vms[*]}" \ + --set tests 'start_all(); join_all();' + ''); # " + + passMeta = drv: + drv + // lib.optionalAttrs (t ? meta) { meta = (drv.meta or { }) // t.meta; }; + + # Run an automated test suite in the given virtual network. + # `driver' is the script that runs the network. + runTests = driver: + stdenv.mkDerivation { + name = "test-run-${driver.testName}"; + + buildCommand = '' + mkdir -p $out + + LOGFILE=/dev/null tests='exec(os.environ["testScript"])' ${driver}/bin/nixos-test-driver + ''; + }; + + test = passMeta (runTests driver); + + nodeNames = builtins.attrNames nodes; + invalidNodeNames = + lib.filter (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null) + nodeNames; + + in if lib.length invalidNodeNames > 0 then + throw '' + Cannot create machines out of (${ + lib.concatStringsSep ", " invalidNodeNames + })! + All machines are referenced as python variables in the testing framework which will break the + script when special characters are used. + + Please stick to alphanumeric chars and underscores as separation. + '' + else + test // { inherit nodes driver test; }; + + testList = let + f = platform: + let makeTest' = makeTest platform; + in test: + if (test.constraints or (_: true)) platform.specs then { + name = platform.prefix + test.name; + value = makeTest' test; + } else + null; + + in lib.lists.crossLists f [ platforms testFiles ]; + +in builtins.listToAttrs (builtins.filter (_: _ != null) testList) + +/* sotest = let + hwTests = with hw; [ multi posix x86 ]; + novaTests = with nova; [ multi posix x86 vmm ]; + allTests = hwTests ++ novaTests; + + projectCfg.boot_items = + + (map (test: { + inherit (test) name; + exec = "bender"; + load = [ "${test.name}.image.elf" ]; + }) hwTests) + + ++ (map (test: { + inherit (test) name; + exec = "bender"; + load = [ "hypervisor serial novga iommu" test.image.name ]; + }) novaTests); + + in buildPackages.stdenv.mkDerivation { + name = "sotest"; + buildCommand = '' + mkdir zip; cd zip + cp "${testPkgs.bender}/bender" bender + cp "${testPkgs.NOVA}/hypervisor-x86_64" hypervisor + ${concatStringsSep "\n" + (map (test: "cp ${test.image}/image.elf ${test.name}.image.elf") + allTests)} + mkdir -p $out/nix-support + ${buildPackages.zip}/bin/zip "$out/binaries.zip" * + cat << EOF > "$out/project.json" + ${builtins.toJSON projectCfg} + EOF + echo file sotest-binaries $out/binaries.zip >> "$out/nix-support/hydra-build-products" + echo file sotest-config $out/project.json >> "$out/nix-support/hydra-build-products" + ''; + }; +*/ diff --git a/tests/driver-hw.nix b/tests/driver-hw.nix deleted file mode 100644 index 123fe0e..0000000 --- a/tests/driver-hw.nix +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -{ addManifest, apps, buildPackages, lib, nixpkgs, testPkgs, legacyPackages }: - -let - testDriver = with buildPackages; - stdenv.mkDerivation { - name = "hw-genode-test-driver"; - preferLocalBuild = true; - - buildInputs = [ makeWrapper expect ]; - - dontUnpack = true; - - installPhase = '' - install -Dm555 ${./hw-test-driver.exp} $out/bin/genode-test-driver - wrapProgram $out/bin/genode-test-driver \ - --prefix PATH : "${lib.makeBinPath [ expect coreutils ]}" - ''; - }; - - runTests = driver: - buildPackages.stdenv.mkDerivation { - name = "hw-" + driver.testName; - preferLocalBuild = true; - - buildCommand = '' - mkdir -p $out/nix-support - - ${driver}/bin/genode-test-driver | tee $out/log - - touch $out/nix-support - echo "report testlog $out log" >> $out/nix-support/hydra-build-products - ''; - }; - - defaultScript = - ''run_genode_until {child "init" exited with exit value 0} 120''; - - mkTest = { name ? "unamed", testScript ? defaultScript, testConfig - , testInputs ? [ ], testEnv ? { }, extraPaths ? [ ], qemuArgs ? [ ], ... - }@t: - let - storeTarball = buildPackages.runCommand "store" { } '' - mkdir -p $out - tar cf "$out/store.tar" --absolute-names ${toString testInputs} ${ - toString extraPaths - } - ''; - storeManifest = lib.mergeManifests (map addManifest testInputs); - manifest = lib.mergeManifests (map addManifest - ([ testPkgs.base-hw-pc testPkgs.sotest-producer storeTarball ] - ++ map testPkgs.genodeSources.depot [ - "init" - "rtc_drv" - "vfs" - "cached_fs_rom" - ])); - testConfig' = "${ - ./test-wrapper.dhall - } (${testConfig}) $(stat --format '%s' ${storeTarball}/store.tar) ${storeManifest} ${manifest}"; - testEnv' = { - DHALL_GENODE = "${testPkgs.dhallGenode}/source.dhall"; - DHALL_GENODE_TEST = "${./test.dhall}"; - } // testEnv; - - image = lib.hwImage ("hw-" + name) testEnv' testConfig'; - - baseSetup = '' - ## - # Wait for a specific output of a already running spawned proce - # - proc wait_for_output { wait_for_re timeout_value running_spawn_id } { - global output - - if {$wait_for_re == "forever"} { - set timeout -1 - interact { - \003 { - send_user "Expect: 'interact' received 'strg+c' and was cancelled\n"; - exit - } - -i $running_spawn_id - } - } else { - set timeout $timeout_value - } - - expect { - -i $running_spawn_id -re $wait_for_re { } - eof { puts stderr "Error: Spawned process died unexpectedly"; exit -1 } - timeout { puts stderr "Error: Test execution timed out"; exit -1 } - } - set output $expect_out(buffer) - } - - proc run_genode_until {{wait_for_re forever} {timeout_value 0} {running_spawn_id -1}} { - # - # If a running_spawn_id is specified, wait for the expected output - # - if {$running_spawn_id != -1} { - wait_for_output $wait_for_re $timeout_value $running_spawn_id - return - } - - global env - global spawn_id - set TEST_MIB [expr (([file size ${image}/image.elf] + $env(TEST_RAM)) >> 20) + 24] - spawn ${buildPackages.qemu_test}/bin/qemu-system-x86_64 \ - -machine q35 -serial mon:stdio -nographic \ - -m size=$TEST_MIB \ - -kernel "${testPkgs.bender}/bender" \ - -initrd "${image}/image.elf" \ - ${toString qemuArgs} - wait_for_output $wait_for_re $timeout_value $spawn_id - } - - # TODO: not in TCL - global env - set out $env(out) - ''; - - driver = with buildPackages; - buildPackages.runCommand "genode-test-driver-${name}" ({ - buildInputs = [ makeWrapper expect ]; - inherit baseSetup testScript; - preferLocalBuild = true; - testName = name; - } // testEnv') '' - mkdir -p $out/bin - echo "$testScript" > $out/test-script - echo "$baseSetup" > $out/base-setup - source ${image.build}/stats - - ln -s ${testDriver}/bin/genode-test-driver $out/bin/ - wrapProgram $out/bin/genode-test-driver \ - --set testScript "$testScript" \ - --set baseSetup "$baseSetup" \ - --set TEST_RAM $RAM \ - ''; - - passMeta = drv: - drv - // lib.optionalAttrs (t ? meta) { meta = (drv.meta or { }) // t.meta; }; - - test = passMeta (runTests driver); - - in test // { - inherit driver image test manifest; - - config = buildPackages.runCommand (name + ".dhall") testEnv' '' - ${apps.dhall.program} <<< "${testConfig'}" > $out - ''; - - iso = apps.hw-iso.function testEnv' testConfig'; - - xml = buildPackages.runCommand (name + ".config") testEnv' - ''${apps.render-init.program} <<< "(${testConfig'}).config" > $out''; - - }; - -in { - callTest = path: args: - (import path ({ - testEnv = { - inherit mkTest lib; - isLinux = false; - isNova = true; - }; - pkgs = testPkgs; - inherit nixpkgs buildPackages legacyPackages; - } // args)); -} diff --git a/tests/driver-nova.nix b/tests/driver-nova.nix deleted file mode 100644 index d2c7f03..0000000 --- a/tests/driver-nova.nix +++ /dev/null @@ -1,179 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -{ addManifest, apps, buildPackages, lib, nixpkgs, testPkgs, legacyPackages }: - -let - testDriver = with buildPackages; - stdenv.mkDerivation { - name = "nova-genode-test-driver"; - preferLocalBuild = true; - - buildInputs = [ makeWrapper expect ]; - - dontUnpack = true; - - installPhase = '' - install -Dm555 ${./nova-test-driver.exp} $out/bin/genode-test-driver - wrapProgram $out/bin/genode-test-driver \ - --prefix PATH : "${lib.makeBinPath [ expect coreutils ]}" - ''; - }; - - runTests = driver: - buildPackages.stdenv.mkDerivation { - name = "nova-" + driver.testName; - preferLocalBuild = true; - - buildCommand = '' - mkdir -p $out/nix-support - - ${driver}/bin/genode-test-driver | tee $out/log - - touch $out/nix-support - echo "report testlog $out log" >> $out/nix-support/hydra-build-products - ''; - }; - - defaultScript = - ''run_genode_until {child "init" exited with exit value 0} 120''; - - mkTest = { name ? "unamed", testScript ? defaultScript, testConfig - , testInputs ? [ ], testEnv ? { }, extraPaths ? [ ], qemuArgs ? [ ], ... - }@t: - let - storeTarball = buildPackages.runCommand "store" { } '' - mkdir -p $out - tar cf "$out/store.tar" --absolute-names ${toString testInputs} ${ - toString extraPaths - } - ''; - storeManifest = lib.mergeManifests (map addManifest testInputs); - manifest = lib.mergeManifests (map addManifest - ([ testPkgs.base-nova testPkgs.sotest-producer storeTarball ] - ++ map testPkgs.genodeSources.depot [ - "init" - "rtc_drv" - "vfs" - "cached_fs_rom" - ])); - testConfig' = "${ - ./test-wrapper.dhall - } (${testConfig}) $(stat --format '%s' ${storeTarball}/store.tar) ${storeManifest} ${manifest}"; - testEnv' = { - DHALL_GENODE = "${testPkgs.dhallGenode}/source.dhall"; - DHALL_GENODE_TEST = "${./test.dhall}"; - } // testEnv; - - image = lib.novaImage ("nova-" + name) testEnv' testConfig'; - build = lib.compileBoot name testEnv' testConfig'; - - baseSetup = '' - ## - # Wait for a specific output of a already running spawned proce - # - proc wait_for_output { wait_for_re timeout_value running_spawn_id } { - global output - - if {$wait_for_re == "forever"} { - set timeout -1 - interact { - \003 { - send_user "Expect: 'interact' received 'strg+c' and was cancelled\n"; - exit - } - -i $running_spawn_id - } - } else { - set timeout $timeout_value - } - - expect { - -i $running_spawn_id -re $wait_for_re { } - eof { puts stderr "Error: Spawned process died unexpectedly"; exit -1 } - timeout { puts stderr "Error: Test execution timed out"; exit -1 } - } - set output $expect_out(buffer) - } - - proc run_genode_until {{wait_for_re forever} {timeout_value 0} {running_spawn_id -1}} { - # - # If a running_spawn_id is specified, wait for the expected output - # - if {$running_spawn_id != -1} { - wait_for_output $wait_for_re $timeout_value $running_spawn_id - return - } - - global env - global spawn_id - set TEST_MIB [expr (([file size ${image}/image.elf] + $env(TEST_RAM)) >> 20) + 24] - spawn ${buildPackages.qemu_test}/bin/qemu-system-x86_64 \ - -machine q35 -cpu phenom -smp 2 \ - -serial mon:stdio -nographic \ - -m size=$TEST_MIB \ - -kernel "${testPkgs.bender}/bender" \ - -initrd "${testPkgs.NOVA}/hypervisor-x86_64 arg=iommu novpid serial,${image}/image.elf" \ - ${toString qemuArgs} - wait_for_output $wait_for_re $timeout_value $spawn_id - } - - # TODO: not in TCL - global env - set out $env(out) - ''; - - driver = with buildPackages; - buildPackages.runCommand "genode-test-driver-${name}" ({ - buildInputs = [ makeWrapper expect ]; - inherit baseSetup testScript; - preferLocalBuild = true; - testName = name; - } // testEnv') '' - mkdir -p $out/bin - echo "$testScript" > $out/test-script - echo "$baseSetup" > $out/base-setup - source ${image.build}/stats - - ln -s ${testDriver}/bin/genode-test-driver $out/bin/ - wrapProgram $out/bin/genode-test-driver \ - --set testScript "$testScript" \ - --set baseSetup "$baseSetup" \ - --set TEST_RAM $RAM \ - ''; - - passMeta = drv: - drv - // lib.optionalAttrs (t ? meta) { meta = (drv.meta or { }) // t.meta; }; - - test = passMeta (runTests driver); - - in test // { - inherit driver image test manifest; - inherit (image) build; - - config = buildPackages.runCommand (name + ".dhall") testEnv' '' - ${apps.dhall.program} <<< "${testConfig'}" > $out - ''; - - compile = lib.compileBoot name testConfig'; - - iso = apps.nova-iso.function testEnv' "${testConfig'}"; - - store = storeTarball; - - xml = buildPackages.runCommand (name + ".config") testEnv' - ''${apps.render-init.program} <<< "(${testConfig'}).config" > $out''; - }; - -in { - callTest = path: args: - (import path ({ - testEnv = { - inherit mkTest lib; - isLinux = false; - isNova = true; - }; - pkgs = testPkgs; - inherit nixpkgs buildPackages legacyPackages; - } // args)); -} diff --git a/tests/driver_manager.nix b/tests/driver_manager.nix deleted file mode 100644 index e8d68de..0000000 --- a/tests/driver_manager.nix +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-License-Identifier: CC0-1.0 - -{ testEnv, pkgs, buildPackages, ... }: -with pkgs; - -testEnv.mkTest { - name = "driver_manager"; - meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; - testEnv = { drivers = ./../compositions/pc-drivers.dhall; }; - testInputs = (map pkgs.genodeSources.depot [ - "acpi_drv" - "ahci_drv" - "boot_fb_drv" - "driver_manager" - "dynamic_rom" - "input_filter" - "intel_fb_drv" - "platform_drv" - "ps2_drv" - "report_rom" - "rom_reporter" - "usb_drv" - "vesa_drv" - ]) ++ (map pkgs.genodeSources.make [ "test/driver_manager" ]); - - testScript = '' - catch { exec dd if=/dev/zero of=hdd_disk.raw bs=1M count=32 } - catch { exec ${buildPackages.e2fsprogs}/bin/mke2fs -F bin/hdd_disk.raw } - run_genode_until {.*all expected devices present and accessible.*} 120 - ''; - testConfig = ./driver_manager.dhall; - qemuArgs = [ - "-device ahci,id=ahci" - "-drive id=hdd,file=hdd_disk.raw,format=raw,if=none" - "-device ide-hd,drive=hdd,bus=ahci.1" - ]; -} diff --git a/tests/log.nix b/tests/log.nix index ff3b835..a9a4d50 100644 --- a/tests/log.nix +++ b/tests/log.nix @@ -1,13 +1,16 @@ # SPDX-License-Identifier: CC0-1.0 -{ testEnv, pkgs, ... }: +{ pkgs, ... }: with pkgs; -testEnv.mkTest rec { +{ name = "log"; - meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; - - testConfig = ./log.dhall; - testInputs = [ (pkgs.genodeSources.depot "test-log") ]; - testScript = "run_genode_until {Test done.} 120"; + machine = { + config = ./log.dhall; + inputs = [ (pkgs.genodeSources.depot "test-log") ]; + }; + testScript = '' + start_all() + machine.wait_until_serial_output("Test done.") + ''; } diff --git a/tests/posix.nix b/tests/posix.nix index 667edac..45a594d 100644 --- a/tests/posix.nix +++ b/tests/posix.nix @@ -1,6 +1,6 @@ # SPDX-License-Identifier: CC0-1.0 -{ testEnv, pkgs, legacyPackages, ... }: +{ pkgs, legacyPackages, ... }: with pkgs; let @@ -21,14 +21,15 @@ let uname -a ''; }; -in testEnv.mkTest rec { +in rec { name = "posix"; - testConfig = '' - ${ - ./posix.dhall - } { bash = \"${bash}\", coreutils = \"${coreutils}\", script = \"${script}\" }''; - testInputs = map pkgs.genodeSources.depot [ "libc" "posix" "vfs_pipe" "vfs" ] - ++ [ bash ]; - extraPaths = [ script ] ++ (with legacyPackages; [ coreutils hello ]); - meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; + machine = { + config = '' + ${ + ./posix.dhall + } { bash = \"${bash}\", coreutils = \"${coreutils}\", script = \"${script}\" }''; + inputs = map pkgs.genodeSources.depot [ "libc" "posix" "vfs_pipe" "vfs" ] + ++ [ bash ]; + extraPaths = [ script ] ++ (with legacyPackages; [ coreutils hello ]); + }; } diff --git a/tests/solo5/default.nix b/tests/solo5/default.nix index cdcf730..35c3639 100644 --- a/tests/solo5/default.nix +++ b/tests/solo5/default.nix @@ -1,46 +1,44 @@ # SPDX-License-Identifier: CC0-1.0 -{ testEnv, pkgs, ... }: +{ pkgs, ... }: with pkgs; let - mkTest' = { name, testConfig, testInputs ? [ ], ... }@attrs: - testEnv.mkTest (attrs // { + solo5Test = { name, machine, ... }@args: + args // { name = "solo5-" + name; - inherit testConfig; - testInputs = [ pkgs.solo5 pkgs.solo5.tests ] ++ testInputs; - }); - - applyMkTest = x: { - inherit (x) name; - value = mkTest' x; - }; - - mkTests = testList: builtins.listToAttrs (map applyMkTest testList); + machine = machine // { + inputs = [ pkgs.solo5 pkgs.solo5.tests ] ++ machine.inputs; + }; + }; genodeDepot = pkgs.genodeSources.depot; genodeMake = pkgs.genodeSources.make; - tests = [ - { - name = "multi"; - testConfig = "${./.}/solo5.dhall"; - testInputs = map genodeMake [ "app/ping" ] ++ (map genodeDepot [ +in map solo5Test [ + { + name = "multi"; + machine = { + config = "${./.}/solo5.dhall"; + inputs = map genodeMake [ "app/ping" ] ++ (map genodeDepot [ "ram_block" "nic_bridge" "nic_loopback" "sequence" ]); - } + }; + } - { - name = "ssp"; - testConfig = ./ssp.dhall; - testScript = '' - run_genode_until {Error: stack protector check failed} 30 - ''; - } + { + name = "ssp"; + machine = { + config = ./ssp.dhall; + inputs = [ ]; + }; + testScript = '' + start_all() + machine.wait_until_serial_output("Error: stack protector check failed") + ''; + } - ]; - -in mkTests tests +] diff --git a/tests/test-driver/test-driver.py b/tests/test-driver/test-driver.py new file mode 100644 index 0000000..45137fc --- /dev/null +++ b/tests/test-driver/test-driver.py @@ -0,0 +1,912 @@ +#! /somewhere/python3 +# Copyright (c) 2003-2020 Nixpkgs/NixOS contributors + +from contextlib import contextmanager, _GeneratorContextManager +from queue import Queue, Empty +from typing import Tuple, Any, Callable, Dict, Iterator, Optional, List +from xml.sax.saxutils import XMLGenerator +import _thread +import atexit +import base64 +import codecs +import os +import pathlib +import ptpython.repl +import pty +import re +import shlex +import shutil +import socket +import subprocess +import sys +import tempfile +import time +import unicodedata + +CHAR_TO_KEY = { + "A": "shift-a", + "N": "shift-n", + "-": "0x0C", + "_": "shift-0x0C", + "B": "shift-b", + "O": "shift-o", + "=": "0x0D", + "+": "shift-0x0D", + "C": "shift-c", + "P": "shift-p", + "[": "0x1A", + "{": "shift-0x1A", + "D": "shift-d", + "Q": "shift-q", + "]": "0x1B", + "}": "shift-0x1B", + "E": "shift-e", + "R": "shift-r", + ";": "0x27", + ":": "shift-0x27", + "F": "shift-f", + "S": "shift-s", + "'": "0x28", + '"': "shift-0x28", + "G": "shift-g", + "T": "shift-t", + "`": "0x29", + "~": "shift-0x29", + "H": "shift-h", + "U": "shift-u", + "\\": "0x2B", + "|": "shift-0x2B", + "I": "shift-i", + "V": "shift-v", + ",": "0x33", + "<": "shift-0x33", + "J": "shift-j", + "W": "shift-w", + ".": "0x34", + ">": "shift-0x34", + "K": "shift-k", + "X": "shift-x", + "/": "0x35", + "?": "shift-0x35", + "L": "shift-l", + "Y": "shift-y", + " ": "spc", + "M": "shift-m", + "Z": "shift-z", + "\n": "ret", + "!": "shift-0x02", + "@": "shift-0x03", + "#": "shift-0x04", + "$": "shift-0x05", + "%": "shift-0x06", + "^": "shift-0x07", + "&": "shift-0x08", + "*": "shift-0x09", + "(": "shift-0x0A", + ")": "shift-0x0B", +} + +# Forward references +log: "Logger" +machines: "List[Machine]" + + +def eprint(*args: object, **kwargs: Any) -> None: + print(*args, file=sys.stderr, **kwargs) + + +def make_command(args: list) -> str: + return " ".join(map(shlex.quote, (map(str, args)))) + + +def retry(fn: Callable) -> None: + """Call the given function repeatedly, with 1 second intervals, + until it returns True or a timeout is reached. + """ + + for _ in range(900): + if fn(False): + return + time.sleep(1) + + if not fn(True): + raise Exception("action timed out") + + +class Logger: + def __init__(self) -> None: + self.logfile = os.environ.get("LOGFILE", "/dev/null") + self.logfile_handle = codecs.open(self.logfile, "wb") + self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8") + self.queue: "Queue[Dict[str, str]]" = Queue() + + self.xml.startDocument() + self.xml.startElement("logfile", attrs={}) + + def close(self) -> None: + self.xml.endElement("logfile") + self.xml.endDocument() + self.logfile_handle.close() + + def sanitise(self, message: str) -> str: + return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C") + + def maybe_prefix(self, message: str, attributes: Dict[str, str]) -> str: + if "machine" in attributes: + return "{}: {}".format(attributes["machine"], message) + return message + + def log_line(self, message: str, attributes: Dict[str, str]) -> None: + self.xml.startElement("line", attributes) + self.xml.characters(message) + self.xml.endElement("line") + + def log(self, message: str, attributes: Dict[str, str] = {}) -> None: + eprint(self.maybe_prefix(message, attributes)) + self.drain_log_queue() + self.log_line(message, attributes) + + def enqueue(self, message: Dict[str, str]) -> None: + self.queue.put(message) + + def drain_log_queue(self) -> None: + try: + while True: + item = self.queue.get_nowait() + attributes = {"machine": item["machine"], "type": "serial"} + self.log_line(self.sanitise(item["msg"]), attributes) + except Empty: + pass + + @contextmanager + def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]: + eprint(self.maybe_prefix(message, attributes)) + + self.xml.startElement("nest", attrs={}) + self.xml.startElement("head", attributes) + self.xml.characters(message) + self.xml.endElement("head") + + tic = time.time() + self.drain_log_queue() + yield + self.drain_log_queue() + toc = time.time() + self.log("({:.2f} seconds)".format(toc - tic)) + + self.xml.endElement("nest") + + +class Machine: + def __init__(self, args: Dict[str, Any]) -> None: + if "name" in args: + self.name = args["name"] + else: + self.name = "machine" + cmd = args.get("startCommand", None) + if cmd: + match = re.search("bin/run-(.+)-vm$", cmd) + if match: + self.name = match.group(1) + + self.script = args.get("startCommand", self.create_startcommand(args)) + + tmp_dir = os.environ.get("TMPDIR", tempfile.gettempdir()) + + def create_dir(name: str) -> str: + path = os.path.join(tmp_dir, name) + os.makedirs(path, mode=0o700, exist_ok=True) + return path + + self.state_dir = create_dir("vm-state-{}".format(self.name)) + self.shared_dir = create_dir("shared-xchg") + + self.booted = False + self.connected = False + self.pid: Optional[int] = None + self.socket = None + self.monitor: Optional[socket.socket] = None + self.logger: Logger = args["log"] + self.serialQueue: "Queue[str]" = Queue() + + self.allow_reboot = args.get("allowReboot", False) + + @staticmethod + def create_startcommand(args: Dict[str, str]) -> str: + net_backend = "-netdev user,id=net0" + net_frontend = "-device virtio-net-pci,netdev=net0" + + start_command = "qemu-system-x86_64 -m 384 $QEMU_OPTS " + + if "hda" in args: + hda_path = os.path.abspath(args["hda"]) + if args.get("hdaInterface", "") == "scsi": + start_command += ( + "-drive id=hda,file=" + + hda_path + + ",werror=report,if=none " + + "-device scsi-hd,drive=hda " + ) + else: + start_command += ( + "-drive file=" + + hda_path + + ",if=" + + args["hdaInterface"] + + ",werror=report " + ) + + if "cdrom" in args: + start_command += "-cdrom " + args["cdrom"] + " " + + if "usb" in args: + start_command += ( + "-device piix3-usb-uhci -drive " + + "id=usbdisk,file=" + + args["usb"] + + ",if=none,readonly " + + "-device usb-storage,drive=usbdisk " + ) + if "bios" in args: + start_command += "-bios " + args["bios"] + " " + + start_command += args.get("qemuFlags", "") + + return start_command + + def is_up(self) -> bool: + return self.booted and self.connected + + def log(self, msg: str) -> None: + self.logger.log(msg, {"machine": self.name}) + + def nested(self, msg: str, attrs: Dict[str, str] = {}) -> _GeneratorContextManager: + my_attrs = {"machine": self.name} + my_attrs.update(attrs) + return self.logger.nested(msg, my_attrs) + + def wait_for_monitor_prompt(self) -> str: + assert self.monitor is not None + answer = "" + while True: + undecoded_answer = self.monitor.recv(1024) + if not undecoded_answer: + break + answer += undecoded_answer.decode() + if answer.endswith("(qemu) "): + break + return answer + + def send_monitor_command(self, command: str) -> str: + message = ("{}\n".format(command)).encode() + self.log("sending monitor command: {}".format(command)) + assert self.monitor is not None + self.monitor.send(message) + return self.wait_for_monitor_prompt() + + def wait_for_unit(self, unit: str, user: Optional[str] = None) -> None: + """Wait for a systemd unit to get into "active" state. + Throws exceptions on "failed" and "inactive" states as well as + after timing out. + """ + + def check_active(_: Any) -> bool: + info = self.get_unit_info(unit, user) + state = info["ActiveState"] + if state == "failed": + raise Exception('unit "{}" reached state "{}"'.format(unit, state)) + + if state == "inactive": + status, jobs = self.systemctl("list-jobs --full 2>&1", user) + if "No jobs" in jobs: + info = self.get_unit_info(unit, user) + if info["ActiveState"] == state: + raise Exception( + ( + 'unit "{}" is inactive and there ' "are no pending jobs" + ).format(unit) + ) + + return state == "active" + + retry(check_active) + + def get_unit_info(self, unit: str, user: Optional[str] = None) -> Dict[str, str]: + status, lines = self.systemctl('--no-pager show "{}"'.format(unit), user) + if status != 0: + raise Exception( + 'retrieving systemctl info for unit "{}" {} failed with exit code {}'.format( + unit, "" if user is None else 'under user "{}"'.format(user), status + ) + ) + + line_pattern = re.compile(r"^([^=]+)=(.*)$") + + def tuple_from_line(line: str) -> Tuple[str, str]: + match = line_pattern.match(line) + assert match is not None + return match[1], match[2] + + return dict( + tuple_from_line(line) + for line in lines.split("\n") + if line_pattern.match(line) + ) + + def systemctl(self, q: str, user: Optional[str] = None) -> Tuple[int, str]: + if user is not None: + q = q.replace("'", "\\'") + return self.execute( + ( + "su -l {} --shell /bin/sh -c " + "$'XDG_RUNTIME_DIR=/run/user/`id -u` " + "systemctl --user {}'" + ).format(user, q) + ) + return self.execute("systemctl {}".format(q)) + + def require_unit_state(self, unit: str, require_state: str = "active") -> None: + with self.nested( + "checking if unit ‘{}’ has reached state '{}'".format(unit, require_state) + ): + info = self.get_unit_info(unit) + state = info["ActiveState"] + if state != require_state: + raise Exception( + "Expected unit ‘{}’ to to be in state ".format(unit) + + "'{}' but it is in state ‘{}’".format(require_state, state) + ) + + def execute(self, command: str) -> Tuple[int, str]: + self.connect() + + out_command = "( {} ); echo '|!=EOF' $?\n".format(command) + self.shell.send(out_command.encode()) + + output = "" + status_code_pattern = re.compile(r"(.*)\|\!=EOF\s+(\d+)") + + while True: + chunk = self.shell.recv(4096).decode(errors="ignore") + match = status_code_pattern.match(chunk) + if match: + output += match[1] + status_code = int(match[2]) + return (status_code, output) + output += chunk + + def succeed(self, *commands: str) -> str: + """Execute each command and check that it succeeds.""" + output = "" + for command in commands: + with self.nested("must succeed: {}".format(command)): + (status, out) = self.execute(command) + if status != 0: + self.log("output: {}".format(out)) + raise Exception( + "command `{}` failed (exit code {})".format(command, status) + ) + output += out + return output + + def fail(self, *commands: str) -> None: + """Execute each command and check that it fails.""" + for command in commands: + with self.nested("must fail: {}".format(command)): + status, output = self.execute(command) + if status == 0: + raise Exception( + "command `{}` unexpectedly succeeded".format(command) + ) + + def wait_until_succeeds(self, command: str) -> str: + """Wait until a command returns success and return its output. + Throws an exception on timeout. + """ + output = "" + + def check_success(_: Any) -> bool: + nonlocal output + status, output = self.execute(command) + return status == 0 + + with self.nested("waiting for success: {}".format(command)): + retry(check_success) + return output + + def wait_until_fails(self, command: str) -> str: + """Wait until a command returns failure. + Throws an exception on timeout. + """ + output = "" + + def check_failure(_: Any) -> bool: + nonlocal output + status, output = self.execute(command) + return status != 0 + + with self.nested("waiting for failure: {}".format(command)): + retry(check_failure) + return output + + def wait_for_shutdown(self) -> None: + if not self.booted: + return + + with self.nested("waiting for the VM to power off"): + sys.stdout.flush() + self.process.wait() + + self.pid = None + self.booted = False + self.connected = False + + def get_tty_text(self, tty: str) -> str: + status, output = self.execute( + "fold -w$(stty -F /dev/tty{0} size | " + "awk '{{print $2}}') /dev/vcs{0}".format(tty) + ) + return output + + def wait_until_tty_matches(self, tty: str, regexp: str) -> None: + """Wait until the visible output on the chosen TTY matches regular + expression. Throws an exception on timeout. + """ + matcher = re.compile(regexp) + + def tty_matches(last: bool) -> bool: + text = self.get_tty_text(tty) + if last: + self.log( + f"Last chance to match /{regexp}/ on TTY{tty}, " + f"which currently contains: {text}" + ) + return len(matcher.findall(text)) > 0 + + with self.nested("waiting for {} to appear on tty {}".format(regexp, tty)): + retry(tty_matches) + + def wait_until_serial_output(self, regexp: str) -> None: + """Wait until the serial output matches regular expression. + Throws an exception on timeout. + """ + matcher = re.compile(regexp) + + def serial_matches(last: bool) -> bool: + while not self.serialQueue.empty(): + text = self.serialQueue.get() + if last: + self.log( + f"Last chance to match /{regexp}/ on serial, " + f"which currently contains: {text}" + ) + if len(matcher.findall(text)) > 0: + return True + return False + + with self.nested("waiting for {} to appear on serial output".format(regexp)): + retry(serial_matches) + + def send_chars(self, chars: List[str]) -> None: + with self.nested("sending keys ‘{}‘".format(chars)): + for char in chars: + self.send_key(char) + + def wait_for_file(self, filename: str) -> None: + """Waits until the file exists in machine's file system.""" + + def check_file(_: Any) -> bool: + status, _ = self.execute("test -e {}".format(filename)) + return status == 0 + + with self.nested("waiting for file ‘{}‘".format(filename)): + retry(check_file) + + def wait_for_open_port(self, port: int) -> None: + def port_is_open(_: Any) -> bool: + status, _ = self.execute("nc -z localhost {}".format(port)) + return status == 0 + + with self.nested("waiting for TCP port {}".format(port)): + retry(port_is_open) + + def wait_for_closed_port(self, port: int) -> None: + def port_is_closed(_: Any) -> bool: + status, _ = self.execute("nc -z localhost {}".format(port)) + return status != 0 + + retry(port_is_closed) + + def start_job(self, jobname: str, user: Optional[str] = None) -> Tuple[int, str]: + return self.systemctl("start {}".format(jobname), user) + + def stop_job(self, jobname: str, user: Optional[str] = None) -> Tuple[int, str]: + return self.systemctl("stop {}".format(jobname), user) + + def wait_for_job(self, jobname: str) -> None: + self.wait_for_unit(jobname) + + def connect(self) -> None: + if self.connected: + return + + with self.nested("waiting for the VM to finish booting"): + self.start() + + tic = time.time() + self.shell.recv(1024) + # TODO: Timeout + toc = time.time() + + self.log("connected to guest root shell") + self.log("(connecting took {:.2f} seconds)".format(toc - tic)) + self.connected = True + + def screenshot(self, filename: str) -> None: + out_dir = os.environ.get("out", os.getcwd()) + word_pattern = re.compile(r"^\w+$") + if word_pattern.match(filename): + filename = os.path.join(out_dir, "{}.png".format(filename)) + tmp = "{}.ppm".format(filename) + + with self.nested( + "making screenshot {}".format(filename), + {"image": os.path.basename(filename)}, + ): + self.send_monitor_command("screendump {}".format(tmp)) + ret = subprocess.run("pnmtopng {} > {}".format(tmp, filename), shell=True) + os.unlink(tmp) + if ret.returncode != 0: + raise Exception("Cannot convert screenshot") + + def copy_from_host_via_shell(self, source: str, target: str) -> None: + """Copy a file from the host into the guest by piping it over the + shell into the destination file. Works without host-guest shared folder. + Prefer copy_from_host for whenever possible. + """ + with open(source, "rb") as fh: + content_b64 = base64.b64encode(fh.read()).decode() + self.succeed( + f"mkdir -p $(dirname {target})", + f"echo -n {content_b64} | base64 -d > {target}", + ) + + def copy_from_host(self, source: str, target: str) -> None: + """Copy a file from the host into the guest via the `shared_dir` shared + among all the VMs (using a temporary directory). + """ + host_src = pathlib.Path(source) + vm_target = pathlib.Path(target) + with tempfile.TemporaryDirectory(dir=self.shared_dir) as shared_td: + shared_temp = pathlib.Path(shared_td) + host_intermediate = shared_temp / host_src.name + vm_shared_temp = pathlib.Path("/tmp/shared") / shared_temp.name + vm_intermediate = vm_shared_temp / host_src.name + + self.succeed(make_command(["mkdir", "-p", vm_shared_temp])) + if host_src.is_dir(): + shutil.copytree(host_src, host_intermediate) + else: + shutil.copy(host_src, host_intermediate) + self.succeed("sync") + self.succeed(make_command(["mkdir", "-p", vm_target.parent])) + self.succeed(make_command(["cp", "-r", vm_intermediate, vm_target])) + # Make sure the cleanup is synced into VM + self.succeed("sync") + + def copy_from_vm(self, source: str, target_dir: str = "") -> None: + """Copy a file from the VM (specified by an in-VM source path) to a path + relative to `$out`. The file is copied via the `shared_dir` shared among + all the VMs (using a temporary directory). + """ + # Compute the source, target, and intermediate shared file names + out_dir = pathlib.Path(os.environ.get("out", os.getcwd())) + vm_src = pathlib.Path(source) + with tempfile.TemporaryDirectory(dir=self.shared_dir) as shared_td: + shared_temp = pathlib.Path(shared_td) + vm_shared_temp = pathlib.Path("/tmp/shared") / shared_temp.name + vm_intermediate = vm_shared_temp / vm_src.name + intermediate = shared_temp / vm_src.name + # Copy the file to the shared directory inside VM + self.succeed(make_command(["mkdir", "-p", vm_shared_temp])) + self.succeed(make_command(["cp", "-r", vm_src, vm_intermediate])) + self.succeed("sync") + abs_target = out_dir / target_dir / vm_src.name + abs_target.parent.mkdir(exist_ok=True, parents=True) + # Copy the file from the shared directory outside VM + if intermediate.is_dir(): + shutil.copytree(intermediate, abs_target) + else: + shutil.copy(intermediate, abs_target) + # Make sure the cleanup is synced into VM + self.succeed("sync") + + def dump_tty_contents(self, tty: str) -> None: + """Debugging: Dump the contents of the TTY + """ + self.execute("fold -w 80 /dev/vcs{} | systemd-cat".format(tty)) + + def get_screen_text(self) -> str: + if shutil.which("tesseract") is None: + raise Exception("get_screen_text used but enableOCR is false") + + magick_args = ( + "-filter Catrom -density 72 -resample 300 " + + "-contrast -normalize -despeckle -type grayscale " + + "-sharpen 1 -posterize 3 -negate -gamma 100 " + + "-blur 1x65535" + ) + + tess_args = "-c debug_file=/dev/null --psm 11 --oem 2" + + with self.nested("performing optical character recognition"): + with tempfile.NamedTemporaryFile() as tmpin: + self.send_monitor_command("screendump {}".format(tmpin.name)) + + cmd = "convert {} {} tiff:- | tesseract - - {}".format( + magick_args, tmpin.name, tess_args + ) + ret = subprocess.run(cmd, shell=True, capture_output=True) + if ret.returncode != 0: + raise Exception( + "OCR failed with exit code {}".format(ret.returncode) + ) + + return ret.stdout.decode("utf-8") + + def wait_for_text(self, regex: str) -> None: + def screen_matches(last: bool) -> bool: + text = self.get_screen_text() + matches = re.search(regex, text) is not None + + if last and not matches: + self.log("Last OCR attempt failed. Text was: {}".format(text)) + + return matches + + with self.nested("waiting for {} to appear on screen".format(regex)): + retry(screen_matches) + + def send_key(self, key: str) -> None: + key = CHAR_TO_KEY.get(key, key) + self.send_monitor_command("sendkey {}".format(key)) + + def start(self) -> None: + if self.booted: + return + + self.log("starting vm") + + def create_socket(path: str) -> socket.socket: + if os.path.exists(path): + os.unlink(path) + s = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM) + s.bind(path) + s.listen(1) + return s + + monitor_path = os.path.join(self.state_dir, "monitor") + self.monitor_socket = create_socket(monitor_path) + + shell_path = os.path.join(self.state_dir, "shell") + self.shell_socket = create_socket(shell_path) + + qemu_options = ( + " ".join( + [ + "" if self.allow_reboot else "-no-reboot", + "-monitor unix:{}".format(monitor_path), + "-chardev socket,id=shell,path={}".format(shell_path), + "-device virtio-serial", + "-device virtconsole,chardev=shell", + "-device virtio-rng-pci", + "-serial stdio" if "DISPLAY" in os.environ else "-nographic", + ] + ) + + " " + + os.environ.get("QEMU_OPTS", "") + ) + + environment = dict(os.environ) + environment.update( + { + "TMPDIR": self.state_dir, + "SHARED_DIR": self.shared_dir, + "USE_TMPDIR": "1", + "QEMU_OPTS": qemu_options, + } + ) + + self.process = subprocess.Popen( + self.script, + bufsize=1, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + shell=True, + cwd=self.state_dir, + env=environment, + ) + self.monitor, _ = self.monitor_socket.accept() + self.shell, _ = self.shell_socket.accept() + + def process_serial_output() -> None: + assert self.process.stdout is not None + for _line in self.process.stdout: + # Ignore undecodable bytes that may occur in boot menus + line = _line.decode(errors="ignore").replace("\r", "").rstrip() + eprint("{} # {}".format(self.name, line)) + self.logger.enqueue({"msg": line, "machine": self.name}) + self.serialQueue.put(line) + + _thread.start_new_thread(process_serial_output, ()) + + self.wait_for_monitor_prompt() + + self.pid = self.process.pid + self.booted = True + + self.log("QEMU running (pid {})".format(self.pid)) + + def shutdown(self) -> None: + if not self.booted: + return + + self.shell.send("poweroff\n".encode()) + self.wait_for_shutdown() + + def crash(self) -> None: + if not self.booted: + return + + self.log("forced crash") + self.send_monitor_command("quit") + self.wait_for_shutdown() + + def wait_for_x(self) -> None: + """Wait until it is possible to connect to the X server. Note that + testing the existence of /tmp/.X11-unix/X0 is insufficient. + """ + + def check_x(_: Any) -> bool: + cmd = ( + "journalctl -b SYSLOG_IDENTIFIER=systemd | " + + 'grep "Reached target Current graphical"' + ) + status, _ = self.execute(cmd) + if status != 0: + return False + status, _ = self.execute("[ -e /tmp/.X11-unix/X0 ]") + return status == 0 + + with self.nested("waiting for the X11 server"): + retry(check_x) + + def get_window_names(self) -> List[str]: + return self.succeed( + r"xwininfo -root -tree | sed 's/.*0x[0-9a-f]* \"\([^\"]*\)\".*/\1/; t; d'" + ).splitlines() + + def wait_for_window(self, regexp: str) -> None: + pattern = re.compile(regexp) + + def window_is_visible(last_try: bool) -> bool: + names = self.get_window_names() + if last_try: + self.log( + "Last chance to match {} on the window list,".format(regexp) + + " which currently contains: " + + ", ".join(names) + ) + return any(pattern.search(name) for name in names) + + with self.nested("Waiting for a window to appear"): + retry(window_is_visible) + + def sleep(self, secs: int) -> None: + time.sleep(secs) + + def forward_port(self, host_port: int = 8080, guest_port: int = 80) -> None: + """Forward a TCP port on the host to a TCP port on the guest. + Useful during interactive testing. + """ + self.send_monitor_command( + "hostfwd_add tcp::{}-:{}".format(host_port, guest_port) + ) + + def block(self) -> None: + """Make the machine unreachable by shutting down eth1 (the multicast + interface used to talk to the other VMs). We keep eth0 up so that + the test driver can continue to talk to the machine. + """ + self.send_monitor_command("set_link virtio-net-pci.1 off") + + def unblock(self) -> None: + """Make the machine reachable. + """ + self.send_monitor_command("set_link virtio-net-pci.1 on") + + +def create_machine(args: Dict[str, Any]) -> Machine: + global log + args["log"] = log + args["redirectSerial"] = os.environ.get("USE_SERIAL", "0") == "1" + return Machine(args) + + +def start_all() -> None: + global machines + with log.nested("starting all VMs"): + for machine in machines: + machine.start() + + +def join_all() -> None: + global machines + with log.nested("waiting for all VMs to finish"): + for machine in machines: + machine.wait_for_shutdown() + + +def test_script() -> None: + exec(os.environ["testScript"]) + + +def run_tests() -> None: + global machines + tests = os.environ.get("tests", None) + if tests is not None: + with log.nested("running the VM test script"): + try: + exec(tests, globals()) + except Exception as e: + eprint("error: {}".format(str(e))) + sys.exit(1) + else: + ptpython.repl.embed(locals(), globals()) + + # TODO: Collect coverage data + + for machine in machines: + if machine.is_up(): + machine.execute("sync") + + +@contextmanager +def subtest(name: str) -> Iterator[None]: + with log.nested(name): + try: + yield + return True + except Exception as e: + log.log(f'Test "{name}" failed with error: "{e}"') + raise e + + return False + + +if __name__ == "__main__": + log = Logger() + + vm_scripts = sys.argv[1:] + machines = [create_machine({"startCommand": s}) for s in vm_scripts] + machine_eval = [ + "{0} = machines[{1}]".format(m.name, idx) for idx, m in enumerate(machines) + ] + exec("\n".join(machine_eval)) + + @atexit.register + def clean_up() -> None: + with log.nested("cleaning up"): + for machine in machines: + if machine.pid is None: + continue + log.log("killing {} (pid {})".format(machine.name, machine.pid)) + machine.process.kill() + + log.close() + + tic = time.time() + run_tests() + toc = time.time() + print("test script finished in {:.2f}s".format(toc - tic)) diff --git a/tests/vmm_x86.nix b/tests/vmm_x86.nix index e92b392..e6f3ffd 100644 --- a/tests/vmm_x86.nix +++ b/tests/vmm_x86.nix @@ -1,12 +1,14 @@ # SPDX-License-Identifier: CC0-1.0 -{ testEnv, pkgs, ... }: -with pkgs; +{ pkgs, ... }: -testEnv.mkTest { +{ name = "vmm"; - meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; - - testConfig = ./vmm_x86.dhall; - testInputs = map pkgs.genodeSources.make [ "test/vmm_x86" ]; + constraints = specs: + with builtins; + all (f: any f specs) [ (spec: spec == "nova") (spec: spec == "x86") ]; + machine = { + config = ./vmm_x86.dhall; + inputs = map pkgs.genodeSources.make [ "test/vmm_x86" ]; + }; } diff --git a/tests/x86.nix b/tests/x86.nix index aa2662e..1c4501b 100644 --- a/tests/x86.nix +++ b/tests/x86.nix @@ -1,17 +1,15 @@ # SPDX-License-Identifier: CC0-1.0 -{ testEnv, pkgs, ... }: -with pkgs; - -testEnv.mkTest { +{ pkgs, ... }: { name = "x86"; - meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ]; - - testConfig = ./x86.dhall; - testInputs = (map pkgs.genodeSources.depot [ - "acpi_drv" - "platform_drv" - "report_rom" - "test-signal" - ]) ++ (map pkgs.genodeSources.make [ "test/pci" "test/rtc" ]); + constraints = builtins.any (spec: spec == "x86"); + machine = { + config = ./x86.dhall; + inputs = (map pkgs.genodeSources.depot [ + "acpi_drv" + "platform_drv" + "report_rom" + "test-signal" + ]) ++ (map pkgs.genodeSources.make [ "test/pci" "test/rtc" ]); + }; }