Browse Source

Replace test framework

Adopt the Python test driver from NixOS. Temporarily drop Sotest
runs.
ncurses
Emery Hemingway 1 year ago
parent
commit
40c1977779
  1. 10
      README.md
  2. 2
      flake.nix
  3. 6
      lib/default.nix
  4. 370
      tests/default.nix
  5. 173
      tests/driver-hw.nix
  6. 179
      tests/driver-nova.nix
  7. 37
      tests/driver_manager.nix
  8. 17
      tests/log.nix
  9. 21
      tests/posix.nix
  10. 60
      tests/solo5/default.nix
  11. 912
      tests/test-driver/test-driver.py
  12. 16
      tests/vmm_x86.nix
  13. 24
      tests/x86.nix

10
README.md

@ -135,14 +135,14 @@ Tests are performed using QEMU, the test artifacts are built as follows:
# Build a test log from a QEMU test run:
nix build .#checks.x86_64-linux.nova-x86
# Build an ISO of the test run:
nix build .#checks.x86_64-linux.nova-x86.iso
# Build a tarball of the Nix store internal to a test VM:
nix build .#checks.x86_64-linux.nova-x86.nodes.machine.store
# Build a tarball of the Nix store internal to a test:
nix build .#checks.x86_64-linux.nova-x86.store
# Build the XML configuration of the test VM:
nix build .#checks.x86_64-linux.nova-x86.nodes.machine.xml
# Build the Dhall boot description of the test run:
nix build .#checks.x86_64-linux.nova-x86.config
nix build .#checks.x86_64-linux.nova-x86.nodes.machine.config
```
# System description format

2
flake.nix

@ -1,6 +1,8 @@
# SPDX-License-Identifier: CC0-1.0
{
edition = 201909;
description = "Genode packages";
inputs.nixpkgs.url = "github:ehmry/nixpkgs?ref=genode";

6
lib/default.nix

@ -31,7 +31,7 @@ in rec {
'';
compileBoot = name: env: bootDhall:
runDhallCommand name env ''
runDhallCommand "${name}-boot" env ''
dhall to-directory-tree --output $out \
<<< "${./compile-boot.dhall} (${bootDhall}) \"$out\""
dhall <<< "(${bootDhall}).config" \
@ -41,7 +41,7 @@ in rec {
hwImage = name: env: boot:
nixpkgs.stdenv.mkDerivation {
name = name + ".image.elf";
name = name + "-hw-image";
build = compileBoot name env boot;
nativeBuildInputs = [ buildPackages.dhall ];
buildCommand = let
@ -99,7 +99,7 @@ in rec {
novaImage = name: env: boot:
nixpkgs.stdenv.mkDerivation {
name = name + ".image.elf";
name = name + "-nova-image";
build = compileBoot name env boot;
buildCommand = ''

370
tests/default.nix

@ -1,89 +1,299 @@
# SPDX-License-Identifier: CC0-1.0
let tests = call: { log = call ./log.nix { }; };
in { self, apps, buildPackages, genodepkgs, lib, nixpkgs, legacyPackages }:
{ self, apps, buildPackages, genodepkgs, lib, nixpkgs, legacyPackages }:
let
callTest = path:
import path {
pkgs = testPkgs;
inherit nixpkgs buildPackages legacyPackages;
};
testFiles = map callTest [
./log.nix
./posix.nix
./vmm_x86.nix
./x86.nix
] ++ (callTest ./solo5);
testPkgs = genodepkgs;
addManifest = drv:
drv // {
manifest = nixpkgs.runCommand "${drv.name}.dhall" { inherit drv; } ''
set -eu
echo -n '[' >> $out
find $drv/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out
${if builtins.elem "lib" drv.outputs then
''
find ${drv.lib}/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out''
else
""}
echo -n ']' >> $out
qemu' = buildPackages.qemu_test;
qemuBinary = qemuPkg:
{
aarch64-genode = "${qemuPkg}/bin/qemu-system-aarch64";
x86_64-genode = "${qemuPkg}/bin/qemu-system-x86_64";
}.${genodepkgs.stdenv.hostPlatform.system};
platforms = [
{
prefix = "hw-pc-";
specs = [ "x86" "hw" ];
basePkg = testPkgs.base-hw-pc;
makeImage = lib.hwImage;
startVM = vmName: image: ''
#! ${buildPackages.runtimeShell}
exec ${qemuBinary qemu'} \
-name ${vmName} \
-machine q35 \
-m 384 \
-kernel "${testPkgs.bender}/bender" \
-initrd "${image}/image.elf" \
$QEMU_OPTS \
"$@"
'';
}
{
prefix = "nova-";
specs = [ "x86" "nova" ];
basePkg = testPkgs.base-nova;
makeImage = lib.novaImage;
startVM = vmName: image: ''
#! ${buildPackages.runtimeShell}
exec ${qemuBinary qemu'} \
-name ${vmName} \
-machine q35 \
-m 384 \
-kernel "${testPkgs.bender}/bender" \
-initrd "${testPkgs.NOVA}/hypervisor-x86_64 arg=iommu novpid serial,${image}/image.elf" \
$QEMU_OPTS \
"$@"
'';
}
];
testDriver = with buildPackages;
let testDriverScript = ./test-driver/test-driver.py;
in stdenv.mkDerivation {
name = "nixos-test-driver";
nativeBuildInputs = [ makeWrapper ];
buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ];
checkInputs = with python3Packages; [ pylint mypy ];
dontUnpack = true;
preferLocalBuild = true;
doCheck = true;
checkPhase = ''
mypy --disallow-untyped-defs \
--no-implicit-optional \
--ignore-missing-imports ${testDriverScript}
pylint --errors-only ${testDriverScript}
'';
installPhase = ''
mkdir -p $out/bin
cp ${testDriverScript} $out/bin/nixos-test-driver
chmod u+x $out/bin/nixos-test-driver
# TODO: copy user script part into this file (append)
wrapProgram $out/bin/nixos-test-driver \
--prefix PATH : "${lib.makeBinPath [ qemu' coreutils ]}" \
'';
};
nova = (call:
((tests call) // {
driver_manager = call ./driver_manager.nix { };
posix = call ./posix.nix { };
vmm = call ./vmm_x86.nix { };
x86 = call ./x86.nix { };
} // call ./solo5 { })) (import ./driver-nova.nix {
inherit apps addManifest buildPackages lib nixpkgs testPkgs
legacyPackages;
}).callTest;
hw = (call:
((tests call) // {
posix = call ./posix.nix { };
x86 = call ./x86.nix { };
} // call ./solo5 { })) (import ./driver-hw.nix {
inherit apps addManifest buildPackages lib nixpkgs testPkgs
legacyPackages;
}).callTest;
testsToList = tests:
map (test: {
inherit (test) name;
value = test;
}) (builtins.attrValues tests);
in with builtins;
listToAttrs ((concatLists (map (testsToList) [ hw nova ]))) // {
sotest = let
hwTests = with hw; [ multi posix x86 ];
novaTests = with nova; [ multi posix x86 vmm ];
allTests = hwTests ++ novaTests;
projectCfg.boot_items =
(map (test: {
inherit (test) name;
exec = "bender";
load = [ test.image.name ];
}) hwTests)
++ (map (test: {
inherit (test) name;
exec = "bender";
load = [ "hypervisor serial novga iommu" test.image.name ];
}) novaTests);
in buildPackages.stdenv.mkDerivation {
name = "sotest";
buildCommand = ''
mkdir zip; cd zip
cp "${testPkgs.bender}/bender" bender
cp "${testPkgs.NOVA}/hypervisor-x86_64" hypervisor
${concatStringsSep "\n"
(map (test: "cp ${test.image}/image.elf ${test.image.name}") allTests)}
mkdir -p $out/nix-support
${buildPackages.zip}/bin/zip "$out/binaries.zip" *
cat << EOF > "$out/project.json"
${builtins.toJSON projectCfg}
EOF
echo file sotest-binaries $out/binaries.zip >> "$out/nix-support/hydra-build-products"
echo file sotest-config $out/project.json >> "$out/nix-support/hydra-build-products"
'';
};
}
defaultTestScript = ''
start_all()
machine.wait_until_serial_output('child "init" exited with exit value 0')
'';
makeTest = with buildPackages;
{ prefix, specs, basePkg, makeImage, startVM }:
{ name ? "unnamed", testScript ? defaultTestScript,
# Skip linting (mainly intended for faster dev cycles)
skipLint ? false, ... }@t:
let
testDriverName = "genode-test-driver-${name}";
buildVM = vmName:
{ config, inputs, env ? { }, extraPaths ? [ ] }:
let
storeTarball = buildPackages.runCommand "store" { } ''
mkdir -p $out
tar cf "$out/store.tar" --absolute-names ${toString inputs} ${
toString extraPaths
}
'';
addManifest = drv:
drv // {
manifest =
nixpkgs.runCommand "${drv.name}.dhall" { inherit drv; } ''
set -eu
echo -n '[' >> $out
find $drv/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out
${if builtins.elem "lib" drv.outputs then
''
find ${drv.lib}/ -type f -printf ',{mapKey= "%f",mapValue="%p"}' >> $out''
else
""}
echo -n ']' >> $out
'';
};
storeManifest = lib.mergeManifests (map addManifest inputs);
manifest = lib.mergeManifests (map addManifest
([ basePkg testPkgs.sotest-producer storeTarball ]
++ map testPkgs.genodeSources.depot [
"init"
"rtc_drv"
"vfs"
"cached_fs_rom"
]));
config' = "${
./test-wrapper.dhall
} (${config}) $(stat --format '%s' ${storeTarball}/store.tar) ${storeManifest} ${manifest}";
env' = {
DHALL_GENODE = "${testPkgs.dhallGenode}/source.dhall";
DHALL_GENODE_TEST = "${./test.dhall}";
} // env;
image = makeImage vmName env' config';
startVM' = startVM vmName image;
in {
script = buildPackages.writeScriptBin "run-${vmName}-vm" startVM';
config = lib.runDhallCommand (name + ".dhall") env' ''
${apps.dhall.program} <<< "${config'}" > $out
'';
store = storeTarball;
xml = lib.runDhallCommand (name + ".config") env'
''${apps.render-init.program} <<< "(${config'}).config" > $out'';
};
nodes = lib.mapAttrs buildVM
(t.nodes or (if t ? machine then { machine = t.machine; } else { }));
testScript' =
# Call the test script with the computed nodes.
if lib.isFunction testScript then
testScript { inherit nodes; }
else
testScript;
vms = map (node: node.script) (lib.attrValues nodes);
# Generate onvenience wrappers for running the test driver
# interactively with the specified network, and for starting the
# VMs from the command line.
driver =
let warn = if skipLint then lib.warn "Linting is disabled!" else lib.id;
in warn (runCommand testDriverName {
buildInputs = [ makeWrapper ];
testScript = testScript';
preferLocalBuild = true;
testName = name;
} ''
mkdir -p $out/bin
echo -n "$testScript" > $out/test-script
${lib.optionalString (!skipLint) ''
${python3Packages.black}/bin/black --check --quiet --diff $out/test-script
''}
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/
vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
wrapProgram $out/bin/nixos-test-driver \
--add-flags "''${vms[*]}" \
--run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\""
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
wrapProgram $out/bin/nixos-run-vms \
--add-flags "''${vms[*]}" \
--set tests 'start_all(); join_all();'
''); # "
passMeta = drv:
drv
// lib.optionalAttrs (t ? meta) { meta = (drv.meta or { }) // t.meta; };
# Run an automated test suite in the given virtual network.
# `driver' is the script that runs the network.
runTests = driver:
stdenv.mkDerivation {
name = "test-run-${driver.testName}";
buildCommand = ''
mkdir -p $out
LOGFILE=/dev/null tests='exec(os.environ["testScript"])' ${driver}/bin/nixos-test-driver
'';
};
test = passMeta (runTests driver);
nodeNames = builtins.attrNames nodes;
invalidNodeNames =
lib.filter (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
nodeNames;
in if lib.length invalidNodeNames > 0 then
throw ''
Cannot create machines out of (${
lib.concatStringsSep ", " invalidNodeNames
})!
All machines are referenced as python variables in the testing framework which will break the
script when special characters are used.
Please stick to alphanumeric chars and underscores as separation.
''
else
test // { inherit nodes driver test; };
testList = let
f = platform:
let makeTest' = makeTest platform;
in test:
if (test.constraints or (_: true)) platform.specs then {
name = platform.prefix + test.name;
value = makeTest' test;
} else
null;
in lib.lists.crossLists f [ platforms testFiles ];
in builtins.listToAttrs (builtins.filter (_: _ != null) testList)
/* sotest = let
hwTests = with hw; [ multi posix x86 ];
novaTests = with nova; [ multi posix x86 vmm ];
allTests = hwTests ++ novaTests;
projectCfg.boot_items =
(map (test: {
inherit (test) name;
exec = "bender";
load = [ "${test.name}.image.elf" ];
}) hwTests)
++ (map (test: {
inherit (test) name;
exec = "bender";
load = [ "hypervisor serial novga iommu" test.image.name ];
}) novaTests);
in buildPackages.stdenv.mkDerivation {
name = "sotest";
buildCommand = ''
mkdir zip; cd zip
cp "${testPkgs.bender}/bender" bender
cp "${testPkgs.NOVA}/hypervisor-x86_64" hypervisor
${concatStringsSep "\n"
(map (test: "cp ${test.image}/image.elf ${test.name}.image.elf")
allTests)}
mkdir -p $out/nix-support
${buildPackages.zip}/bin/zip "$out/binaries.zip" *
cat << EOF > "$out/project.json"
${builtins.toJSON projectCfg}
EOF
echo file sotest-binaries $out/binaries.zip >> "$out/nix-support/hydra-build-products"
echo file sotest-config $out/project.json >> "$out/nix-support/hydra-build-products"
'';
};
*/

173
tests/driver-hw.nix

@ -1,173 +0,0 @@
# SPDX-License-Identifier: CC0-1.0
{ addManifest, apps, buildPackages, lib, nixpkgs, testPkgs, legacyPackages }:
let
testDriver = with buildPackages;
stdenv.mkDerivation {
name = "hw-genode-test-driver";
preferLocalBuild = true;
buildInputs = [ makeWrapper expect ];
dontUnpack = true;
installPhase = ''
install -Dm555 ${./hw-test-driver.exp} $out/bin/genode-test-driver
wrapProgram $out/bin/genode-test-driver \
--prefix PATH : "${lib.makeBinPath [ expect coreutils ]}"
'';
};
runTests = driver:
buildPackages.stdenv.mkDerivation {
name = "hw-" + driver.testName;
preferLocalBuild = true;
buildCommand = ''
mkdir -p $out/nix-support
${driver}/bin/genode-test-driver | tee $out/log
touch $out/nix-support
echo "report testlog $out log" >> $out/nix-support/hydra-build-products
'';
};
defaultScript =
''run_genode_until {child "init" exited with exit value 0} 120'';
mkTest = { name ? "unamed", testScript ? defaultScript, testConfig
, testInputs ? [ ], testEnv ? { }, extraPaths ? [ ], qemuArgs ? [ ], ...
}@t:
let
storeTarball = buildPackages.runCommand "store" { } ''
mkdir -p $out
tar cf "$out/store.tar" --absolute-names ${toString testInputs} ${
toString extraPaths
}
'';
storeManifest = lib.mergeManifests (map addManifest testInputs);
manifest = lib.mergeManifests (map addManifest
([ testPkgs.base-hw-pc testPkgs.sotest-producer storeTarball ]
++ map testPkgs.genodeSources.depot [
"init"
"rtc_drv"
"vfs"
"cached_fs_rom"
]));
testConfig' = "${
./test-wrapper.dhall
} (${testConfig}) $(stat --format '%s' ${storeTarball}/store.tar) ${storeManifest} ${manifest}";
testEnv' = {
DHALL_GENODE = "${testPkgs.dhallGenode}/source.dhall";
DHALL_GENODE_TEST = "${./test.dhall}";
} // testEnv;
image = lib.hwImage ("hw-" + name) testEnv' testConfig';
baseSetup = ''
##
# Wait for a specific output of a already running spawned proce
#
proc wait_for_output { wait_for_re timeout_value running_spawn_id } {
global output
if {$wait_for_re == "forever"} {
set timeout -1
interact {
\003 {
send_user "Expect: 'interact' received 'strg+c' and was cancelled\n";
exit
}
-i $running_spawn_id
}
} else {
set timeout $timeout_value
}
expect {
-i $running_spawn_id -re $wait_for_re { }
eof { puts stderr "Error: Spawned process died unexpectedly"; exit -1 }
timeout { puts stderr "Error: Test execution timed out"; exit -1 }
}
set output $expect_out(buffer)
}
proc run_genode_until {{wait_for_re forever} {timeout_value 0} {running_spawn_id -1}} {
#
# If a running_spawn_id is specified, wait for the expected output
#
if {$running_spawn_id != -1} {
wait_for_output $wait_for_re $timeout_value $running_spawn_id
return
}
global env
global spawn_id
set TEST_MIB [expr (([file size ${image}/image.elf] + $env(TEST_RAM)) >> 20) + 24]
spawn ${buildPackages.qemu_test}/bin/qemu-system-x86_64 \
-machine q35 -serial mon:stdio -nographic \
-m size=$TEST_MIB \
-kernel "${testPkgs.bender}/bender" \
-initrd "${image}/image.elf" \
${toString qemuArgs}
wait_for_output $wait_for_re $timeout_value $spawn_id
}
# TODO: not in TCL
global env
set out $env(out)
'';
driver = with buildPackages;
buildPackages.runCommand "genode-test-driver-${name}" ({
buildInputs = [ makeWrapper expect ];
inherit baseSetup testScript;
preferLocalBuild = true;
testName = name;
} // testEnv') ''
mkdir -p $out/bin
echo "$testScript" > $out/test-script
echo "$baseSetup" > $out/base-setup
source ${image.build}/stats
ln -s ${testDriver}/bin/genode-test-driver $out/bin/
wrapProgram $out/bin/genode-test-driver \
--set testScript "$testScript" \
--set baseSetup "$baseSetup" \
--set TEST_RAM $RAM \
'';
passMeta = drv:
drv
// lib.optionalAttrs (t ? meta) { meta = (drv.meta or { }) // t.meta; };
test = passMeta (runTests driver);
in test // {
inherit driver image test manifest;
config = buildPackages.runCommand (name + ".dhall") testEnv' ''
${apps.dhall.program} <<< "${testConfig'}" > $out
'';
iso = apps.hw-iso.function testEnv' testConfig';
xml = buildPackages.runCommand (name + ".config") testEnv'
''${apps.render-init.program} <<< "(${testConfig'}).config" > $out'';
};
in {
callTest = path: args:
(import path ({
testEnv = {
inherit mkTest lib;
isLinux = false;
isNova = true;
};
pkgs = testPkgs;
inherit nixpkgs buildPackages legacyPackages;
} // args));
}

179
tests/driver-nova.nix

@ -1,179 +0,0 @@
# SPDX-License-Identifier: CC0-1.0
{ addManifest, apps, buildPackages, lib, nixpkgs, testPkgs, legacyPackages }:
let
testDriver = with buildPackages;
stdenv.mkDerivation {
name = "nova-genode-test-driver";
preferLocalBuild = true;
buildInputs = [ makeWrapper expect ];
dontUnpack = true;
installPhase = ''
install -Dm555 ${./nova-test-driver.exp} $out/bin/genode-test-driver
wrapProgram $out/bin/genode-test-driver \
--prefix PATH : "${lib.makeBinPath [ expect coreutils ]}"
'';
};
runTests = driver:
buildPackages.stdenv.mkDerivation {
name = "nova-" + driver.testName;
preferLocalBuild = true;
buildCommand = ''
mkdir -p $out/nix-support
${driver}/bin/genode-test-driver | tee $out/log
touch $out/nix-support
echo "report testlog $out log" >> $out/nix-support/hydra-build-products
'';
};
defaultScript =
''run_genode_until {child "init" exited with exit value 0} 120'';
mkTest = { name ? "unamed", testScript ? defaultScript, testConfig
, testInputs ? [ ], testEnv ? { }, extraPaths ? [ ], qemuArgs ? [ ], ...
}@t:
let
storeTarball = buildPackages.runCommand "store" { } ''
mkdir -p $out
tar cf "$out/store.tar" --absolute-names ${toString testInputs} ${
toString extraPaths
}
'';
storeManifest = lib.mergeManifests (map addManifest testInputs);
manifest = lib.mergeManifests (map addManifest
([ testPkgs.base-nova testPkgs.sotest-producer storeTarball ]
++ map testPkgs.genodeSources.depot [
"init"
"rtc_drv"
"vfs"
"cached_fs_rom"
]));
testConfig' = "${
./test-wrapper.dhall
} (${testConfig}) $(stat --format '%s' ${storeTarball}/store.tar) ${storeManifest} ${manifest}";
testEnv' = {
DHALL_GENODE = "${testPkgs.dhallGenode}/source.dhall";
DHALL_GENODE_TEST = "${./test.dhall}";
} // testEnv;
image = lib.novaImage ("nova-" + name) testEnv' testConfig';
build = lib.compileBoot name testEnv' testConfig';
baseSetup = ''
##
# Wait for a specific output of a already running spawned proce
#
proc wait_for_output { wait_for_re timeout_value running_spawn_id } {
global output
if {$wait_for_re == "forever"} {
set timeout -1
interact {
\003 {
send_user "Expect: 'interact' received 'strg+c' and was cancelled\n";
exit
}
-i $running_spawn_id
}
} else {
set timeout $timeout_value
}
expect {
-i $running_spawn_id -re $wait_for_re { }
eof { puts stderr "Error: Spawned process died unexpectedly"; exit -1 }
timeout { puts stderr "Error: Test execution timed out"; exit -1 }
}
set output $expect_out(buffer)
}
proc run_genode_until {{wait_for_re forever} {timeout_value 0} {running_spawn_id -1}} {
#
# If a running_spawn_id is specified, wait for the expected output
#
if {$running_spawn_id != -1} {
wait_for_output $wait_for_re $timeout_value $running_spawn_id
return
}
global env
global spawn_id
set TEST_MIB [expr (([file size ${image}/image.elf] + $env(TEST_RAM)) >> 20) + 24]
spawn ${buildPackages.qemu_test}/bin/qemu-system-x86_64 \
-machine q35 -cpu phenom -smp 2 \
-serial mon:stdio -nographic \
-m size=$TEST_MIB \
-kernel "${testPkgs.bender}/bender" \
-initrd "${testPkgs.NOVA}/hypervisor-x86_64 arg=iommu novpid serial,${image}/image.elf" \
${toString qemuArgs}
wait_for_output $wait_for_re $timeout_value $spawn_id
}
# TODO: not in TCL
global env
set out $env(out)
'';
driver = with buildPackages;
buildPackages.runCommand "genode-test-driver-${name}" ({
buildInputs = [ makeWrapper expect ];
inherit baseSetup testScript;
preferLocalBuild = true;
testName = name;
} // testEnv') ''
mkdir -p $out/bin
echo "$testScript" > $out/test-script
echo "$baseSetup" > $out/base-setup
source ${image.build}/stats
ln -s ${testDriver}/bin/genode-test-driver $out/bin/
wrapProgram $out/bin/genode-test-driver \
--set testScript "$testScript" \
--set baseSetup "$baseSetup" \
--set TEST_RAM $RAM \
'';
passMeta = drv:
drv
// lib.optionalAttrs (t ? meta) { meta = (drv.meta or { }) // t.meta; };
test = passMeta (runTests driver);
in test // {
inherit driver image test manifest;
inherit (image) build;
config = buildPackages.runCommand (name + ".dhall") testEnv' ''
${apps.dhall.program} <<< "${testConfig'}" > $out
'';
compile = lib.compileBoot name testConfig';
iso = apps.nova-iso.function testEnv' "${testConfig'}";
store = storeTarball;
xml = buildPackages.runCommand (name + ".config") testEnv'
''${apps.render-init.program} <<< "(${testConfig'}).config" > $out'';
};
in {
callTest = path: args:
(import path ({
testEnv = {
inherit mkTest lib;
isLinux = false;
isNova = true;
};
pkgs = testPkgs;
inherit nixpkgs buildPackages legacyPackages;
} // args));
}

37
tests/driver_manager.nix

@ -1,37 +0,0 @@
# SPDX-License-Identifier: CC0-1.0
{ testEnv, pkgs, buildPackages, ... }:
with pkgs;
testEnv.mkTest {
name = "driver_manager";
meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ];
testEnv = { drivers = ./../compositions/pc-drivers.dhall; };
testInputs = (map pkgs.genodeSources.depot [
"acpi_drv"
"ahci_drv"
"boot_fb_drv"
"driver_manager"
"dynamic_rom"
"input_filter"
"intel_fb_drv"
"platform_drv"
"ps2_drv"
"report_rom"
"rom_reporter"
"usb_drv"
"vesa_drv"
]) ++ (map pkgs.genodeSources.make [ "test/driver_manager" ]);
testScript = ''
catch { exec dd if=/dev/zero of=hdd_disk.raw bs=1M count=32 }
catch { exec ${buildPackages.e2fsprogs}/bin/mke2fs -F bin/hdd_disk.raw }
run_genode_until {.*all expected devices present and accessible.*} 120
'';
testConfig = ./driver_manager.dhall;
qemuArgs = [
"-device ahci,id=ahci"
"-drive id=hdd,file=hdd_disk.raw,format=raw,if=none"
"-device ide-hd,drive=hdd,bus=ahci.1"
];
}

17
tests/log.nix

@ -1,13 +1,16 @@
# SPDX-License-Identifier: CC0-1.0
{ testEnv, pkgs, ... }:
{ pkgs, ... }:
with pkgs;
testEnv.mkTest rec {
{
name = "log";
meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ];
testConfig = ./log.dhall;
testInputs = [ (pkgs.genodeSources.depot "test-log") ];
testScript = "run_genode_until {Test done.} 120";
machine = {
config = ./log.dhall;
inputs = [ (pkgs.genodeSources.depot "test-log") ];
};
testScript = ''
start_all()
machine.wait_until_serial_output("Test done.")
'';
}

21
tests/posix.nix

@ -1,6 +1,6 @@
# SPDX-License-Identifier: CC0-1.0
{ testEnv, pkgs, legacyPackages, ... }:
{ pkgs, legacyPackages, ... }:
with pkgs;
let
@ -21,14 +21,15 @@ let
uname -a
'';
};
in testEnv.mkTest rec {
in rec {
name = "posix";
testConfig = ''
${
./posix.dhall
} { bash = \"${bash}\", coreutils = \"${coreutils}\", script = \"${script}\" }'';
testInputs = map pkgs.genodeSources.depot [ "libc" "posix" "vfs_pipe" "vfs" ]
++ [ bash ];
extraPaths = [ script ] ++ (with legacyPackages; [ coreutils hello ]);
meta.maintainers = with pkgs.stdenv.lib.maintainers; [ ehmry ];
machine = {
config = ''
${
./posix.dhall
} { bash = \"${bash}\", coreutils = \"${coreutils}\", script = \"${script}\" }'';
inputs = map pkgs.genodeSources.depot [ "libc" "posix" "vfs_pipe" "vfs" ]
++ [ bash ];
extraPaths = [ script ] ++ (with legacyPackages; [ coreutils hello ]);
};
}

60
tests/solo5/default.nix

@ -1,46 +1,44 @@
# SPDX-License-Identifier: CC0-1.0
{ testEnv, pkgs, ... }:
{ pkgs, ... }:
with pkgs;
let
mkTest' = { name, testConfig, testInputs ? [ ], ... }@attrs:
testEnv.mkTest (attrs // {
solo5Test = { name, machine, ... }@args:
args // {
name = "solo5-" + name;
inherit testConfig;
testInputs = [ pkgs.solo5 pkgs.solo5.tests ] ++ testInputs;
});
applyMkTest = x: {
inherit (x) name;
value = mkTest' x;
};
mkTests = testList: builtins.listToAttrs (map applyMkTest testList);
machine = machine // {
inputs = [ pkgs.solo5 pkgs.solo5.tests ] ++ machine.inputs;
};
};
genodeDepot = pkgs.genodeSources.depot;
genodeMake = pkgs.genodeSources.make;
tests = [
{
name = "multi";
testConfig = "${./.}/solo5.dhall";
testInputs = map genodeMake [ "app/ping" ] ++ (map genodeDepot [
in map solo5Test [
{
name = "multi";
machine = {
config = "${./.}/solo5.dhall";
inputs = map genodeMake [ "app/ping" ] ++ (map genodeDepot [
"ram_block"
"nic_bridge"
"nic_loopback"
"sequence"
]);
}
{
name = "ssp";
testConfig = ./ssp.dhall;
testScript = ''
run_genode_until {Error: stack protector check failed} 30
'';
}
];
in mkTests tests
};
}
{
name = "ssp";
machine = {
config = ./ssp.dhall;
inputs = [ ];
};
testScript = ''
start_all()
machine.wait_until_serial_output("Error: stack protector check failed")
'';
}
]

912
tests/test-driver/test-driver.py

@ -0,0 +1,912 @@
#! /somewhere/python3
# Copyright (c) 2003-2020 Nixpkgs/NixOS contributors
from contextlib import contextmanager, _GeneratorContextManager
from queue import Queue, Empty
from typing import Tuple, Any, Callable, Dict, Iterator, Optional, List
from xml.sax.saxutils import XMLGenerator
import _thread
import atexit
import base64
import codecs
import os
import pathlib
import ptpython.repl
import pty
import re
import shlex
import shutil
import socket
import subprocess
import sys
import tempfile
import time
import unicodedata
CHAR_TO_KEY = {
"A": "shift-a",
"N": "shift-n",
"-": "0x0C",
"_": "shift-0x0C",
"B": "shift-b",
"O": "shift-o",
"=": "0x0D",
"+": "shift-0x0D",
"C": "shift-c",
"P": "shift-p",
"[": "0x1A",
"{": "shift-0x1A",
"D": "shift-d",
"Q": "shift-q",
"]": "0x1B",
"}": "shift-0x1B",
"E": "shift-e",
"R": "shift-r",
";": "0x27",
":": "shift-0x27",
"F": "shift-f",
"S": "shift-s",
"'": "0x28",
'"': "shift-0x28",
"G": "shift-g",
"T": "shift-t",
"`": "0x29",
"~": "shift-0x29",
"H": "shift-h",
"U": "shift-u",
"\\": "0x2B",
"|": "shift-0x2B",
"I": "shift-i",
"V": "shift-v",
",": "0x33",
"<": "shift-0x33",
"J": "shift-j",
"W": "shift-w",
".": "0x34",
">": "shift-0x34",
"K": "shift-k",
"X": "shift-x",
"/": "0x35",
"?": "shift-0x35",
"L": "shift-l",
"Y": "shift-y",
" ": "spc",
"M": "shift-m",
"Z": "shift-z",
"\n": "ret",
"!": "shift-0x02",
"@": "shift-0x03",
"#": "shift-0x04",
"$": "shift-0x05",
"%": "shift-0x06",
"^": "shift-0x07",
"&": "shift-0x08",
"*": "shift-0x09",
"(": "shift-0x0A",
")": "shift-0x0B",
}
# Forward references
log: "Logger"
machines: "List[Machine]"
def eprint(*args: object, **kwargs: Any) -> None:
print(*args, file=sys.stderr, **kwargs)
def make_command(args: list) -> str:
return " ".join(map(shlex.quote, (map(str, args))))
def retry(fn: Callable) -> None:
"""Call the given function repeatedly, with 1 second intervals,
until it returns True or a timeout is reached.
"""
for _ in range(900):
if fn(False):
return
time.sleep(1)
if not fn(True):
raise Exception("action timed out")
class Logger:
def __init__(self) -> None:
self.logfile = os.environ.get("LOGFILE", "/dev/null")
self.logfile_handle = codecs.open(self.logfile, "wb")
self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8")
self.queue: "Queue[Dict[str, str]]" = Queue()
self.xml.startDocument()
self.xml.startElement("logfile", attrs={})
def close(self) -> None:
self.xml.endElement("logfile")
self.xml.endDocument()
self.logfile_handle.close()
def sanitise(self, message: str) -> str:
return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C")
def maybe_prefix(self, message: str, attributes: Dict[str, str]) -> str:
if "machine" in attributes:
return "{}: {}".format(attributes["machine"], message)
return message
def log_line(self, message: str, attributes: Dict[str, str]) -> None:
self.xml.startElement("line", attributes)
self.xml.characters(message)
self.xml.endElement("line")
def log(self, message: str, attributes: Dict[str, str] = {}) -> None:
eprint(self.maybe_prefix(message, attributes))
self.drain_log_queue()
self.log_line(message, attributes)
def enqueue(self, message: Dict[str, str]) -> None:
self.queue.put(message)
def drain_log_queue(self) -> None:
try:
while True:
item = self.queue.get_nowait()
attributes = {"machine": item["machine"], "type": "serial"}
self.log_line(self.sanitise(item["msg"]), attributes)
except Empty:
pass
@contextmanager
def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]:
eprint(self.maybe_prefix(message, attributes))
self.xml.startElement("nest", attrs={})
self.xml.startElement("head", attributes)
self.xml.characters(message)
self.xml.endElement("head")
tic = time.time()
self.drain_log_queue()
yield
self.drain_log_queue()
toc = time.time()
self.log("({:.2f} seconds)".format(toc - tic))
self.xml.endElement("nest")
class Machine:
def __init__(self, args: Dict[str, Any]) -> None:
if "name" in args:
self.name = args["name"]
else:
self.name = "machine"
cmd = args.get("startCommand", None)
if cmd:
match = re.search("bin/run-(.+)-vm$", cmd)
if match:
self.name = match.group(1)
self.script = args.get("startCommand", self.create_startcommand(args))
tmp_dir = os.environ.get("TMPDIR", tempfile.gettempdir())
def create_dir(name: str) -> str:
path = os.path.join(tmp_dir, name)
os.makedirs(path, mode=0o700, exist_ok=True)
return path
self.state_dir = create_dir("vm-state-{}".format(self.name))
self.shared_dir = create_dir("shared-xchg")
self.booted = False
self.connected = False
self.pid: Optional[int] = None
self.socket = None
self.monitor: Optional[socket.socket] = None
self.logger: Logger = args["log"]
self.serialQueue: "Queue[str]" = Queue()
self.allow_reboot = args.get("allowReboot", False)
@staticmethod
def create_startcommand(args: Dict[str, str]) -> str:
net_backend = "-netdev user,id=net0"
net_frontend = "-device virtio-net-pci,netdev=net0"
start_command = "qemu-system-x86_64 -m 384 $QEMU_OPTS "
if "hda" in args:
hda_path = os.path.abspath(args["hda"])
if args.get("hdaInterface", "") == "scsi":
start_command += (
"-drive id=hda,file="
+ hda_path
+ ",werror=report,if=none "
+ "-device scsi-hd,drive=hda "
)
else:
start_command += (
"-drive file="
+ hda_path
+ ",if="
+ args["hdaInterface"]
+ ",werror=report "
)
if "cdrom" in args:
start_command += "-cdrom " + args["cdrom"] + " "
if "usb" in args:
start_command += (
"-device piix3-usb-uhci -drive "
+ "id=usbdisk,file="
+ args["usb"]
+ ",if=none,readonly "
+ "-device usb-storage,drive=usbdisk "
)
if "bios" in args:
start_command += "-bios " + args["bios"] + " "
start_command += args.get("qemuFlags", "")
return start_command
def is_up(self) -> bool:
return self.booted and self.connected
def log(self, msg: str) -> None:
self.logger.log(msg, {"machine": self.name})
def nested(self, msg: str, attrs: Dict[str, str] = {}) -> _GeneratorContextManager:
my_attrs = {"machine": self.name}
my_attrs.update(attrs)
return self.logger.nested(msg, my_attrs)
def wait_for_monitor_prompt(self) -> str:
assert self.monitor is not None
answer = ""
while True:
undecoded_answer = self.monitor.recv(1024)
if not undecoded_answer:
break
answer += undecoded_answer.decode()
if answer.endswith("(qemu) "):
break
return answer
def send_monitor_command(self, command: str) -> str:
message = ("{}\n".format(command)).encode()
self.log("sending monitor command: {}".format(command))
assert self.monitor is not None
self.monitor.send(message)
return self.wait_for_monitor_prompt()
def wait_for_unit(self, unit: str, user: Optional[str] = None) -> None:
"""Wait for a systemd unit to get into "active" state.
Throws exceptions on "failed" and "inactive" states as well as
after timing out.
"""
def check_active(_: Any) -> bool:
info = self.get_unit_info(unit, user)
state = info["ActiveState"]
if state == "failed":
raise Exception('unit "{}" reached state "{}"'.format(unit, state))
if state == "inactive":
status, jobs = self.systemctl("list-jobs --full 2>&1", user)
if "No jobs" in jobs:
info = self.get_unit_info(unit, user)
if info["ActiveState"] == state:
raise Exception(
(
'unit "{}" is inactive and there ' "are no pending jobs"
).format(unit)
)
return state == "active"
retry(check_active)
def get_unit_info(self, unit: str, user: Optional[str] = None) -> Dict[str, str]:
status, lines = self.systemctl('--no-pager show "{}"'.format(unit), user)
if status != 0:
raise Exception(
'retrieving systemctl info for unit "{}" {} failed with exit code {}'.format(
unit, "" if user is None else 'under user "{}"'.format(user), status
)
)
line_pattern = re.compile(r"^([^=]+)=(.*)$")
def tuple_from_line(line: str) -> Tuple[str, str]:
match = line_pattern.match(line)
assert match is not None
return match[1], match[2]
return dict(
tuple_from_line(line)
for line in lines.split("\n")
if line_pattern.match(line)
)
def systemctl(self, q: str, user: Optional[str] = None) -> Tuple[int, str]:
if user is not None:
q = q.replace("'", "\\'")
return self.execute(
(
"su -l {} --shell /bin/sh -c "
"$'XDG_RUNTIME_DIR=/run/user/`id -u` "
"systemctl --user {}'"
).format(user, q)
)
return self.execute("systemctl {}".format(q))
def require_unit_state(self, unit: str, require_state: str = "active") -> None:
with self.nested(
"checking if unit ‘{}’ has reached state '{}'".format(unit, require_state)
):
info = self.get_unit_info(unit)
state = info["ActiveState"]
if state != require_state:
raise Exception(
"Expected unit ‘{}’ to to be in state ".format(unit)
+ "'{}' but it is in state ‘{}".format(require_state, state)
)
def execute(self, command: str) -> Tuple[int, str]:
self.connect()
out_command = "( {} ); echo '|!=EOF' $?\n".format(command)
self.shell.send(out_command.encode())
output = ""
status_code_pattern = re.compile(r"(.*)\|\!=EOF\s+(\d+)")
while True:
chunk = self.shell.recv(4096).decode(errors="ignore")
match = status_code_pattern.match(chunk)
if match:
output += match[1]
status_code = int(match[2])
return (status_code, output)
output += chunk
def succeed(self, *commands: str) -> str:
"""Execute each command and check that it succeeds."""
output = ""
for command in commands:
with self.nested("must succeed: {}".format(command)):
(status, out) = self.execute(command)
if status != 0:
self.log("output: {}".format(out))
raise Exception(
"command `{}` failed (exit code {})".format(command, status)
)
output += out
return output
def fail(self, *commands: str) -> None:
"""Execute each command and check that it fails."""
for command in commands:
with self.nested("must fail: {}".format(command)):
status, output = self.execute(command)
if status == 0:
raise Exception(
"command `{}` unexpectedly succeeded".format(command)
)
def wait_until_succeeds(self, command: str) -> str:
"""Wait until a command returns success and return its output.
Throws an exception on timeout.
"""
output = ""
def check_success(_: Any) -> bool:
nonlocal output
status, output = self.execute(command)
return status == 0
with self.nested("waiting for success: {}".format(command)):
retry(check_success)
return output
def wait_until_fails(self, command: str) -> str:
"""Wait until a command returns failure.
Throws an exception on timeout.
"""
output = ""
def check_failure(_: Any) -> bool:
nonlocal output
status, output = self.execute(command)
return status != 0
with self.nested("waiting for failure: {}".format(command)):
retry(check_failure)
return output
def wait_for_shutdown(self) -> None:
if not self.booted:
return
with self.nested("waiting for the VM to power off"):
sys.stdout.flush()
self.process.wait()
self.pid = None
self.booted = False
self.connected = False
def get_tty_text(self, tty: str) -> str:
status, output = self.execute(
"fold -w$(stty -F /dev/tty{0} size | "
"awk '{{print $2}}') /dev/vcs{0}".format(tty)
)
return output
def wait_until_tty_matches(self, tty: str, regexp: str) -> None:
"""Wait until the visible output on the chosen TTY matches regular
expression. Throws an exception on timeout.
"""
matcher = re.compile(regexp)
def tty_matches(last: bool) -> bool:
text = self.get_tty_text(tty)
if last:
self.log(
f"Last chance to match /{regexp}/ on TTY{tty}, "
f"which currently contains: {text}"
)
return len(matcher.findall(text)) > 0
with self.nested("waiting for {} to appear on tty {}".format(regexp, tty)):
retry(tty_matches)