diff --git a/repos/gems/run/depot_autopilot.run b/repos/gems/run/depot_autopilot.run index 26ded8fd2..2406594eb 100644 --- a/repos/gems/run/depot_autopilot.run +++ b/repos/gems/run/depot_autopilot.run @@ -1,28 +1,8 @@ -#################### -## User interface ## -#################### - # -# Additional environment variables considered by this run script +# Integration of Depot Autopilot into a Run-tool-based test infrastructure # -# TEST_PKGS list of test package-archives if set (default list below) -# TEST_SRCS list of test source-archive if TEST_PKGS is set (default list below) -# TEST_BUILDS list of test build-targets that shall be build from repos -# TEST_MODULES list of test boot-modules to overlay the test depot-content -# -# To get a hint which build components and which boot modules you may want to -# enter for a given test package-archive, please see these files: -# -# repos//recipes/pkg//archives -# repos//recipes/pkg//runtime ( tag) -# -# Example: -# -# ! KERNEL="nova" \ -# TEST_PKGS="test-libc_vfs test-log" \ -# TEST_BUILDS="server/ram_fs test/libc_vfs" \ -# TEST_MODULES="ram_fs test-libc_vfs vfs.lib.so" \ -# make run/depot_autopilot +# For a detailed documentation of the user interface and the features of this +# Run script please see _repos/gems/src/app/depot_autopilot/README_. # @@ -235,6 +215,7 @@ proc prepare_to_run_genode { } { global test_srcs global test_builds global test_modules + global test_repeat global last_test_pkg global run_genode_failed global serial_id @@ -364,7 +345,7 @@ proc prepare_to_run_genode { } { - + @@ -482,6 +463,55 @@ proc init_previous_results {} { set previous_skipped 0 } +# +# Initialize variables that can be configured via the user interface +# +proc init_test_setting {} { + + global test_pkgs + global test_srcs + global test_builds + global test_modules + global test_repeat + global default_test_pkgs + global default_test_srcs + global skip_test_pkg + + # + # Initialize the lists of test package-archives, source-archives, + # build-targets, and boot-modules to be considered by the run script + # + set test_pkgs [get_env_var TEST_PKGS ""] + set test_srcs [get_env_var TEST_SRCS ""] + set test_builds [get_env_var TEST_BUILDS ""] + set test_modules [get_env_var TEST_MODULES ""] + set test_repeat [get_env_var TEST_REPEAT "false"] + + # + # If the user didn't declare an individual list of test package-archives, use + # the default lists of test package-archives and source-archives + # + set nr_of_tests_to_run 0 + if {$test_pkgs == ""} { + foreach test_pkg $default_test_pkgs { + append test_pkgs " $test_pkg " + if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} { + incr nr_of_tests_to_run + } + } + foreach test_src_pkg $default_test_srcs { + append test_srcs " $test_src_pkg " + } + } else { + foreach test_pkg $test_pkgs { + if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} { + incr nr_of_tests_to_run + } + } + } +} + + rename exit run_tool_exit proc exit {{status 0}} { @@ -677,40 +707,8 @@ if {[info exists qemu_args]} { set initial_qemu_args $qemu_args } -# -# Initialize the lists of test package-archives, source-archives, -# build-targets, and boot-modules to be considered by the run script -# -set test_pkgs [get_env_var TEST_PKGS ""] -set test_srcs [get_env_var TEST_SRCS ""] -set test_builds [get_env_var TEST_BUILDS ""] -set test_modules [get_env_var TEST_MODULES ""] - -# -# If the user didn't declare an individual list of test package-archives, use -# the default lists of test package-archives and source-archives -# -set nr_of_tests_to_run 0 -if {$test_pkgs == ""} { - foreach test_pkg $default_test_pkgs { - append test_pkgs " $test_pkg " - if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} { - incr nr_of_tests_to_run - } - } - foreach test_src_pkg $default_test_srcs { - append test_srcs " $test_src_pkg " - } -} else { - foreach test_pkg $test_pkgs { - if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} { - incr nr_of_tests_to_run - } - } -} -puts "Number of tests to run: $nr_of_tests_to_run" - -# initialize variables that hold previous results +# initialize global variables +init_test_setting init_previous_results # generic preparation for each system boot @@ -787,13 +785,16 @@ while {1} { puts $last_test_result exit -1 } - if {$previous_results != ""} { append previous_results \012 } append previous_results $last_test_result } - + # if the Autopilot is currently repeating, reset repeat-influenced variables + if {[llength $test_pkgs] == 0} { + init_test_setting + init_previous_results + } # determine timeout for the next timeout set min_timeout 10 regexp {depot_autopilot\] --- Run "(.*?)" \(max ([0-9]*?) } $output] ignore last_test_pkg min_timeout diff --git a/repos/gems/src/app/depot_autopilot/README b/repos/gems/src/app/depot_autopilot/README index a50d2f3d2..5278a9950 100644 --- a/repos/gems/src/app/depot_autopilot/README +++ b/repos/gems/src/app/depot_autopilot/README @@ -20,22 +20,21 @@ provided: platforms are comparable -The Depot Autopilot is accompanied by the run script +The Depot Autopilot is accompanied by the Run script 'repos/gems/run/depot_autopilot.run' which doesn't describe a classical test -but rather helps integrating the Autopilot into a linux-based test +but rather helps integrating the Autopilot into a Linux-based test environment. The run script provides the following features: -* The user only lists the desired test packages and selectively defines the - platform compatibility for tests where this is necessary -* The run script executes the Autopilot scenario on the target platform, - repeadedly if necessary, until an overview with all tests can be presented -* Each time the system gets booted, the Autopilot is configured only with - the remaining tests -* Tests that caused the run script to re-boot the target platform are listed - as "failed" with cause "reboot" in the overview -* Exits with 0 if a complete overview can be accomplished and all tests are - either "ok" or "skipped" and otherwise with a value < 0 -* Provides a convenient interface for debugging single tests +* Creates a complete Autopilot scenario from a list of local test packages +* Optional platform dependencies can be defined for each test package +* Simple interface for selectively using test ingredients built from source +* Can be combined with a list of source packages and gcov to show test coverage +* Exits with 0 if all tests are either "ok" or "skipped", otherwise with < 0 +* The target is re-booted if the scenario gets stuck +* On each re-boot the Autopilot is configured only with the remaining tests +* Tests that caused a re-boot are listed as "failed" with cause "reboot" +* The final Autopilot result overview reflects all results from all boots +* If a complete result overview can't be obtained, a partial one is simulated Depot Autopilot component @@ -46,7 +45,7 @@ Configuration This is an example configuration of the Depot Autopilot: -! +! ! ! ! @@ -79,11 +78,19 @@ This is an example configuration of the Depot Autopilot: appropriate variant of the test packages. Must be one of "x86_64", "x86_32", "arm_v7a". -:config children_label_prefix: +:: Label prefix of LOG sessions of the components of a test. This is required to relate incoming LOG-session request to a running test. +:: + + Can be one of + + "false" - process the given test list only once, + "until_forever" - endlessly repeat processing the given test list, + "until_failed" - repeat processing the given test list until it fails. + :: Contains Init configuration that shall always be added to the configuration @@ -319,8 +326,65 @@ Integration components as part of the LOG output of the Depot Autopilot. +Depot Autopilot Run script +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, the Run script tries to evaluate all tests that are internally +configured, that is, normally, all that are available as package recipes in +the main repositories. This includes one test that is prepared for the later +examination by gcov as well as gcov itself, which is executed like a test at +the end of the test list to examine the above mentioned adapted test. The test +list is processed only once and all test ingredients are used from packages, +assuming the depot user is 'genodelabs'. For this default behavior just call +the script like any "normal" Run script: + +! make run/depot_autopilot KERNEL="nova" + +However, this is normally a pretty time intensive process and meant for the +integration into automatic test infrastructures rather than manual evaluation. +This is why you might want to adapt the scenario, for instance, for debugging +one or more tests that failed during your automated testing. For this purpose, +a user interface is given through specific environment variables. This is an +example containing all of these variables: + +! make run/depot_autopilot \ +! > TEST_PKGS="test-libc_vfs test-log" \ +! > TEST_SRCS="test-xml_generator test-xml_node" \ +! > TEST_BUILDS="server/ram_fs test/libc_vfs" \ +! > TEST_MODULES="ram_fs test-libc_vfs vfs.lib.so" \ +! > TEST_REPEAT="until_failed" \ +! > KERNEL="nova" + +:TEST_PKGS: + + List of test package-archives that supersedes the internal list if set. + +:TEST_SRCS: + + List of test source-archives that supersedes the internal list if set. + +:TEST_BUILDS: + + List of test build-targets that be build from the local repositories. + +:TEST_MODULES: + + List of test boot-modules that overlay the contents of the package-archives. + +:TEST_REPEAT: + + See the attribute of the Depot Autopilot. + +To get a hint which build components (TEST_BUILDS) and which boot modules +(TEST_MODULES) you may want to enter for a given test package, you may have +a look at the package recipe: + +! repos//recipes/pkg//archives +! repos//recipes/pkg//runtime ( tag) + + Examples -------- -See the run script 'repos/gems/run/depot_autopilot.run' for a comprehensive -example of how to use the Depot Autopilot. +Please see the run script 'repos/gems/run/depot_autopilot.run' for a +comprehensive example of how to use the Depot Autopilot. diff --git a/repos/gems/src/app/depot_autopilot/child.cc b/repos/gems/src/app/depot_autopilot/child.cc index 0c1140973..30a1d69c4 100644 --- a/repos/gems/src/app/depot_autopilot/child.cc +++ b/repos/gems/src/app/depot_autopilot/child.cc @@ -375,6 +375,19 @@ Child::Child(Allocator &alloc, { } +Child::~Child() +{ + while (Timeout_event *event = _timeout_events.first()) { + _timeout_events.remove(event); + destroy(_alloc, event); + } + while (Log_event *event = _log_events.first()) { + _log_events.remove(event); + destroy(_alloc, event); + } +} + + void Child::log_session_write(Log_event::Line const &log_line) { if (_skip) { @@ -684,10 +697,14 @@ Child::State_name Child::_padded_state_name() const } +void Child::print_conclusion() +{ + log(" ", _conclusion); +} + void Child::conclusion(Result &result) { struct Bad_state : Exception { }; - log(" ", _conclusion); switch (_state) { case SUCCEEDED: result.succeeded++; break; case FAILED: result.failed++; break; diff --git a/repos/gems/src/app/depot_autopilot/child.h b/repos/gems/src/app/depot_autopilot/child.h index 5df699413..0e90c3dd6 100644 --- a/repos/gems/src/app/depot_autopilot/child.h +++ b/repos/gems/src/app/depot_autopilot/child.h @@ -221,8 +221,12 @@ class Depot_deploy::Child : public List_model::Element Timer::Connection &timer, Genode::Signal_context_capability const &config_handler); + ~Child(); + void log_session_write(Log_event::Line const &log_line); + void print_conclusion(); + void conclusion(Result &result); void event_occured(Event const &event, diff --git a/repos/gems/src/app/depot_autopilot/children.h b/repos/gems/src/app/depot_autopilot/children.h index 0416dac16..f22d73152 100644 --- a/repos/gems/src/app/depot_autopilot/children.h +++ b/repos/gems/src/app/depot_autopilot/children.h @@ -158,6 +158,13 @@ class Depot_deploy::Children return finished; } + void print_conclusion() + { + _children.for_each([&] (Child &child) { + child.print_conclusion(); + }); + } + void conclusion(Result &result) { _children.for_each([&] (Child &child) { diff --git a/repos/gems/src/app/depot_autopilot/config.xsd b/repos/gems/src/app/depot_autopilot/config.xsd index a360e4201..d88bbd645 100644 --- a/repos/gems/src/app/depot_autopilot/config.xsd +++ b/repos/gems/src/app/depot_autopilot/config.xsd @@ -4,6 +4,14 @@ + + + + + + + + @@ -91,6 +99,7 @@ + diff --git a/repos/gems/src/app/depot_autopilot/main.cc b/repos/gems/src/app/depot_autopilot/main.cc index 6423aa5c2..65c4d2b9d 100644 --- a/repos/gems/src/app/depot_autopilot/main.cc +++ b/repos/gems/src/app/depot_autopilot/main.cc @@ -112,91 +112,133 @@ class Depot_deploy::Log_root : public Root_component struct Depot_deploy::Main { - typedef String<128> Name; - - Env &_env; - Attached_rom_dataspace _config { _env, "config" }; - Attached_rom_dataspace _blueprint { _env, "blueprint" }; - Expanding_reporter _query_reporter { _env, "query" , "query"}; - Expanding_reporter _init_config_reporter { _env, "config", "init.config"}; - Heap _heap { _env.ram(), _env.rm() }; - Reconstructible _children_label_prefix { }; - Timer::Connection _timer { _env }; - Signal_handler
_config_handler { _env.ep(), *this, &Main::_handle_config }; - Children _children { _heap, _timer, _config_handler }; - Log_root _log_root { _env.ep(), _heap, _children, *_children_label_prefix }; - - void _handle_config() + struct Repeatable { - _config.update(); - _blueprint.update(); + Env &_env; + Heap &_heap; + Signal_transmitter _repeat_handler; + Attached_rom_dataspace _config { _env, "config" }; + Attached_rom_dataspace _blueprint { _env, "blueprint" }; + Expanding_reporter _query_reporter { _env, "query" , "query"}; + Expanding_reporter _init_config_reporter { _env, "config", "init.config"}; + Reconstructible _children_label_prefix { }; + Timer::Connection _timer { _env }; + Signal_handler _config_handler { _env.ep(), *this, &Repeatable::_handle_config }; + Children _children { _heap, _timer, _config_handler }; - Xml_node const config = _config.xml(); + void _handle_config() + { + _config.update(); + _blueprint.update(); - _children_label_prefix.construct(config.attribute_value("children_label_prefix", String<160>())); - _children.apply_config(config); - _children.apply_blueprint(_blueprint.xml()); + Xml_node const config = _config.xml(); - /* determine CPU architecture of deployment */ - typedef String<16> Arch; - Arch const arch = config.attribute_value("arch", Arch()); - if (!arch.valid()) - warning("config lacks 'arch' attribute"); + _children_label_prefix.construct(config.attribute_value("children_label_prefix", String<160>())); + _children.apply_config(config); + _children.apply_blueprint(_blueprint.xml()); - /* generate init config containing all configured start nodes */ - bool finished; - _init_config_reporter.generate([&] (Xml_generator &xml) { - Xml_node static_config = config.sub_node("static"); - xml.append(static_config.content_base(), static_config.content_size()); - Child::Depot_rom_server const parent { }; - finished = _children.gen_start_nodes(xml, config.sub_node("common_routes"), - parent, parent); - }); - if (finished) { + /* determine CPU architecture of deployment */ + typedef String<16> Arch; + Arch const arch = config.attribute_value("arch", Arch()); + if (!arch.valid()) + warning("config lacks 'arch' attribute"); - Result result; - unsigned long previous_time_sec { 0UL }; - if (config.has_sub_node("previous-results")) { - Xml_node const previous_results = config.sub_node("previous-results"); - previous_time_sec += previous_results.attribute_value("time_sec", 0UL); - result.succeeded += previous_results.attribute_value("succeeded", 0UL); - result.failed += previous_results.attribute_value("failed", 0UL); - result.skipped += previous_results.attribute_value("skipped", 0UL); - } - unsigned long const time_us { _timer.curr_time().trunc_to_plain_us().value }; - unsigned long time_ms { time_us / 1000UL }; - unsigned long const time_sec { time_ms / 1000UL }; - time_ms = time_ms - time_sec * 1000UL; + /* generate init config containing all configured start nodes */ + bool finished; + _init_config_reporter.generate([&] (Xml_generator &xml) { + Xml_node static_config = config.sub_node("static"); + xml.append(static_config.content_base(), static_config.content_size()); + Child::Depot_rom_server const parent { }; + finished = _children.gen_start_nodes(xml, config.sub_node("common_routes"), + parent, parent); + }); + if (finished) { - log("\n--- Finished after ", time_sec + previous_time_sec, ".", time_ms < 10 ? "00" : time_ms < 100 ? "0" : "", time_ms, " sec ---\n"); - if (config.has_sub_node("previous-results")) { - Xml_node const previous_results = config.sub_node("previous-results"); - if (previous_results.content_size()) { - log(Cstring(previous_results.content_base(), previous_results.content_size())); + Result result; + unsigned long previous_time_sec { 0UL }; + if (config.has_sub_node("previous-results")) { + Xml_node const previous_results = config.sub_node("previous-results"); + previous_time_sec += previous_results.attribute_value("time_sec", 0UL); + result.succeeded += previous_results.attribute_value("succeeded", 0UL); + result.failed += previous_results.attribute_value("failed", 0UL); + result.skipped += previous_results.attribute_value("skipped", 0UL); + } + unsigned long const time_us { _timer.curr_time().trunc_to_plain_us().value }; + unsigned long time_ms { time_us / 1000UL }; + unsigned long const time_sec { time_ms / 1000UL }; + time_ms = time_ms - time_sec * 1000UL; + + _children.conclusion(result); + int exit_code = result.failed ? -1 : 0; + + typedef String<12> Repeat; + Repeat repeat = config.attribute_value("repeat", Repeat("false")); + if (repeat == Repeat("until_forever") || + (repeat == Repeat("until_failed") && exit_code == 0)) { + + char const *empty_config_str = ""; + Xml_node const empty_config(empty_config_str, 10); + _children.apply_config(empty_config); + _query_reporter.generate([&] (Xml_generator &xml) { xml.attribute("arch", arch); }); + _init_config_reporter.generate([&] (Xml_generator &) { }); + _repeat_handler.submit(); + return; + + } else { + + log("\n--- Finished after ", time_sec + previous_time_sec, ".", time_ms < 10 ? "00" : time_ms < 100 ? "0" : "", time_ms, " sec ---\n"); + if (config.has_sub_node("previous-results")) { + Xml_node const previous_results = config.sub_node("previous-results"); + if (previous_results.content_size()) { + log(Cstring(previous_results.content_base(), previous_results.content_size())); + } + } + _children.print_conclusion(); + log(""); + log(result); + log(""); + _env.parent().exit(exit_code); } } - _children.conclusion(result); - log(""); - log(result); - log(""); - _env.parent().exit(result.failed ? -1 : 0); + + /* update query for blueprints of all unconfigured start nodes */ + if (arch.valid()) { + _query_reporter.generate([&] (Xml_generator &xml) { + xml.attribute("arch", arch); + _children.gen_queries(xml); + }); + } } - /* update query for blueprints of all unconfigured start nodes */ - if (arch.valid()) { - _query_reporter.generate([&] (Xml_generator &xml) { - xml.attribute("arch", arch); - _children.gen_queries(xml); - }); + Repeatable(Env &env, + Signal_context_capability const &repeat_handler, + Heap &heap) + : + _env(env), + _heap(heap), + _repeat_handler(repeat_handler) + { + _config .sigh(_config_handler); + _blueprint.sigh(_config_handler); + + _handle_config(); } + }; + + + Env &_env; + Signal_handler
_repeat_handler { _env.ep(), *this, &Main::_handle_repeat }; + Heap _heap { _env.ram(), _env.rm() }; + Reconstructible _repeatable { _env, _repeat_handler, _heap }; + Log_root _log_root { _env.ep(), _heap, _repeatable->_children, *_repeatable->_children_label_prefix }; + + void _handle_repeat() + { + _repeatable.construct(_env, _repeat_handler, _heap); } Main(Env &env) : _env(env) { - _config .sigh(_config_handler); - _blueprint.sigh(_config_handler); - - _handle_config(); _env.parent().announce(_env.ep().manage(_log_root)); } };