ehmry/genode
ehmry
/
genode
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

depot_autopilot: provide repeat mode 

Adds an config attribute to the Depot Autopilot component:

:<config repeat>:

  Can be one of

    "false"         - process the given test list only once,
    "until_forever" - endlessly repeat processing the given test list,
    "until_failed"  - repeat processing the given test list until it fails.

Adds an environment variable to the Depot Autopilot Run script:

:TEST_REPEAT:

  Same as the <config repeat> attribute of the Depot Autopilot.

This is useful when having to debug very sporadic errors during one test
or a series of tests.
This commit is contained in:
Martin Stein 2018-12-19 15:07:46 +01:00 committed by Norman Feske
parent b7cb5839eb
commit f0842a27c5
7 changed files with 291 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
repos/gems/run/depot_autopilot.run
View File

@ -1,28 +1,8 @@
####################
## User interface ##
####################
#
# Additional environment variables considered by this run script
# Integration of Depot Autopilot into a Run-tool-based test infrastructure
#
# TEST_PKGS list of test package-archives if set (default list below)
# TEST_SRCS list of test source-archive if TEST_PKGS is set (default list below)
# TEST_BUILDS list of test build-targets that shall be build from repos
# TEST_MODULES list of test boot-modules to overlay the test depot-content
#
# To get a hint which build components and which boot modules you may want to
# enter for a given test package-archive, please see these files:
#
# repos/<REPO>/recipes/pkg/<TEST_PKG>/archives
# repos/<REPO>/recipes/pkg/<TEST_PKG>/runtime (<content> tag)
#
# Example:
#
# ! KERNEL="nova" \
# TEST_PKGS="test-libc_vfs test-log" \
# TEST_BUILDS="server/ram_fs test/libc_vfs" \
# TEST_MODULES="ram_fs test-libc_vfs vfs.lib.so" \
# make run/depot_autopilot
# For a detailed documentation of the user interface and the features of this
# Run script please see _repos/gems/src/app/depot_autopilot/README_.
#
@ -235,6 +215,7 @@ proc prepare_to_run_genode { } {
global test_srcs
global test_builds
global test_modules
global test_repeat
global last_test_pkg
global run_genode_failed
global serial_id
@ -364,7 +345,7 @@ proc prepare_to_run_genode { } {
<start name="depot_autopilot" priority="-1">
<resource name="RAM" quantum="2M"/>
<provides> <service name="LOG"/> </provides>
<config arch="} [depot_spec] {" children_label_prefix="dynamic -> ">
<config repeat="} $test_repeat {" arch="} [depot_spec] {" children_label_prefix="dynamic -> ">
<static>
<parent-provides>
<service name="ROM"/>
@ -482,6 +463,55 @@ proc init_previous_results {} {
set previous_skipped 0
}
#
# Initialize variables that can be configured via the user interface
#
proc init_test_setting {} {
global test_pkgs
global test_srcs
global test_builds
global test_modules
global test_repeat
global default_test_pkgs
global default_test_srcs
global skip_test_pkg
#
# Initialize the lists of test package-archives, source-archives,
# build-targets, and boot-modules to be considered by the run script
#
set test_pkgs [get_env_var TEST_PKGS ""]
set test_srcs [get_env_var TEST_SRCS ""]
set test_builds [get_env_var TEST_BUILDS ""]
set test_modules [get_env_var TEST_MODULES ""]
set test_repeat [get_env_var TEST_REPEAT "false"]
#
# If the user didn't declare an individual list of test package-archives, use
# the default lists of test package-archives and source-archives
#
set nr_of_tests_to_run 0
if {$test_pkgs == ""} {
foreach test_pkg $default_test_pkgs {
append test_pkgs " $test_pkg "
if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} {
incr nr_of_tests_to_run
}
}
foreach test_src_pkg $default_test_srcs {
append test_srcs " $test_src_pkg "
}
} else {
foreach test_pkg $test_pkgs {
if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} {
incr nr_of_tests_to_run
}
}
}
}
rename exit run_tool_exit
proc exit {{status 0}} {
@ -677,40 +707,8 @@ if {[info exists qemu_args]} {
set initial_qemu_args $qemu_args
}
#
# Initialize the lists of test package-archives, source-archives,
# build-targets, and boot-modules to be considered by the run script
#
set test_pkgs [get_env_var TEST_PKGS ""]
set test_srcs [get_env_var TEST_SRCS ""]
set test_builds [get_env_var TEST_BUILDS ""]
set test_modules [get_env_var TEST_MODULES ""]
#
# If the user didn't declare an individual list of test package-archives, use
# the default lists of test package-archives and source-archives
#
set nr_of_tests_to_run 0
if {$test_pkgs == ""} {
foreach test_pkg $default_test_pkgs {
append test_pkgs " $test_pkg "
if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} {
incr nr_of_tests_to_run
}
}
foreach test_src_pkg $default_test_srcs {
append test_srcs " $test_src_pkg "
}
} else {
foreach test_pkg $test_pkgs {
if {!([info exists skip_test_pkg($test_pkg)] && $skip_test_pkg($test_pkg))} {
incr nr_of_tests_to_run
}
}
}
puts "Number of tests to run: $nr_of_tests_to_run"
# initialize variables that hold previous results
# initialize global variables
init_test_setting
init_previous_results
# generic preparation for each system boot
@ -787,13 +785,16 @@ while {1} {
puts $last_test_result
exit -1
}
if {$previous_results != ""} {
append previous_results \012
}
append previous_results $last_test_result
}
# if the Autopilot is currently repeating, reset repeat-influenced variables
if {[llength $test_pkgs] == 0} {
init_test_setting
init_previous_results
}
# determine timeout for the next timeout
set min_timeout 10
regexp {depot_autopilot\] --- Run "(.*?)" \(max ([0-9]*?) } $output] ignore last_test_pkg min_timeout

98
repos/gems/src/app/depot_autopilot/README
View File

@ -20,22 +20,21 @@ provided:
platforms are comparable
The Depot Autopilot is accompanied by the run script
The Depot Autopilot is accompanied by the Run script
'repos/gems/run/depot_autopilot.run' which doesn't describe a classical test
but rather helps integrating the Autopilot into a linux-based test
but rather helps integrating the Autopilot into a Linux-based test
environment. The run script provides the following features:
* The user only lists the desired test packages and selectively defines the
platform compatibility for tests where this is necessary
* The run script executes the Autopilot scenario on the target platform,
repeadedly if necessary, until an overview with all tests can be presented
* Each time the system gets booted, the Autopilot is configured only with
the remaining tests
* Tests that caused the run script to re-boot the target platform are listed
as "failed" with cause "reboot" in the overview
* Exits with 0 if a complete overview can be accomplished and all tests are
either "ok" or "skipped" and otherwise with a value < 0
* Provides a convenient interface for debugging single tests
* Creates a complete Autopilot scenario from a list of local test packages
* Optional platform dependencies can be defined for each test package
* Simple interface for selectively using test ingredients built from source
* Can be combined with a list of source packages and gcov to show test coverage
* Exits with 0 if all tests are either "ok" or "skipped", otherwise with < 0
* The target is re-booted if the scenario gets stuck
* On each re-boot the Autopilot is configured only with the remaining tests
* Tests that caused a re-boot are listed as "failed" with cause "reboot"
* The final Autopilot result overview reflects all results from all boots
* If a complete result overview can't be obtained, a partial one is simulated
Depot Autopilot component
@ -46,7 +45,7 @@ Configuration
This is an example configuration of the Depot Autopilot:
! <config arch="x86_64" children_label_prefix="dynamic -> ">
! <config arch="x86_64" children_label_prefix="dynamic -> " repeat="false">
! <static>
! <parent-provides>
! <service name="ROM"/>
@ -79,11 +78,19 @@ This is an example configuration of the Depot Autopilot:
appropriate variant of the test packages. Must be one of "x86_64", "x86_32",
"arm_v7a".
:config children_label_prefix:
:<config children_label_prefix>:
Label prefix of LOG sessions of the components of a test. This is required
to relate incoming LOG-session request to a running test.
:<config repeat>:
Can be one of
"false" - process the given test list only once,
"until_forever" - endlessly repeat processing the given test list,
"until_failed" - repeat processing the given test list until it fails.
:<config><static>:
Contains Init configuration that shall always be added to the configuration
@ -319,8 +326,65 @@ Integration
components as part of the LOG output of the Depot Autopilot.
Depot Autopilot Run script
~~~~~~~~~~~~~~~~~~~~~~~~~~
By default, the Run script tries to evaluate all tests that are internally
configured, that is, normally, all that are available as package recipes in
the main repositories. This includes one test that is prepared for the later
examination by gcov as well as gcov itself, which is executed like a test at
the end of the test list to examine the above mentioned adapted test. The test
list is processed only once and all test ingredients are used from packages,
assuming the depot user is 'genodelabs'. For this default behavior just call
the script like any "normal" Run script:
! make run/depot_autopilot KERNEL="nova"
However, this is normally a pretty time intensive process and meant for the
integration into automatic test infrastructures rather than manual evaluation.
This is why you might want to adapt the scenario, for instance, for debugging
one or more tests that failed during your automated testing. For this purpose,
a user interface is given through specific environment variables. This is an
example containing all of these variables:
! make run/depot_autopilot \
! > TEST_PKGS="test-libc_vfs test-log" \
! > TEST_SRCS="test-xml_generator test-xml_node" \
! > TEST_BUILDS="server/ram_fs test/libc_vfs" \
! > TEST_MODULES="ram_fs test-libc_vfs vfs.lib.so" \
! > TEST_REPEAT="until_failed" \
! > KERNEL="nova"
:TEST_PKGS:
List of test package-archives that supersedes the internal list if set.
:TEST_SRCS:
List of test source-archives that supersedes the internal list if set.
:TEST_BUILDS:
List of test build-targets that be build from the local repositories.
:TEST_MODULES:
List of test boot-modules that overlay the contents of the package-archives.
:TEST_REPEAT:
See the <config repeat> attribute of the Depot Autopilot.
To get a hint which build components (TEST_BUILDS) and which boot modules
(TEST_MODULES) you may want to enter for a given test package, you may have
a look at the package recipe:
! repos/<REPO>/recipes/pkg/<TEST_PKG>/archives
! repos/<REPO>/recipes/pkg/<TEST_PKG>/runtime (<content> tag)
Examples
--------
See the run script 'repos/gems/run/depot_autopilot.run' for a comprehensive
example of how to use the Depot Autopilot.
Please see the run script 'repos/gems/run/depot_autopilot.run' for a
comprehensive example of how to use the Depot Autopilot.

19
repos/gems/src/app/depot_autopilot/child.cc
View File

@ -375,6 +375,19 @@ Child::Child(Allocator &alloc,
{ }
Child::~Child()
{
while (Timeout_event *event = _timeout_events.first()) {
_timeout_events.remove(event);
destroy(_alloc, event);
}
while (Log_event *event = _log_events.first()) {
_log_events.remove(event);
destroy(_alloc, event);
}
}
void Child::log_session_write(Log_event::Line const &log_line)
{
if (_skip) {
@ -684,10 +697,14 @@ Child::State_name Child::_padded_state_name() const
}
void Child::print_conclusion()
{
log(" ", _conclusion);
}
void Child::conclusion(Result &result)
{
struct Bad_state : Exception { };
log(" ", _conclusion);
switch (_state) {
case SUCCEEDED: result.succeeded++; break;
case FAILED: result.failed++; break;

4
repos/gems/src/app/depot_autopilot/child.h
View File

@ -221,8 +221,12 @@ class Depot_deploy::Child : public List_model<Child>::Element
Timer::Connection &timer,
Genode::Signal_context_capability const &config_handler);
~Child();
void log_session_write(Log_event::Line const &log_line);
void print_conclusion();
void conclusion(Result &result);
void event_occured(Event const &event,

7
repos/gems/src/app/depot_autopilot/children.h
View File

@ -158,6 +158,13 @@ class Depot_deploy::Children
return finished;
}
void print_conclusion()
{
_children.for_each([&] (Child &child) {
child.print_conclusion();
});
}
void conclusion(Result &result)
{
_children.for_each([&] (Child &child) {

9
repos/gems/src/app/depot_autopilot/config.xsd
View File

@ -4,6 +4,14 @@
<xs:include schemaLocation="base_types.xsd"/>
<xs:include schemaLocation="timeout_types.xsd"/>
<xs:simpleType name="Repeat">
<xs:restriction base="xs:string">
<xs:enumeration value="false" />
<xs:enumeration value="until_forever" />
<xs:enumeration value="until_failed" />
</xs:restriction>
</xs:simpleType><!-- Architecture -->
<xs:simpleType name="Architecture">
<xs:restriction base="xs:string">
<xs:enumeration value="x86_64" />
@ -91,6 +99,7 @@
</xs:choice>
<xs:attribute name="arch" type="Architecture" />
<xs:attribute name="repeat" type="Repeat" />
<xs:attribute name="children_label_prefix" type="Session_label" />
<xs:attribute name="ld_verbose" type="Boolean" />
</xs:complexType>

180
repos/gems/src/app/depot_autopilot/main.cc
View File

@ -112,91 +112,133 @@ class Depot_deploy::Log_root : public Root_component<Log_session_component>
struct Depot_deploy::Main
{
typedef String<128> Name;
Env &_env;
Attached_rom_dataspace _config { _env, "config" };
Attached_rom_dataspace _blueprint { _env, "blueprint" };
Expanding_reporter _query_reporter { _env, "query" , "query"};
Expanding_reporter _init_config_reporter { _env, "config", "init.config"};
Heap _heap { _env.ram(), _env.rm() };
Reconstructible<Session_label> _children_label_prefix { };
Timer::Connection _timer { _env };
Signal_handler<Main> _config_handler { _env.ep(), *this, &Main::_handle_config };
Children _children { _heap, _timer, _config_handler };
Log_root _log_root { _env.ep(), _heap, _children, *_children_label_prefix };
void _handle_config()
struct Repeatable
{
_config.update();
_blueprint.update();
Env &_env;
Heap &_heap;
Signal_transmitter _repeat_handler;
Attached_rom_dataspace _config { _env, "config" };
Attached_rom_dataspace _blueprint { _env, "blueprint" };
Expanding_reporter _query_reporter { _env, "query" , "query"};
Expanding_reporter _init_config_reporter { _env, "config", "init.config"};
Reconstructible<Session_label> _children_label_prefix { };
Timer::Connection _timer { _env };
Signal_handler<Repeatable> _config_handler { _env.ep(), *this, &Repeatable::_handle_config };
Children _children { _heap, _timer, _config_handler };
Xml_node const config = _config.xml();
void _handle_config()
{
_config.update();
_blueprint.update();
_children_label_prefix.construct(config.attribute_value("children_label_prefix", String<160>()));
_children.apply_config(config);
_children.apply_blueprint(_blueprint.xml());
Xml_node const config = _config.xml();
/* determine CPU architecture of deployment */
typedef String<16> Arch;
Arch const arch = config.attribute_value("arch", Arch());
if (!arch.valid())
warning("config lacks 'arch' attribute");
_children_label_prefix.construct(config.attribute_value("children_label_prefix", String<160>()));
_children.apply_config(config);
_children.apply_blueprint(_blueprint.xml());
/* generate init config containing all configured start nodes */
bool finished;
_init_config_reporter.generate([&] (Xml_generator &xml) {
Xml_node static_config = config.sub_node("static");
xml.append(static_config.content_base(), static_config.content_size());
Child::Depot_rom_server const parent { };
finished = _children.gen_start_nodes(xml, config.sub_node("common_routes"),
parent, parent);
});
if (finished) {
/* determine CPU architecture of deployment */
typedef String<16> Arch;
Arch const arch = config.attribute_value("arch", Arch());
if (!arch.valid())
warning("config lacks 'arch' attribute");
Result result;
unsigned long previous_time_sec { 0UL };
if (config.has_sub_node("previous-results")) {
Xml_node const previous_results = config.sub_node("previous-results");
previous_time_sec += previous_results.attribute_value("time_sec", 0UL);
result.succeeded += previous_results.attribute_value("succeeded", 0UL);
result.failed += previous_results.attribute_value("failed", 0UL);
result.skipped += previous_results.attribute_value("skipped", 0UL);
}
unsigned long const time_us { _timer.curr_time().trunc_to_plain_us().value };
unsigned long time_ms { time_us / 1000UL };
unsigned long const time_sec { time_ms / 1000UL };
time_ms = time_ms - time_sec * 1000UL;
/* generate init config containing all configured start nodes */
bool finished;
_init_config_reporter.generate([&] (Xml_generator &xml) {
Xml_node static_config = config.sub_node("static");
xml.append(static_config.content_base(), static_config.content_size());
Child::Depot_rom_server const parent { };
finished = _children.gen_start_nodes(xml, config.sub_node("common_routes"),
parent, parent);
});
if (finished) {
log("\n--- Finished after ", time_sec + previous_time_sec, ".", time_ms < 10 ? "00" : time_ms < 100 ? "0" : "", time_ms, " sec ---\n");
if (config.has_sub_node("previous-results")) {
Xml_node const previous_results = config.sub_node("previous-results");
if (previous_results.content_size()) {
log(Cstring(previous_results.content_base(), previous_results.content_size()));
Result result;
unsigned long previous_time_sec { 0UL };
if (config.has_sub_node("previous-results")) {
Xml_node const previous_results = config.sub_node("previous-results");
previous_time_sec += previous_results.attribute_value("time_sec", 0UL);
result.succeeded += previous_results.attribute_value("succeeded", 0UL);
result.failed += previous_results.attribute_value("failed", 0UL);
result.skipped += previous_results.attribute_value("skipped", 0UL);
}
unsigned long const time_us { _timer.curr_time().trunc_to_plain_us().value };
unsigned long time_ms { time_us / 1000UL };
unsigned long const time_sec { time_ms / 1000UL };
time_ms = time_ms - time_sec * 1000UL;
_children.conclusion(result);
int exit_code = result.failed ? -1 : 0;
typedef String<12> Repeat;
Repeat repeat = config.attribute_value("repeat", Repeat("false"));
if (repeat == Repeat("until_forever") ||
(repeat == Repeat("until_failed") && exit_code == 0)) {
char const *empty_config_str = "<config/>";
Xml_node const empty_config(empty_config_str, 10);
_children.apply_config(empty_config);
_query_reporter.generate([&] (Xml_generator &xml) { xml.attribute("arch", arch); });
_init_config_reporter.generate([&] (Xml_generator &) { });
_repeat_handler.submit();
return;
} else {
log("\n--- Finished after ", time_sec + previous_time_sec, ".", time_ms < 10 ? "00" : time_ms < 100 ? "0" : "", time_ms, " sec ---\n");
if (config.has_sub_node("previous-results")) {
Xml_node const previous_results = config.sub_node("previous-results");
if (previous_results.content_size()) {
log(Cstring(previous_results.content_base(), previous_results.content_size()));
}
}
_children.print_conclusion();
log("");
log(result);
log("");
_env.parent().exit(exit_code);
}
}
_children.conclusion(result);
log("");
log(result);
log("");
_env.parent().exit(result.failed ? -1 : 0);
/* update query for blueprints of all unconfigured start nodes */
if (arch.valid()) {
_query_reporter.generate([&] (Xml_generator &xml) {
xml.attribute("arch", arch);
_children.gen_queries(xml);
});
}
}
/* update query for blueprints of all unconfigured start nodes */
if (arch.valid()) {
_query_reporter.generate([&] (Xml_generator &xml) {
xml.attribute("arch", arch);
_children.gen_queries(xml);
});
Repeatable(Env &env,
Signal_context_capability const &repeat_handler,
Heap &heap)
:
_env(env),
_heap(heap),
_repeat_handler(repeat_handler)
{
_config .sigh(_config_handler);
_blueprint.sigh(_config_handler);
_handle_config();
}
};
Env &_env;
Signal_handler<Main> _repeat_handler { _env.ep(), *this, &Main::_handle_repeat };
Heap _heap { _env.ram(), _env.rm() };
Reconstructible<Repeatable> _repeatable { _env, _repeat_handler, _heap };
Log_root _log_root { _env.ep(), _heap, _repeatable->_children, *_repeatable->_children_label_prefix };
void _handle_repeat()
{
_repeatable.construct(_env, _repeat_handler, _heap);
}
Main(Env &env) : _env(env)
{
_config .sigh(_config_handler);
_blueprint.sigh(_config_handler);
_handle_config();
_env.parent().announce(_env.ep().manage(_log_root));
}
};