ehmry/genode
ehmry
/
genode
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

depot_download: support downloading index files 

With this commit, the 'installation' input of the depot-download
subsystem accepts <index> nodes in addition to <archive> nodes. Each
index node refers to one index file specified via the 'path' attribute.

This commit also improves the tracking of failure states. Once an
installation job failed (due to a download of verification error),
it won't get re-scheduled. In the past, such failure states were not kept
across subsequent import iterations, which could result in infinite
re-attempts when an installation contained archives from multiple users.
The the progress of the download process is now reflected by the
"progress" attribute on the download manager's state report, which
allows the final report to contain the list of installed/failed archives
along with the overall progress/completed state. The detection of the
latter is important for the sculpt manager for reattempting the
deployment of the completed packages.

The patch enhances the depot_download.run script to stress the new
abilities. In particular, the scenario downloads a mix of index files
(one present, one missing) and archives, from two different depot users
(genodelabs and nfeske).

Issue #3172
This commit is contained in:
Norman Feske 2019-02-20 09:51:55 +01:00 committed by Christian Helmuth
parent 1b518965cc
commit ac68073ffc
12 changed files with 357 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
repos/gems/include/depot/archive.h
View File

@ -88,8 +88,30 @@ struct Depot::Archive
throw Unknown_archive_type(); throw Unknown_archive_type();
} }
static Name name (Path const &path) { return _path_element<Name>(path, 2); } /**
static Version version(Path const &path) { return _path_element<Name>(path, 3); } * Return true if 'path' refers to an index file
*/
static bool index(Path const &path)
{
return _path_element<Name>(path, 1) == "index";
}
static Name name (Path const &path) { return _path_element<Name>(path, 2); }
static Version version (Path const &path) { return _path_element<Version>(path, 3); }
static Version index_version(Path const &path) { return _path_element<Version>(path, 2); }
/**
* Return name of compressed file to download for the given depot path
*
* Archives are shipped as tar.xz files whereas index files are shipped
* as xz-compressed files.
*/
static Archive::Path download_file_path(Archive::Path path)
{
return Archive::index(path) ? Archive::Path(path, ".xz")
: Archive::Path(path, ".tar.xz");
}
}; };
#endif /* _INCLUDE__DEPOT__ARCHIVE_H_ */ #endif /* _INCLUDE__DEPOT__ARCHIVE_H_ */

3
repos/gems/recipes/raw/depot_download/depot_download.config
View File

@ -26,6 +26,8 @@
report="manager -> init_config"/> report="manager -> init_config"/>
<policy label="manager -> dependencies" <policy label="manager -> dependencies"
report="dynamic -> depot_query -> dependencies"/> report="dynamic -> depot_query -> dependencies"/>
<policy label="manager -> index"
report="dynamic -> depot_query -> index"/>
<policy label="manager -> user" <policy label="manager -> user"
report="dynamic -> depot_query -> user"/> report="dynamic -> depot_query -> user"/>
<policy label="manager -> init_state" <policy label="manager -> init_state"
@ -82,6 +84,7 @@
<service name="Report" label="state"> <parent label="state"/> </service> <service name="Report" label="state"> <parent label="state"/> </service>
<service name="Report"> <child name="report_rom"/> </service> <service name="Report"> <child name="report_rom"/> </service>
<service name="ROM" label="dependencies"> <child name="report_rom"/> </service> <service name="ROM" label="dependencies"> <child name="report_rom"/> </service>
<service name="ROM" label="index"> <child name="report_rom"/> </service>
<service name="ROM" label="user"> <child name="report_rom"/> </service> <service name="ROM" label="user"> <child name="report_rom"/> </service>
<service name="ROM" label="init_state"> <child name="report_rom"/> </service> <service name="ROM" label="init_state"> <child name="report_rom"/> </service>
<service name="ROM" label="verified"> <child name="report_rom"/> </service> <service name="ROM" label="verified"> <child name="report_rom"/> </service>

37
repos/gems/run/depot_download.run
View File

@ -51,11 +51,11 @@ set config {
append_platform_drv_config append_platform_drv_config
proc depot_user_download { } { proc depot_user_download { user } {
return [exec cat [genode_dir]/depot/[depot_user]/download] } return [exec cat [genode_dir]/depot/$user/download] }
proc depot_user_pubkey { } { proc depot_user_pubkey { user } {
return [exec cat [genode_dir]/depot/[depot_user]/pubkey] } return [exec cat [genode_dir]/depot/$user/pubkey] }
append config { append config {
<start name="timer"> <start name="timer">
@ -69,10 +69,15 @@ append config {
<config> <config>
<vfs> <vfs>
<dir name="depot"> <dir name="depot">
<dir name="} [depot_user] {"> <dir name="nfeske">
<ram/> <ram/>
<inline name="download">} [depot_user_download] {</inline> <inline name="download">} [depot_user_download nfeske] {</inline>
<inline name="pubkey">} [depot_user_pubkey] {</inline> <inline name="pubkey">} [depot_user_pubkey nfeske] {</inline>
</dir>
<dir name="genodelabs">
<ram/>
<inline name="download">} [depot_user_download genodelabs] {</inline>
<inline name="pubkey">} [depot_user_pubkey genodelabs] {</inline>
</dir> </dir>
</dir> </dir>
<dir name="public"> <ram/> </dir> <dir name="public"> <ram/> </dir>
@ -109,10 +114,12 @@ append config {
set fd [open [run_dir]/genode/installation w] set fd [open [run_dir]/genode/installation w]
puts $fd " puts $fd {
<installation arch=\"x86_64\"> <installation arch="x86_64">
<archive path=\"[depot_user]/pkg/wm/2018-02-26\"/> <archive path="genodelabs/pkg/wm/2018-02-26"/>
</installation>" <index path="nfeske/index/19.02"/>
<index path="nfeske/index/19.03"/>
</installation>}
close $fd close $fd
@ -129,5 +136,11 @@ build_boot_image $boot_modules
append qemu_args " -nographic -net nic,model=e1000 -net user " append qemu_args " -nographic -net nic,model=e1000 -net user "
run_genode_until {.*\[init -> depot_download -> manager\] installation complete.*\n} 150 # watch the state reports generated by the depot-download manager
set expected_pattern {}
append expected_pattern {.*path="genodelabs/pkg/wm/2018-02-26" state="done".*}
append expected_pattern {.*path="nfeske/index/19.02" state="done".*}
append expected_pattern {.*path="nfeske/index/19.03" state="failed".*}
run_genode_until $expected_pattern 150

40
repos/gems/src/app/depot_download_manager/gen_depot_query.cc
View File

@ -16,7 +16,8 @@
void Depot_download_manager::gen_depot_query_start_content(Xml_generator &xml, void Depot_download_manager::gen_depot_query_start_content(Xml_generator &xml,
Xml_node installation, Xml_node installation,
Archive::User const &next_user, Archive::User const &next_user,
Depot_query_version version) Depot_query_version version,
List_model<Job> const &jobs)
{ {
gen_common_start_content(xml, "depot_query", gen_common_start_content(xml, "depot_query",
Cap_quota{100}, Ram_quota{2*1024*1024}); Cap_quota{100}, Ram_quota{2*1024*1024});
@ -33,7 +34,28 @@ void Depot_download_manager::gen_depot_query_start_content(Xml_generator &xml,
}); });
}); });
/*
* Filter out failed parts of the installation from being re-queried.
* The inclusion of those parts may otherwise result in an infinite
* loop if the installation is downloaded from a mix of depot users.
*/
auto job_failed = [&] (Xml_node node)
{
Archive::Path const path = node.attribute_value("path", Archive::Path());
bool failed = false;
jobs.for_each([&] (Job const &job) {
if (job.path == path && job.failed)
failed = true; });
return failed;
};
installation.for_each_sub_node("archive", [&] (Xml_node archive) { installation.for_each_sub_node("archive", [&] (Xml_node archive) {
if (job_failed(archive))
return;
xml.node("dependencies", [&] () { xml.node("dependencies", [&] () {
xml.attribute("path", archive.attribute_value("path", Archive::Path())); xml.attribute("path", archive.attribute_value("path", Archive::Path()));
xml.attribute("source", archive.attribute_value("source", true)); xml.attribute("source", archive.attribute_value("source", true));
@ -41,6 +63,22 @@ void Depot_download_manager::gen_depot_query_start_content(Xml_generator &xml,
}); });
}); });
installation.for_each_sub_node("index", [&] (Xml_node index) {
if (job_failed(index))
return;
xml.node("index", [&] () {
Archive::Path const path = index.attribute_value("path", Archive::Path());
if (!Archive::index(path)) {
warning("malformed index path '", path, "'");
return;
}
xml.attribute("user", Archive::user(path));
xml.attribute("version", Archive::_path_element<Archive::Version>(path, 2));
});
});
if (next_user.valid()) if (next_user.valid())
xml.node("user", [&] () { xml.attribute("name", next_user); }); xml.node("user", [&] () { xml.attribute("name", next_user); });
}); });

12
repos/gems/src/app/depot_download_manager/gen_extract.cc
View File

@ -54,15 +54,13 @@ void Depot_download_manager::gen_extract_start_content(Xml_generator &xml,
import.for_each_verified_archive([&] (Archive::Path const &path) { import.for_each_verified_archive([&] (Archive::Path const &path) {
typedef String<160> Path; typedef String<160> Path;
typedef String<16> Ext;
Ext const ext (".tar.xz");
Path const tar_path ("/public/", path, ext);
Path const dst_path ("/depot/", without_last_path_element(path));
xml.node("extract", [&] () { xml.node("extract", [&] () {
xml.attribute("archive", tar_path); xml.attribute("archive", Path("/public/", Archive::download_file_path(path)));
xml.attribute("to", dst_path); xml.attribute("to", Path("/depot/", without_last_path_element(path)));
if (Archive::index(path))
xml.attribute("name", Archive::index_version(path));
}); });
}); });
}); });

8
repos/gems/src/app/depot_download_manager/gen_fetchurl.cc
View File

@ -58,11 +58,11 @@ void Depot_download_manager::gen_fetchurl_start_content(Xml_generator &xml,
import.for_each_download([&] (Archive::Path const &path) { import.for_each_download([&] (Archive::Path const &path) {
typedef String<160> Remote; typedef String<160> Remote;
typedef String<160> Local; typedef String<160> Local;
typedef String<16> Ext; typedef String<100> File_path;
Ext const ext (".tar.xz"); File_path const file_path = Archive::download_file_path(path);
Remote const remote (current_user_url, "/", path, ext); Remote const remote (current_user_url, "/", file_path);
Local const local ("/download/", path, ext); Local const local ("/download/", file_path);
xml.node("fetch", [&] () { xml.node("fetch", [&] () {
xml.attribute("url", remote); xml.attribute("url", remote);

6
repos/gems/src/app/depot_download_manager/gen_verify.cc
View File

@ -48,14 +48,12 @@ void Depot_download_manager::gen_verify_start_content(Xml_generator &xml,
import.for_each_unverified_archive([&] (Archive::Path const &path) { import.for_each_unverified_archive([&] (Archive::Path const &path) {
typedef String<160> Path; typedef String<160> Path;
typedef String<16> Ext;
Ext const ext (".tar.xz"); Path const file_path ("/public/", Archive::download_file_path(path));
Path const tar_path ("/public/", path, ext);
Path const pubkey_path (user_path, "/pubkey"); Path const pubkey_path (user_path, "/pubkey");
xml.node("verify", [&] () { xml.node("verify", [&] () {
xml.attribute("path", tar_path); xml.attribute("path", file_path);
xml.attribute("pubkey", pubkey_path); xml.attribute("pubkey", pubkey_path);
}); });
}); });

85
repos/gems/src/app/depot_download_manager/import.h
View File

@ -41,6 +41,15 @@ class Depot_download_manager::Import
{ {
typedef String<32> Bytes; typedef String<32> Bytes;
Bytes total, now; Bytes total, now;
bool complete() const
{
/* fetchurl has not yet determined the file size */
if (total == "0.0")
return false;
return now == total;
}
}; };
virtual Info download_progress(Archive::Path const &) const = 0; virtual Info download_progress(Archive::Path const &) const = 0;
@ -57,6 +66,7 @@ class Depot_download_manager::Import
enum State { DOWNLOAD_IN_PROGRESS, enum State { DOWNLOAD_IN_PROGRESS,
DOWNLOAD_COMPLETE, DOWNLOAD_COMPLETE,
DOWNLOAD_UNAVAILABLE, DOWNLOAD_UNAVAILABLE,
VERIFICATION_IN_PROGRESS,
VERIFIED, VERIFIED,
VERIFICATION_FAILED, VERIFICATION_FAILED,
UNPACKED }; UNPACKED };
@ -71,12 +81,13 @@ class Depot_download_manager::Import
char const *state_text() const char const *state_text() const
{ {
switch (state) { switch (state) {
case DOWNLOAD_IN_PROGRESS: return "download"; case DOWNLOAD_IN_PROGRESS: return "download";
case DOWNLOAD_COMPLETE: return "verify"; case DOWNLOAD_COMPLETE: return "fetched";
case DOWNLOAD_UNAVAILABLE: return "unavailable"; case DOWNLOAD_UNAVAILABLE: return "unavailable";
case VERIFIED: return "extract"; case VERIFICATION_IN_PROGRESS: return "verify";
case VERIFICATION_FAILED: return "verification failed"; case VERIFIED: return "extract";
case UNPACKED: return "done"; case VERIFICATION_FAILED: return "corrupted";
case UNPACKED: return "done";
}; };
return ""; return "";
} }
@ -111,22 +122,35 @@ class Depot_download_manager::Import
/** /**
* Constructor * Constructor
* *
* \param user depot origin to use for the import * \param user depot origin to use for the import
* \param node XML node containing any number of '<missing>' sub nodes * \param dependencies information about '<missing>' archives
* \param index information about '<missing>' index files
* *
* The import constructor considers only those '<missing>' sub nodes as * The import constructor considers only those '<missing>' sub nodes as
* items that match the 'user'. The remaining sub nodes are imported in * items that match the 'user'. The remaining sub nodes are imported in
* a future iteration. * a future iteration.
*/ */
Import(Allocator &alloc, Archive::User const &user, Xml_node node) Import(Allocator &alloc, Archive::User const &user,
Xml_node dependencies, Xml_node index)
: :
_alloc(alloc) _alloc(alloc)
{ {
node.for_each_sub_node("missing", [&] (Xml_node item) { dependencies.for_each_sub_node("missing", [&] (Xml_node item) {
Archive::Path const path = item.attribute_value("path", Archive::Path()); Archive::Path const path = item.attribute_value("path", Archive::Path());
if (Archive::user(path) == user) if (Archive::user(path) == user)
new (alloc) Item(_items, path); new (alloc) Item(_items, path);
}); });
index.for_each_sub_node("missing", [&] (Xml_node item) {
Archive::Path const
path(item.attribute_value("user", Archive::User()),
"/index/",
item.attribute_value("version", Archive::Version()));
if (Archive::user(path) == user)
new (alloc) Item(_items, path);
});
} }
~Import() ~Import()
@ -139,11 +163,16 @@ class Depot_download_manager::Import
return _item_state_exists(Item::DOWNLOAD_IN_PROGRESS); return _item_state_exists(Item::DOWNLOAD_IN_PROGRESS);
} }
bool unverified_archives_available() const bool completed_downloads_available() const
{ {
return _item_state_exists(Item::DOWNLOAD_COMPLETE); return _item_state_exists(Item::DOWNLOAD_COMPLETE);
} }
bool unverified_archives_available() const
{
return _item_state_exists(Item::VERIFICATION_IN_PROGRESS);
}
bool verified_archives_available() const bool verified_archives_available() const
{ {
return _item_state_exists(Item::VERIFIED); return _item_state_exists(Item::VERIFIED);
@ -158,7 +187,7 @@ class Depot_download_manager::Import
template <typename FN> template <typename FN>
void for_each_unverified_archive(FN const &fn) const void for_each_unverified_archive(FN const &fn) const
{ {
_for_each_item(Item::DOWNLOAD_COMPLETE, fn); _for_each_item(Item::VERIFICATION_IN_PROGRESS, fn);
} }
template <typename FN> template <typename FN>
@ -173,6 +202,13 @@ class Depot_download_manager::Import
_for_each_item(Item::UNPACKED, fn); _for_each_item(Item::UNPACKED, fn);
} }
template <typename FN>
void for_each_failed_archive(FN const &fn) const
{
_for_each_item(Item::DOWNLOAD_UNAVAILABLE, fn);
_for_each_item(Item::VERIFICATION_FAILED, fn);
}
void all_downloads_completed() void all_downloads_completed()
{ {
_items.for_each([&] (Item &item) { _items.for_each([&] (Item &item) {
@ -180,7 +216,26 @@ class Depot_download_manager::Import
item.state = Item::DOWNLOAD_COMPLETE; }); item.state = Item::DOWNLOAD_COMPLETE; });
} }
void all_downloads_unavailable() void verify_all_downloaded_archives()
{
_items.for_each([&] (Item &item) {
if (item.state == Item::DOWNLOAD_COMPLETE)
item.state = Item::VERIFICATION_IN_PROGRESS; });
}
void apply_download_progress(Download_progress const &progress)
{
_items.for_each([&] (Item &item) {
if (item.state == Item::DOWNLOAD_IN_PROGRESS
&& progress.download_progress(item.path).complete()) {
item.state = Item::DOWNLOAD_COMPLETE;
}
});
}
void all_remaining_downloads_unavailable()
{ {
_items.for_each([&] (Item &item) { _items.for_each([&] (Item &item) {
if (item.state == Item::DOWNLOAD_IN_PROGRESS) if (item.state == Item::DOWNLOAD_IN_PROGRESS)
@ -190,7 +245,7 @@ class Depot_download_manager::Import
void archive_verified(Archive::Path const &archive) void archive_verified(Archive::Path const &archive)
{ {
_items.for_each([&] (Item &item) { _items.for_each([&] (Item &item) {
if (item.state == Item::DOWNLOAD_COMPLETE) if (item.state == Item::VERIFICATION_IN_PROGRESS)
if (item.path == archive) if (item.path == archive)
item.state = Item::VERIFIED; }); item.state = Item::VERIFIED; });
} }
@ -198,7 +253,7 @@ class Depot_download_manager::Import
void archive_verification_failed(Archive::Path const &archive) void archive_verification_failed(Archive::Path const &archive)
{ {
_items.for_each([&] (Item &item) { _items.for_each([&] (Item &item) {
if (item.state == Item::DOWNLOAD_COMPLETE) if (item.state == Item::VERIFICATION_IN_PROGRESS)
if (item.path == archive) if (item.path == archive)
item.state = Item::VERIFICATION_FAILED; }); item.state = Item::VERIFICATION_FAILED; });
} }

65
repos/gems/src/app/depot_download_manager/job.h Normal file
View File

@ -0,0 +1,65 @@
/*
* \brief Failure state of jobs submitted via the 'installation'
* \author Norman Feske
* \date 2019-02-21
*/
/*
* Copyright (C) 2019 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
#ifndef _JOB_H_
#define _JOB_H_
/* Genode includes */
#include <util/list_model.h>
#include <base/allocator.h>
#include "types.h"
namespace Depot_download_manager {
using namespace Depot;
struct Job;
}
struct Depot_download_manager::Job : List_model<Job>::Element
{
bool started = false;
bool failed = false;
Archive::Path const path;
Job(Archive::Path const &path) : path(path) { }
struct Update_policy
{
typedef Job Element;
Allocator &_alloc;
Update_policy(Allocator &alloc) : _alloc(alloc) { }
void destroy_element(Job &elem) { destroy(_alloc, &elem); }
Job &create_element(Xml_node elem_node)
{
return *new (_alloc)
Job(elem_node.attribute_value("path", Archive::Path()));
}
void update_element(Job &, Xml_node) { }
static bool element_matches_xml_node(Job const &job, Xml_node node)
{
return node.attribute_value("path", Archive::Path()) == job.path;
}
static bool node_is_element(Xml_node) { return true; }
};
};
#endif /* _JOB_H_ */

119
repos/gems/src/app/depot_download_manager/main.cc
View File

@ -58,6 +58,7 @@ struct Depot_download_manager::Main : Import::Download_progress
Attached_rom_dataspace _installation { _env, "installation" }; Attached_rom_dataspace _installation { _env, "installation" };
Attached_rom_dataspace _dependencies { _env, "dependencies" }; Attached_rom_dataspace _dependencies { _env, "dependencies" };
Attached_rom_dataspace _index { _env, "index" };
Attached_rom_dataspace _init_state { _env, "init_state" }; Attached_rom_dataspace _init_state { _env, "init_state" };
Attached_rom_dataspace _fetchurl_progress { _env, "fetchurl_progress" }; Attached_rom_dataspace _fetchurl_progress { _env, "fetchurl_progress" };
@ -104,6 +105,8 @@ struct Depot_download_manager::Main : Import::Download_progress
Archive::User _next_user { }; Archive::User _next_user { };
List_model<Job> _jobs { };
Constructible<Import> _import { }; Constructible<Import> _import { };
/** /**
@ -113,7 +116,7 @@ struct Depot_download_manager::Main : Import::Download_progress
{ {
Info result { Info::Bytes(), Info::Bytes() }; Info result { Info::Bytes(), Info::Bytes() };
try { try {
Url const url_path(_current_user_url(), "/", path, ".tar.xz"); Url const url_path(_current_user_url(), "/", Archive::download_file_path(path));
/* search fetchurl progress report for matching 'url_path' */ /* search fetchurl progress report for matching 'url_path' */
_fetchurl_progress.xml().for_each_sub_node("fetch", [&] (Xml_node fetch) { _fetchurl_progress.xml().for_each_sub_node("fetch", [&] (Xml_node fetch) {
@ -128,8 +131,32 @@ struct Depot_download_manager::Main : Import::Download_progress
void _update_state_report() void _update_state_report()
{ {
_state_reporter.generate([&] (Xml_generator &xml) { _state_reporter.generate([&] (Xml_generator &xml) {
if (_import.constructed())
_import->report(xml, *this); }); /* produce detailed reports while the installation is in progress */
if (_import.constructed()) {
xml.attribute("progress", "yes");
_import->report(xml, *this);
}
/* once all imports have settled, present the final results */
else {
_jobs.for_each([&] (Job const &job) {
if (!job.started)
return;
/*
* If a job has been started and has not failed, it must
* have succeeded at the time when the import is finished.
*/
char const *type = Archive::index(job.path) ? "index" : "archive";
xml.node(type, [&] () {
xml.attribute("path", job.path);
xml.attribute("state", job.failed ? "failed" : "done");
});
});
}
});
} }
void _generate_init_config(Xml_generator &); void _generate_init_config(Xml_generator &);
@ -143,6 +170,10 @@ struct Depot_download_manager::Main : Import::Download_progress
void _handle_installation() void _handle_installation()
{ {
_installation.update(); _installation.update();
Job::Update_policy policy(_heap);
_jobs.update_from_xml(policy, _installation.xml());
_generate_init_config(); _generate_init_config();
} }
@ -165,18 +196,29 @@ struct Depot_download_manager::Main : Import::Download_progress
void _handle_fetchurl_progress() void _handle_fetchurl_progress()
{ {
_fetchurl_progress.update(); _fetchurl_progress.update();
if (_import.constructed()) {
_import->apply_download_progress(*this);
/* proceed with next import step if all downloads are done or failed */
if (!_import->downloads_in_progress())
_generate_init_config();
}
_update_state_report(); _update_state_report();
} }
Main(Env &env) : _env(env) Main(Env &env) : _env(env)
{ {
_dependencies .sigh(_query_result_handler); _dependencies .sigh(_query_result_handler);
_index .sigh(_query_result_handler);
_current_user .sigh(_query_result_handler); _current_user .sigh(_query_result_handler);
_init_state .sigh(_init_state_handler); _init_state .sigh(_init_state_handler);
_verified .sigh(_init_state_handler); _verified .sigh(_init_state_handler);
_installation .sigh(_installation_handler); _installation .sigh(_installation_handler);
_fetchurl_progress.sigh(_fetchurl_progress_handler); _fetchurl_progress.sigh(_fetchurl_progress_handler);
_handle_installation();
_generate_init_config(); _generate_init_config();
} }
}; };
@ -223,7 +265,7 @@ void Depot_download_manager::Main::_generate_init_config(Xml_generator &xml)
xml.node("start", [&] () { xml.node("start", [&] () {
gen_depot_query_start_content(xml, _installation.xml(), gen_depot_query_start_content(xml, _installation.xml(),
_next_user, _depot_query_count); }); _next_user, _depot_query_count, _jobs); });
if (_import.constructed() && _import->downloads_in_progress()) { if (_import.constructed() && _import->downloads_in_progress()) {
try { try {
@ -259,23 +301,54 @@ void Depot_download_manager::Main::_handle_query_result()
return; return;
_dependencies.update(); _dependencies.update();
_index.update();
_current_user.update(); _current_user.update();
Xml_node const dependencies = _dependencies.xml(); Xml_node const dependencies = _dependencies.xml();
if (dependencies.num_sub_nodes() == 0) Xml_node const index = _index.xml();
if (dependencies.num_sub_nodes() == 0 && index.num_sub_nodes() == 0)
return; return;
if (!dependencies.has_sub_node("missing")) { bool const missing_dependencies = dependencies.has_sub_node("missing");
bool const missing_index_files = index.has_sub_node("missing");
if (!missing_dependencies && !missing_index_files) {
log("installation complete."); log("installation complete.");
_update_state_report(); _update_state_report();
return; return;
} }
Archive::Path const path = /**
dependencies.sub_node("missing").attribute_value("path", Archive::Path()); * Select depot user for next import
*
* Prefer the downloading of index files over archives because index files
* are quick to download and important for interactivity.
*/
auto select_next_user = [&] ()
{
Archive::User user { };
if (Archive::user(path) != _current_user_name()) { if (missing_index_files)
_next_user = Archive::user(path); index.with_sub_node("missing", [&] (Xml_node missing) {
user = missing.attribute_value("user", Archive::User()); });
if (user.valid())
return user;
dependencies.with_sub_node("missing", [&] (Xml_node missing) {
user = Archive::user(missing.attribute_value("path", Archive::Path())); });
if (!user.valid())
warning("unable to select depot user for next import");
return user;
};
Archive::User const next_user = select_next_user();
if (next_user != _current_user_name()) {
_next_user = next_user;
/* query user info from 'depot_query' */ /* query user info from 'depot_query' */
_generate_init_config(); _generate_init_config();
@ -283,7 +356,14 @@ void Depot_download_manager::Main::_handle_query_result()
} }
/* start new import */ /* start new import */
_import.construct(_heap, _current_user_name(), _dependencies.xml()); _import.construct(_heap, _current_user_name(), dependencies, index);
/* mark imported jobs as started */
_import->for_each_download([&] (Archive::Path const &path) {
_jobs.for_each([&] (Job &job) {
if (job.path == path)
job.started = true; }); });
_fetchurl_attempt = 0; _fetchurl_attempt = 0;
_update_state_report(); _update_state_report();
@ -316,7 +396,7 @@ void Depot_download_manager::Main::_handle_init_state()
_fetchurl_count.value++; _fetchurl_count.value++;
if (_fetchurl_attempt++ >= _fetchurl_max_attempts) { if (_fetchurl_attempt++ >= _fetchurl_max_attempts) {
import.all_downloads_unavailable(); import.all_remaining_downloads_unavailable();
_fetchurl_attempt = 0; _fetchurl_attempt = 0;
} }
@ -326,11 +406,16 @@ void Depot_download_manager::Main::_handle_init_state()
if (fetchurl_state.exited && fetchurl_state.code == 0) { if (fetchurl_state.exited && fetchurl_state.code == 0) {
import.all_downloads_completed(); import.all_downloads_completed();
/* kill fetchurl, start untar */ /* kill fetchurl, start verify */
reconfigure_init = true; reconfigure_init = true;
} }
} }
if (!import.downloads_in_progress() && import.completed_downloads_available()) {
import.verify_all_downloaded_archives();
reconfigure_init = true;
}
if (import.unverified_archives_available()) { if (import.unverified_archives_available()) {
_verified.xml().for_each_sub_node([&] (Xml_node node) { _verified.xml().for_each_sub_node([&] (Xml_node node) {
@ -341,7 +426,7 @@ void Depot_download_manager::Main::_handle_init_state()
/* determine matching archive path */ /* determine matching archive path */
Path path; Path path;
import.for_each_unverified_archive([&] (Archive::Path const &archive) { import.for_each_unverified_archive([&] (Archive::Path const &archive) {
if (abs_path == Path("/public/", archive, ".tar.xz")) if (abs_path == Path("/public/", Archive::download_file_path(archive)))
path = archive; }); path = archive; });
if (path.valid()) { if (path.valid()) {
@ -375,6 +460,12 @@ void Depot_download_manager::Main::_handle_init_state()
} }
} }
/* flag failed jobs to prevent re-attempts in subsequent import iterations */
import.for_each_failed_archive([&] (Archive::Path const &path) {
_jobs.for_each([&] (Job &job) {
if (job.path == path)
job.failed = true; }); });
/* report before destructing '_import' to avoid empty intermediate reports */ /* report before destructing '_import' to avoid empty intermediate reports */
if (reconfigure_init) if (reconfigure_init)
_update_state_report(); _update_state_report();

4
repos/gems/src/app/depot_download_manager/xml.h
View File

@ -27,6 +27,7 @@
/* local includes */ /* local includes */
#include "import.h" #include "import.h"
#include "job.h"
namespace Depot_download_manager { namespace Depot_download_manager {
@ -84,7 +85,8 @@ namespace Depot_download_manager {
void gen_depot_query_start_content(Xml_generator &, void gen_depot_query_start_content(Xml_generator &,
Xml_node installation, Xml_node installation,
Archive::User const &, Archive::User const &,
Depot_query_version); Depot_query_version,
List_model<Job> const &);
void gen_fetchurl_start_content(Xml_generator &, Import const &, Url const &, void gen_fetchurl_start_content(Xml_generator &, Import const &, Url const &,
Fetchurl_version); Fetchurl_version);

18
repos/gems/src/app/sculpt_manager/main.cc
View File

@ -173,10 +173,16 @@ struct Sculpt::Main : Input_event_handler,
Signal_handler<Main> _update_state_handler { Signal_handler<Main> _update_state_handler {
_env.ep(), *this, &Main::_handle_update_state }; _env.ep(), *this, &Main::_handle_update_state };
bool _update_running() const { return _storage._sculpt_partition.valid() /**
&& !_prepare_in_progress() * Condition for spawning the update subsystem
&& _network.ready() */
&& _deploy.update_needed(); }; bool _update_running() const
{
return _storage._sculpt_partition.valid()
&& !_prepare_in_progress()
&& _network.ready()
&& _deploy.update_needed();
}
/************ /************
@ -281,7 +287,7 @@ struct Sculpt::Main : Input_event_handler,
_deploy.gen_child_diagnostics(xml); _deploy.gen_child_diagnostics(xml);
Xml_node const state = _update_state_rom.xml(); Xml_node const state = _update_state_rom.xml();
if (_update_running() && state.has_sub_node("archive")) if (_update_running() && state.attribute_value("progress", false))
gen_download_status(xml, state); gen_download_status(xml, state);
}); });
}); });
@ -809,7 +815,7 @@ void Sculpt::Main::_handle_update_state()
generate_dialog(); generate_dialog();
bool const installation_complete = bool const installation_complete =
!_update_state_rom.xml().has_sub_node("archive"); !_update_state_rom.xml().attribute_value("progress", false);
if (installation_complete) if (installation_complete)
_deploy.reattempt_after_installation(); _deploy.reattempt_after_installation();