From 854a154fb4c4b5ac76897a175182dfae579e3631 Mon Sep 17 00:00:00 2001 From: Norman Feske Date: Sat, 22 Jun 2019 14:27:38 +0200 Subject: [PATCH] depot_query: cache file-system accesses This patch introduces two caches to the depot-query tool. - A stat cache remembers the results of 'Directory::file_exists' calls. - The 'Cached_rom_query' caches the result of scanning the depot for a given ROM module and pkg path. To elminates the need to parse 'archive' files of pkgs referenced from other pkgs or for the repeated instantation of the same pkg. Both caches are bypassed whenever referring to the 'local' depot user. Fixes #3427 --- repos/gems/recipes/src/depot_query/content.mk | 2 +- repos/gems/src/app/depot_query/main.cc | 209 ++++++++++++++++-- 2 files changed, 192 insertions(+), 19 deletions(-) diff --git a/repos/gems/recipes/src/depot_query/content.mk b/repos/gems/recipes/src/depot_query/content.mk index 22ffb5017..19ba54374 100644 --- a/repos/gems/recipes/src/depot_query/content.mk +++ b/repos/gems/recipes/src/depot_query/content.mk @@ -2,7 +2,7 @@ SRC_DIR := src/app/depot_query include $(GENODE_DIR)/repos/base/recipes/src/content.inc -MIRROR_FROM_REP_DIR := include/depot include/gems/vfs.h +MIRROR_FROM_REP_DIR := include/depot include/gems/vfs.h include/gems/lru_cache.h content: $(MIRROR_FROM_REP_DIR) diff --git a/repos/gems/src/app/depot_query/main.cc b/repos/gems/src/app/depot_query/main.cc index 59dd7fdfb..c983c0975 100644 --- a/repos/gems/src/app/depot_query/main.cc +++ b/repos/gems/src/app/depot_query/main.cc @@ -18,11 +18,19 @@ #include #include #include +#include namespace Depot_query { + using namespace Depot; + + typedef String<64> Rom_label; + struct Recursion_limit; struct Dependencies; + class Stat_cache; + struct Rom_query; + class Cached_rom_query; struct Main; } @@ -144,7 +152,172 @@ class Depot_query::Dependencies }; -struct Depot_query::Main +class Depot_query::Stat_cache +{ + private: + + struct Key + { + struct Value + { + Archive::Path path; + + bool operator > (Value const &other) const + { + return strcmp(path.string(), other.path.string()) > 0; + } + + bool operator == (Value const &other) const + { + return path == other.path; + } + + } value; + }; + + struct Result { bool file_exists; }; + + typedef Lru_cache Cache; + + Cache::Size const _size; + + Cache _cache; + + Directory const &_dir; + + public: + + Stat_cache(Directory const &dir, Allocator &alloc, Xml_node const config) + : + _size({.value = config.attribute_value("stat_cache", Number_of_bytes(64*1024)) + / Cache::element_size()}), + _cache(alloc, _size), + _dir(dir) + { } + + bool file_exists(Archive::Path const path) + { + /* don't cache the state of the 'local' depot user */ + if (Archive::user(path) == "local") + return _dir.file_exists(path); + + bool result = false; + + auto hit_fn = [&] (Result const &cached_result) + { + result = cached_result.file_exists; + }; + + auto miss_fn = [&] (Cache::Missing_element &missing_element) + { + Result const stat_result { _dir.file_exists(path) }; + + /* + * Don't cache negative results because files may appear + * during installation. Later queries may find files absent + * from earlier queries. + */ + if (stat_result.file_exists) + missing_element.construct(stat_result); + }; + + Key const key { .value = { .path = path } }; + (void)_cache.try_apply(key, hit_fn, miss_fn); + + return result; + } +}; + + +struct Depot_query::Rom_query : Interface +{ + /** + * Look up ROM module 'rom_label' in the archives referenced by 'pkg_path' + * + * \throw Directory::Nonexistent_directory + * \throw Directory::Nonexistent_file + * \throw File::Truncated_during_read + * \throw Recursion_limit::Reached + */ + virtual Archive::Path find_rom_in_pkg(Directory::Path const &pkg_path, + Rom_label const &rom_label, + Recursion_limit recursion_limit) = 0; +}; + + +class Depot_query::Cached_rom_query : public Rom_query +{ + private: + + struct Key + { + struct Value + { + Archive::Path pkg; + Rom_label rom; + + bool operator > (Value const &other) const + { + return strcmp(pkg.string(), other.pkg.string()) > 0 + && strcmp(rom.string(), other.rom.string()) > 0; + } + + bool operator == (Value const &other) const + { + return pkg == other.pkg && rom == other.rom; + } + + } value; + }; + + typedef Lru_cache Cache; + + Cache::Size const _size; + + Cache mutable _cache; + + Rom_query &_rom_query; + + public: + + Cached_rom_query(Rom_query &rom_query, Allocator &alloc, Xml_node const config) + : + _size({.value = config.attribute_value("rom_query_cache", Number_of_bytes(64*1024)) + / Cache::element_size() }), + _cache(alloc, _size), + _rom_query(rom_query) + { } + + Archive::Path find_rom_in_pkg(Directory::Path const &pkg_path, + Rom_label const &rom_label, + Recursion_limit recursion_limit) override + { + /* don't cache the state of the 'local' depot user */ + if (Archive::user(pkg_path) == "local") + return _rom_query.find_rom_in_pkg(pkg_path, rom_label, recursion_limit); + + Archive::Path result { }; + + auto hit_fn = [&] (Archive::Path const &path) { result = path; }; + + auto miss_fn = [&] (Cache::Missing_element &missing_element) + { + Archive::Path const path = + _rom_query.find_rom_in_pkg(pkg_path, rom_label, recursion_limit); + + if (path.valid()) + missing_element.construct(path); + }; + + Key const key { .value = { .pkg = pkg_path, .rom = rom_label } }; + (void)_cache.try_apply(key, hit_fn, miss_fn); + + return result; + } +}; + + +struct Depot_query::Main : private Rom_query { Env &_env; @@ -158,6 +331,8 @@ struct Depot_query::Main Directory _depot_dir { _root, "depot" }; + Stat_cache _depot_stat_cache { _depot_dir, _heap, _config.xml() }; + Signal_handler
_config_handler { _env.ep(), *this, &Main::_handle_config }; @@ -182,7 +357,6 @@ struct Depot_query::Main obj.destruct(); } - typedef String<64> Rom_label; typedef String<16> Architecture; typedef String<32> Version; @@ -210,17 +384,13 @@ struct Depot_query::Main }); } + Cached_rom_query _cached_rom_query { *this, _heap, _config.xml() }; + /** - * Look up ROM module 'rom_label' in the archives referenced by 'pkg_path' - * - * \throw Directory::Nonexistent_directory - * \throw Directory::Nonexistent_file - * \throw File::Truncated_during_read - * \throw Recursion_limit::Reached + * Rom_query interface */ - Archive::Path _find_rom_in_pkg(Directory::Path const &pkg_path, - Rom_label const &rom_label, - Recursion_limit recursion_limit); + Archive::Path find_rom_in_pkg(Directory::Path const &, Rom_label const &, + Recursion_limit) override; void _query_blueprint(Directory::Path const &, Xml_generator &); void _collect_source_dependencies(Archive::Path const &, Dependencies &, Recursion_limit); @@ -344,9 +514,9 @@ struct Depot_query::Main Depot_query::Archive::Path -Depot_query::Main::_find_rom_in_pkg(Directory::Path const &pkg_path, - Rom_label const &rom_label, - Recursion_limit recursion_limit) +Depot_query::Main::find_rom_in_pkg(Directory::Path const &pkg_path, + Rom_label const &rom_label, + Recursion_limit recursion_limit) { /* * \throw Directory::Nonexistent_directory @@ -375,7 +545,7 @@ Depot_query::Main::_find_rom_in_pkg(Directory::Path const &pkg_path, Archive::name(archive_path), "/", Archive::version(archive_path), "/", rom_label); - if (_depot_dir.file_exists(rom_path)) + if (_depot_stat_cache.file_exists(rom_path)) result = rom_path; } break; @@ -387,16 +557,19 @@ Depot_query::Main::_find_rom_in_pkg(Directory::Path const &pkg_path, Archive::name(archive_path), "/", Archive::version(archive_path), "/", rom_label); - if (_depot_dir.file_exists(rom_path)) + if (_depot_stat_cache.file_exists(rom_path)) result = rom_path; } break; case Archive::PKG: + Archive::Path const result_from_pkg = - _find_rom_in_pkg(archive_path, rom_label, recursion_limit); + _cached_rom_query.find_rom_in_pkg(archive_path, rom_label, recursion_limit); + if (result_from_pkg.valid()) result = result_from_pkg; + break; } }); @@ -448,7 +621,7 @@ void Depot_query::Main::_query_blueprint(Directory::Path const &pkg_path, Xml_ge } Archive::Path const rom_path = - _find_rom_in_pkg(pkg_path, label, Recursion_limit{8}); + _cached_rom_query.find_rom_in_pkg(pkg_path, label, Recursion_limit{8}); if (rom_path.valid()) { xml.node("rom", [&] () {