From 95ece89cf81cc00834c42093e2d2588e0c5cf132 Mon Sep 17 00:00:00 2001 From: Martin Stein Date: Sun, 12 May 2019 03:13:02 +0200 Subject: [PATCH] tool/depot: improve handling of missing ports * The extract tool determines and reports all missing ports at once. * The extract tool automatically prepares all missing ports if PREPARE_PORTS=1. * The missing_ports tool prints a list of missing ports for given archives. Fixes #3353 --- tool/depot/extract | 118 ++++++++----------- tool/depot/missing_ports | 52 +++++++++ tool/depot/mk/content_env_missing_ports.mk | 57 +++++++++ tool/depot/mk/extract_post_dependencies.inc | 123 ++++++++++++++++++++ tool/depot/mk/extract_pre_dependencies.inc | 28 +++++ 5 files changed, 310 insertions(+), 68 deletions(-) create mode 100755 tool/depot/missing_ports create mode 100644 tool/depot/mk/content_env_missing_ports.mk create mode 100644 tool/depot/mk/extract_post_dependencies.inc create mode 100644 tool/depot/mk/extract_pre_dependencies.inc diff --git a/tool/depot/extract b/tool/depot/extract index eaeba0402..08204a609 100755 --- a/tool/depot/extract +++ b/tool/depot/extract @@ -48,6 +48,8 @@ define HELP_MESSAGE archive version needs to be updated. The default is to use the current date. + PREPARE_PORTS=1 Prepare missing ports automatically. + endef export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..) @@ -57,71 +59,9 @@ BIN_PKG_PATH_ELEMS := 4 include $(GENODE_DIR)/tool/depot/mk/front_end.inc include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc - - -# -# Collect dependencies for all specified arguments -# -# The following accessor functions used by 'mk/dependencies.inc'. The -# information found in the 'archives' file of a package recipe has the -# placeholder '_' for the user. Only archives with this placeholder are -# considered. The '_user_pkg_archives' function transforms those archive paths -# into user-specific archive paths. -# - -_file_in_depot = $(wildcard $(DEPOT_DIR)/$(call archive_user,$1)/src/$(call archive_recipe,$1)/$2) -_file_in_recipe = $(addsuffix /$2,$(call recipe_dir,src/$(call archive_recipe,$1))) - -_file_in_depot_or_recipe = $(if $(call _file_in_depot,$1,$2),\ - $(call _file_in_depot,$1,$2),\ - $(call _file_in_recipe,$1,$2)) - -api_file = $(call _file_in_depot_or_recipe,$1,api) -used_apis_file = $(call _file_in_depot_or_recipe,$1,used_apis) - -_pkg_archives_file = $(call recipe_dir,pkg/$(call archive_recipe,$1))/archives -_user_pkg_archives = $(patsubst _/%,$(call archive_user,$1)/%,\ - $(call grep_archive_user,_,\ - $(call file_content,$(call _pkg_archives_file,$1)))) - -pkg_src_archives = $(call grep_archive_type,src,$(call _user_pkg_archives,$1)) -pkg_raw_archives = $(call grep_archive_type,raw,$(call _user_pkg_archives,$1)) -pkg_pkg_archives = $(call grep_archive_type,pkg,$(call _user_pkg_archives,$1)) - +include $(GENODE_DIR)/tool/depot/mk/extract_pre_dependencies.inc include $(GENODE_DIR)/tool/depot/mk/dependencies.inc - - -# -# Obtain version information from recipes -# -# The 'archive_curr_version' function takes the archive type and name as -# arguments and returns the version identifier as present in the corresponding -# recipe. The nested foreach loop populates 'ARCHIVE_VERSION' with the version -# identifier for each archive. -# -# If an archive is given with a complete (versioned) name, we don't need to -# consult any recipe but only check if the corresponding archive exists within -# the depot. For binary archives, it suffices that the corresponding source -# archive is present. -# - -$(foreach TYPE,api src raw pkg,\ - $(foreach PATH,${ARCHIVES(${TYPE})},\ - $(eval ARCHIVE_VERSION(${PATH}) := $(call archive_curr_version,$(PATH))))) - -archive_exists_in_depot = $(wildcard $(DEPOT_DIR)/$1) - -ARCHIVES_WITH_NO_VERSION := $(sort \ - $(foreach TYPE,api src raw pkg,\ - $(foreach A,${ARCHIVES(${TYPE})},\ - $(if $(call archive_exists_in_depot,$A),,\ - $(if ${ARCHIVE_VERSION($A)},,$A))))) - -checked_versions_defined: -ifneq ($(ARCHIVES_WITH_NO_VERSION),) - @echo "Error: incomplete or missing recipe ($(sort $(ARCHIVES_WITH_NO_VERSION)))"; false -endif - +include $(GENODE_DIR)/tool/depot/mk/extract_post_dependencies.inc # # Generate makefile for archive-extraction stage @@ -140,7 +80,7 @@ wipe_existing_archives: $(foreach A,${ARCHIVES(${TYPE})},\ $(call versioned_archive,$A)))) -$(EXTRACT_MK_FILE): checked_versions_defined checked_no_uncategorized +$(EXTRACT_MK_FILE): checked_versions_defined checked_no_uncategorized checked_ports_exist $(VERBOSE)mkdir -p $(dir $@) $(VERBOSE)( echo -e "all:\n"; \ echo "TOOL_DIR := $(GENODE_DIR)/tool"; \ @@ -177,9 +117,11 @@ $(EXTRACT_MK_FILE): checked_versions_defined checked_no_uncategorized # Invoke sub make to process generated makefile # execute_generated_extract_mk_file: $(EXTRACT_MK_FILE) - $(VERBOSE)$(MAKE) $(if $(VERBOSE),--quiet) -f $(EXTRACT_MK_FILE) \ - -C $(DEPOT_DIR) VERBOSE=$(VERBOSE) \ - UPDATE_VERSIONS=$(UPDATE_VERSIONS) + $(VERBOSE)$(MAKE) $(QUIET) \ + -f $(EXTRACT_MK_FILE) \ + -C $(DEPOT_DIR) \ + VERBOSE=$(VERBOSE) \ + UPDATE_VERSIONS=$(UPDATE_VERSIONS) ifneq ($(FORCE),) execute_generated_extract_mk_file: wipe_existing_archives @@ -188,3 +130,43 @@ endif $(MAKECMDGOALS): execute_generated_extract_mk_file @true +# +# Return shell script that handles missing ports +# +# \param $1 sorted list of missing ports file +# \param $2 value of $(MAKE) +# +ifeq ($(PREPARE_PORTS),1) + +# automatic prepare enabled: prepare missing ports +handle_missing_ports = $(if $1,$(VERBOSE)( \ + echo -e ""; \ + echo -e "Ports not prepared or outdated:"; \ + echo -e " $(1)"; \ + echo -e ""; \ + echo -e "Preparing ports..."; \ + echo -e ""; \ + $2 $(QUIET) \ + -f $(GENODE_DIR)/tool/ports/prepare_port \ + $(1); ),) + +else + +# automatic prepare not enabled: generate error +handle_missing_ports = $(if $1,$(VERBOSE)( \ + echo -e ""; \ + echo -e "Error: Ports not prepared or outdated:"; \ + echo -e " $(1)"; \ + echo -e ""; \ + echo -e "You can prepare respectively update them as follows:"; \ + echo -e " $(GENODE_DIR)/tool/ports/prepare_port $(1)"; \ + echo -e ""; \ + false),) + +endif + +# +# Check whether all required ports exist and if not, abort or prepare them +# +checked_ports_exist: update_missing_ports_file + $(call handle_missing_ports,$(call sorted_file_content,$(MISSING_PORTS_FILE)),$(MAKE)) diff --git a/tool/depot/missing_ports b/tool/depot/missing_ports new file mode 100755 index 000000000..f26f0bea8 --- /dev/null +++ b/tool/depot/missing_ports @@ -0,0 +1,52 @@ +#!/usr/bin/make -f + +# +# \brief Print a list of missing ports for given API/source/raw archives +# \author Martin Stein +# \date 2019-05-15 +# + +define HELP_MESSAGE + + Print a list of missing ports for given API/source/raw archives + + usage: + + $(firstword $(MAKEFILE_LIST)) ... + + The argument denotes the archive to process in the + form of a path. The first path element corresponds to the identity + of the archive creator, the second element corresponds to the type + of the archive, and the third element refers to the recipe of + the archive description. + + E.g., the user 'alan' may have the following archives: + + alan/api/libc - an API archive for the libc + alan/src/zlib - a source archive for the zlib library + + The following arguments tweak the operation of the tool: + + VERBOSE= Show individual operations. + + -j Enable the parallel processing of packages where + denotes the level of parallelism. + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..) + +# the missing-ports tool expects archive paths given without the version element +BIN_PKG_PATH_ELEMS := 4 + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc +include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc +include $(GENODE_DIR)/tool/depot/mk/extract_pre_dependencies.inc +include $(GENODE_DIR)/tool/depot/mk/dependencies.inc +include $(GENODE_DIR)/tool/depot/mk/extract_post_dependencies.inc + +$(MAKECMDGOALS): dump_missing_ports_file + $(VERBOSE)true + +dump_missing_ports_file: update_missing_ports_file + $(VERBOSE)echo $(call sorted_file_content,$(MISSING_PORTS_FILE)) diff --git a/tool/depot/mk/content_env_missing_ports.mk b/tool/depot/mk/content_env_missing_ports.mk new file mode 100644 index 000000000..c09b3a8d1 --- /dev/null +++ b/tool/depot/mk/content_env_missing_ports.mk @@ -0,0 +1,57 @@ +# +# \brief Environment for content.mk files when determining missing ports +# \author Martin Stein +# \date 2019-05-12 +# +# GENODE_DIR - root directory of the Genode source tree +# CONTRIB_DIR - directory for 3rd-party code +# CONTENT_MK - content.mk file to process +# REP_DIR - repository directory of the content.mk file +# MISSING_PORTS_FILE - file to write the names of missing ports to +# VERBOSE - verbosity +# + +# +# Functions for disabling and re-enabling evaluation of $(shell ...) +# +ORIGINAL_SHELL := $(SHELL) +enable_shell = $(eval SHELL:=$(ORIGINAL_SHELL)) +disable_shell = $(eval SHELL:=true) + +# +# Disable shell calls so the content.mk file will not evaluate something like +# $(shell find $(PORT_DIR) ...) while 'PORT_DIR' is empty because we have +# overridden the port_dir function. +# +$(disable_shell) + +# +# If a port is missing, append its name to the missing ports file +# +_assert = $(if $1,$1,$(shell echo $2 >> $(MISSING_PORTS_FILE))) + +# +# Utility to query the port directory for a given path to a port description. +# +# Example: +# +# $(call port_dir,$(GENODE_DIR)/repos/libports/ports/libpng) +# +_port_hash = $(shell cat $(call _assert,$(wildcard $1.hash),$(notdir $1))) +_port_dir = $(wildcard $(CONTRIB_DIR)/$(notdir $1)-$(call _port_hash,$1)) +port_dir = $(call enable_shell)$(call _assert,$(call _port_dir,$1),$(notdir $1))$(call disable_shell) + +# +# Prevent the evaluation of mirror_from_rep_dir in content.mk +# +mirror_from_rep_dir = $(error mirror_from_rep_dir called outside of target) + +# +# Prevent the evaluation of the first target in the content.mk file +# +prevent_execution_of_content_targets: + +# +# Include the content.mk file to evaluate all calls to the port_dir function +# +include $(CONTENT_MK) diff --git a/tool/depot/mk/extract_post_dependencies.inc b/tool/depot/mk/extract_post_dependencies.inc new file mode 100644 index 000000000..d1f06e823 --- /dev/null +++ b/tool/depot/mk/extract_post_dependencies.inc @@ -0,0 +1,123 @@ +# +# Obtain version information from recipes +# +# The 'archive_curr_version' function takes the archive type and name as +# arguments and returns the version identifier as present in the corresponding +# recipe. The nested foreach loop populates 'ARCHIVE_VERSION' with the version +# identifier for each archive. +# +# If an archive is given with a complete (versioned) name, we don't need to +# consult any recipe but only check if the corresponding archive exists within +# the depot. For binary archives, it suffices that the corresponding source +# archive is present. +# + +$(foreach TYPE,api src raw pkg,\ + $(foreach PATH,${ARCHIVES(${TYPE})},\ + $(eval ARCHIVE_VERSION(${PATH}) := $(call archive_curr_version,$(PATH))))) + +archive_exists_in_depot = $(wildcard $(DEPOT_DIR)/$1) + +ARCHIVES_WITH_NO_VERSION := $(sort \ + $(foreach TYPE,api src raw pkg,\ + $(foreach A,${ARCHIVES(${TYPE})},\ + $(if $(call archive_exists_in_depot,$A),,\ + $(if ${ARCHIVE_VERSION($A)},,$A))))) + +checked_versions_defined: +ifneq ($(ARCHIVES_WITH_NO_VERSION),) + @echo "Error: incomplete or missing recipe ($(sort $(ARCHIVES_WITH_NO_VERSION)))"; false +endif + +# +# Read content of a file as list, sort it and remove duplicates +# +# \param $1 absolute file path +# +sorted_file_content = $(if $(wildcard $1),$(shell cat $1 | sort -u),\ + $(error Failed to read file $1)) + +# +# Absolute path to content.mk file for given archive +# +# \param $1 archive type, can be 'src', 'api' or 'raw' +# \param $2 api, src or raw archive name in the form 'genodelabs/api/base' +# +content_mk_file = $(addsuffix /content.mk,$(call recipe_dir,$1/$(call archive_recipe,$2))) + +# +# Return a given string minus another given string +# +# \param $1 string that shall be returned minus the other one +# \param $2 string that shall be removed from the other one +# +remove_from_string=$(1:$2=) + +# +# Repository directory for a given recipe name +# +# \param $1 recipe type in the form 'api' +# \param $2 recipe name in the form 'base' +# +rep_dir_of_recipe=$(call remove_from_string,$(call recipe_dir,$1/$2),/recipes/$1/$2) + +# +# Repository directory for a given archive +# +# \param $1 archive type in the form 'api' +# \param $2 archive name in the form 'genodelabs/api/base' +# +rep_dir_of_archive=$(call rep_dir_of_recipe,$1,$(call archive_recipe,$2)) + +# path to temporary file that is used to buffer the names of missing ports +MISSING_PORTS_FILE := $(DEPOT_DIR)/var/missing_ports + +# path to temporary make file that is created to fill the missing-ports file +GEN_MISSING_PORTS_MK := $(DEPOT_DIR)/var/gen_missing_ports.mk + +# wether to invoke sub-makes with '--quiet' +QUIET = $(if $(VERBOSE),--quiet) + +# +# Invoke sub-make to create or update missing-ports file +# +update_missing_ports_file: checked_versions_defined checked_no_uncategorized + $(VERBOSE)mkdir -p $(dir $(GEN_MISSING_PORTS_MK)) + $(VERBOSE)( echo -e "all:\n"; \ + echo -e "MAKE := $(MAKE)\n"; \ + $(foreach TYPE,api src raw pkg,\ + $(foreach A,${ARCHIVES(${TYPE})},\ + target=$(call versioned_archive,$A); \ + content_mk=$(call content_mk_file,$(TYPE),$A); \ + rep_dir=$(call rep_dir_of_archive,$(TYPE),$A); \ + echo "ARCHIVES(${TYPE}) += $$target"; \ + echo "CONTENT_MK($$target) := $$content_mk"; \ + echo "REP_DIR($$target) := $$rep_dir"; \ + ) ) \ + echo -e ""; \ + $(foreach A,${ARCHIVES(pkg)},\ + $(foreach DEP,$(call pkg_pkg_archives,$A),\ + echo -e "$(call versioned_archive,$A) :" \ + "$(call versioned_archive,$(DEP))";)) \ + echo -e ""; \ + echo -e "\$${ARCHIVES(src)} : \$${ARCHIVES(api)}"; \ + echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(api)}"; \ + echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(src)}"; \ + echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(raw)}"; \ + echo -e "\nTARGETS := \$$(foreach T,api src raw,\$${ARCHIVES(\$$T)})"; \ + echo -e "\nall: \$$(TARGETS)"; \ + echo -e "\n\$$(TARGETS):"; \ + echo -e "\t$(VERBOSE)\$$(MAKE) \\"; \ + echo -e "\t $(QUIET) \\"; \ + echo -e "\t -f $(GENODE_DIR)/tool/depot/mk/content_env_missing_ports.mk \\"; \ + echo -e "\t GENODE_DIR=$(GENODE_DIR) \\"; \ + echo -e "\t CONTRIB_DIR=$(GENODE_DIR)/contrib \\"; \ + echo -e "\t CONTENT_MK=\$${CONTENT_MK(\$$@)} \\"; \ + echo -e "\t REP_DIR=\$${REP_DIR(\$$@)} \\"; \ + echo -e "\t MISSING_PORTS_FILE=$(MISSING_PORTS_FILE) \\"; \ + echo -e "\t VERBOSE=$(VERBOSE)"; \ + ) > $(GEN_MISSING_PORTS_MK) + $(VERBOSE)mkdir -p $(dir $(MISSING_PORTS_FILE)) + $(VERBOSE)rm -f $(MISSING_PORTS_FILE) + $(VERBOSE)touch $(MISSING_PORTS_FILE) + $(VERBOSE)$(MAKE) $(QUIET) -C $(DEPOT_DIR) -f $(GEN_MISSING_PORTS_MK); diff --git a/tool/depot/mk/extract_pre_dependencies.inc b/tool/depot/mk/extract_pre_dependencies.inc new file mode 100644 index 000000000..f6e333142 --- /dev/null +++ b/tool/depot/mk/extract_pre_dependencies.inc @@ -0,0 +1,28 @@ +# +# Collect dependencies for all specified arguments +# +# The following accessor functions used by 'mk/dependencies.inc'. The +# information found in the 'archives' file of a package recipe has the +# placeholder '_' for the user. Only archives with this placeholder are +# considered. The '_user_pkg_archives' function transforms those archive paths +# into user-specific archive paths. +# + +_file_in_depot = $(wildcard $(DEPOT_DIR)/$(call archive_user,$1)/src/$(call archive_recipe,$1)/$2) +_file_in_recipe = $(addsuffix /$2,$(call recipe_dir,src/$(call archive_recipe,$1))) + +_file_in_depot_or_recipe = $(if $(call _file_in_depot,$1,$2),\ + $(call _file_in_depot,$1,$2),\ + $(call _file_in_recipe,$1,$2)) + +api_file = $(call _file_in_depot_or_recipe,$1,api) +used_apis_file = $(call _file_in_depot_or_recipe,$1,used_apis) + +_pkg_archives_file = $(call recipe_dir,pkg/$(call archive_recipe,$1))/archives +_user_pkg_archives = $(patsubst _/%,$(call archive_user,$1)/%,\ + $(call grep_archive_user,_,\ + $(call file_content,$(call _pkg_archives_file,$1)))) + +pkg_src_archives = $(call grep_archive_type,src,$(call _user_pkg_archives,$1)) +pkg_raw_archives = $(call grep_archive_type,raw,$(call _user_pkg_archives,$1)) +pkg_pkg_archives = $(call grep_archive_type,pkg,$(call _user_pkg_archives,$1))