tool/depot: improve handling of missing ports
* The extract tool determines and reports all missing ports at once. * The extract tool automatically prepares all missing ports if PREPARE_PORTS=1. * The missing_ports tool prints a list of missing ports for given archives. Fixes #3353
This commit is contained in:
Martin Stein
Christian Helmuth
parent
6d8d6b5552
commit
95ece89cf8
|
|
@ -48,6 +48,8 @@ define HELP_MESSAGE
|
|||
archive version needs to be updated. The default
|
||||
is to use the current date.
|
||||
|
||||
PREPARE_PORTS=1 Prepare missing ports automatically.
|
||||
|
||||
endef
|
||||
|
||||
export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..)
|
||||
|
|
@ -57,71 +59,9 @@ BIN_PKG_PATH_ELEMS := 4
|
|||
|
||||
include $(GENODE_DIR)/tool/depot/mk/front_end.inc
|
||||
include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc
|
||||
|
||||
|
||||
#
|
||||
# Collect dependencies for all specified arguments
|
||||
#
|
||||
# The following accessor functions used by 'mk/dependencies.inc'. The
|
||||
# information found in the 'archives' file of a package recipe has the
|
||||
# placeholder '_' for the user. Only archives with this placeholder are
|
||||
# considered. The '_user_pkg_archives' function transforms those archive paths
|
||||
# into user-specific archive paths.
|
||||
#
|
||||
|
||||
_file_in_depot = $(wildcard $(DEPOT_DIR)/$(call archive_user,$1)/src/$(call archive_recipe,$1)/$2)
|
||||
_file_in_recipe = $(addsuffix /$2,$(call recipe_dir,src/$(call archive_recipe,$1)))
|
||||
|
||||
_file_in_depot_or_recipe = $(if $(call _file_in_depot,$1,$2),\
|
||||
$(call _file_in_depot,$1,$2),\
|
||||
$(call _file_in_recipe,$1,$2))
|
||||
|
||||
api_file = $(call _file_in_depot_or_recipe,$1,api)
|
||||
used_apis_file = $(call _file_in_depot_or_recipe,$1,used_apis)
|
||||
|
||||
_pkg_archives_file = $(call recipe_dir,pkg/$(call archive_recipe,$1))/archives
|
||||
_user_pkg_archives = $(patsubst _/%,$(call archive_user,$1)/%,\
|
||||
$(call grep_archive_user,_,\
|
||||
$(call file_content,$(call _pkg_archives_file,$1))))
|
||||
|
||||
pkg_src_archives = $(call grep_archive_type,src,$(call _user_pkg_archives,$1))
|
||||
pkg_raw_archives = $(call grep_archive_type,raw,$(call _user_pkg_archives,$1))
|
||||
pkg_pkg_archives = $(call grep_archive_type,pkg,$(call _user_pkg_archives,$1))
|
||||
|
||||
include $(GENODE_DIR)/tool/depot/mk/extract_pre_dependencies.inc
|
||||
include $(GENODE_DIR)/tool/depot/mk/dependencies.inc
|
||||
|
||||
|
||||
#
|
||||
# Obtain version information from recipes
|
||||
#
|
||||
# The 'archive_curr_version' function takes the archive type and name as
|
||||
# arguments and returns the version identifier as present in the corresponding
|
||||
# recipe. The nested foreach loop populates 'ARCHIVE_VERSION' with the version
|
||||
# identifier for each archive.
|
||||
#
|
||||
# If an archive is given with a complete (versioned) name, we don't need to
|
||||
# consult any recipe but only check if the corresponding archive exists within
|
||||
# the depot. For binary archives, it suffices that the corresponding source
|
||||
# archive is present.
|
||||
#
|
||||
|
||||
$(foreach TYPE,api src raw pkg,\
|
||||
$(foreach PATH,${ARCHIVES(${TYPE})},\
|
||||
$(eval ARCHIVE_VERSION(${PATH}) := $(call archive_curr_version,$(PATH)))))
|
||||
|
||||
archive_exists_in_depot = $(wildcard $(DEPOT_DIR)/$1)
|
||||
|
||||
ARCHIVES_WITH_NO_VERSION := $(sort \
|
||||
$(foreach TYPE,api src raw pkg,\
|
||||
$(foreach A,${ARCHIVES(${TYPE})},\
|
||||
$(if $(call archive_exists_in_depot,$A),,\
|
||||
$(if ${ARCHIVE_VERSION($A)},,$A)))))
|
||||
|
||||
checked_versions_defined:
|
||||
ifneq ($(ARCHIVES_WITH_NO_VERSION),)
|
||||
@echo "Error: incomplete or missing recipe ($(sort $(ARCHIVES_WITH_NO_VERSION)))"; false
|
||||
endif
|
||||
|
||||
include $(GENODE_DIR)/tool/depot/mk/extract_post_dependencies.inc
|
||||
|
||||
#
|
||||
# Generate makefile for archive-extraction stage
|
||||
|
|
@ -140,7 +80,7 @@ wipe_existing_archives:
|
|||
$(foreach A,${ARCHIVES(${TYPE})},\
|
||||
$(call versioned_archive,$A))))
|
||||
|
||||
$(EXTRACT_MK_FILE): checked_versions_defined checked_no_uncategorized
|
||||
$(EXTRACT_MK_FILE): checked_versions_defined checked_no_uncategorized checked_ports_exist
|
||||
$(VERBOSE)mkdir -p $(dir $@)
|
||||
$(VERBOSE)( echo -e "all:\n"; \
|
||||
echo "TOOL_DIR := $(GENODE_DIR)/tool"; \
|
||||
|
|
@ -177,9 +117,11 @@ $(EXTRACT_MK_FILE): checked_versions_defined checked_no_uncategorized
|
|||
# Invoke sub make to process generated makefile
|
||||
#
|
||||
execute_generated_extract_mk_file: $(EXTRACT_MK_FILE)
|
||||
$(VERBOSE)$(MAKE) $(if $(VERBOSE),--quiet) -f $(EXTRACT_MK_FILE) \
|
||||
-C $(DEPOT_DIR) VERBOSE=$(VERBOSE) \
|
||||
UPDATE_VERSIONS=$(UPDATE_VERSIONS)
|
||||
$(VERBOSE)$(MAKE) $(QUIET) \
|
||||
-f $(EXTRACT_MK_FILE) \
|
||||
-C $(DEPOT_DIR) \
|
||||
VERBOSE=$(VERBOSE) \
|
||||
UPDATE_VERSIONS=$(UPDATE_VERSIONS)
|
||||
|
||||
ifneq ($(FORCE),)
|
||||
execute_generated_extract_mk_file: wipe_existing_archives
|
||||
|
|
@ -188,3 +130,43 @@ endif
|
|||
$(MAKECMDGOALS): execute_generated_extract_mk_file
|
||||
@true
|
||||
|
||||
#
|
||||
# Return shell script that handles missing ports
|
||||
#
|
||||
# \param $1 sorted list of missing ports file
|
||||
# \param $2 value of $(MAKE)
|
||||
#
|
||||
ifeq ($(PREPARE_PORTS),1)
|
||||
|
||||
# automatic prepare enabled: prepare missing ports
|
||||
handle_missing_ports = $(if $1,$(VERBOSE)( \
|
||||
echo -e ""; \
|
||||
echo -e "Ports not prepared or outdated:"; \
|
||||
echo -e " $(1)"; \
|
||||
echo -e ""; \
|
||||
echo -e "Preparing ports..."; \
|
||||
echo -e ""; \
|
||||
$2 $(QUIET) \
|
||||
-f $(GENODE_DIR)/tool/ports/prepare_port \
|
||||
$(1); ),)
|
||||
|
||||
else
|
||||
|
||||
# automatic prepare not enabled: generate error
|
||||
handle_missing_ports = $(if $1,$(VERBOSE)( \
|
||||
echo -e ""; \
|
||||
echo -e "Error: Ports not prepared or outdated:"; \
|
||||
echo -e " $(1)"; \
|
||||
echo -e ""; \
|
||||
echo -e "You can prepare respectively update them as follows:"; \
|
||||
echo -e " $(GENODE_DIR)/tool/ports/prepare_port $(1)"; \
|
||||
echo -e ""; \
|
||||
false),)
|
||||
|
||||
endif
|
||||
|
||||
#
|
||||
# Check whether all required ports exist and if not, abort or prepare them
|
||||
#
|
||||
checked_ports_exist: update_missing_ports_file
|
||||
$(call handle_missing_ports,$(call sorted_file_content,$(MISSING_PORTS_FILE)),$(MAKE))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
#!/usr/bin/make -f
|
||||
|
||||
#
|
||||
# \brief Print a list of missing ports for given API/source/raw archives
|
||||
# \author Martin Stein
|
||||
# \date 2019-05-15
|
||||
#
|
||||
|
||||
define HELP_MESSAGE
|
||||
|
||||
Print a list of missing ports for given API/source/raw archives
|
||||
|
||||
usage:
|
||||
|
||||
$(firstword $(MAKEFILE_LIST)) <archive-path>...
|
||||
|
||||
The <archive-path> argument denotes the archive to process in the
|
||||
form of a path. The first path element corresponds to the identity
|
||||
of the archive creator, the second element corresponds to the type
|
||||
of the archive, and the third element refers to the recipe of
|
||||
the archive description.
|
||||
|
||||
E.g., the user 'alan' may have the following archives:
|
||||
|
||||
alan/api/libc - an API archive for the libc
|
||||
alan/src/zlib - a source archive for the zlib library
|
||||
|
||||
The following arguments tweak the operation of the tool:
|
||||
|
||||
VERBOSE= Show individual operations.
|
||||
|
||||
-j<N> Enable the parallel processing of packages where
|
||||
<N> denotes the level of parallelism.
|
||||
|
||||
endef
|
||||
|
||||
export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..)
|
||||
|
||||
# the missing-ports tool expects archive paths given without the version element
|
||||
BIN_PKG_PATH_ELEMS := 4
|
||||
|
||||
include $(GENODE_DIR)/tool/depot/mk/front_end.inc
|
||||
include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc
|
||||
include $(GENODE_DIR)/tool/depot/mk/extract_pre_dependencies.inc
|
||||
include $(GENODE_DIR)/tool/depot/mk/dependencies.inc
|
||||
include $(GENODE_DIR)/tool/depot/mk/extract_post_dependencies.inc
|
||||
|
||||
$(MAKECMDGOALS): dump_missing_ports_file
|
||||
$(VERBOSE)true
|
||||
|
||||
dump_missing_ports_file: update_missing_ports_file
|
||||
$(VERBOSE)echo $(call sorted_file_content,$(MISSING_PORTS_FILE))
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
#
|
||||
# \brief Environment for content.mk files when determining missing ports
|
||||
# \author Martin Stein
|
||||
# \date 2019-05-12
|
||||
#
|
||||
# GENODE_DIR - root directory of the Genode source tree
|
||||
# CONTRIB_DIR - directory for 3rd-party code
|
||||
# CONTENT_MK - content.mk file to process
|
||||
# REP_DIR - repository directory of the content.mk file
|
||||
# MISSING_PORTS_FILE - file to write the names of missing ports to
|
||||
# VERBOSE - verbosity
|
||||
#
|
||||
|
||||
#
|
||||
# Functions for disabling and re-enabling evaluation of $(shell ...)
|
||||
#
|
||||
ORIGINAL_SHELL := $(SHELL)
|
||||
enable_shell = $(eval SHELL:=$(ORIGINAL_SHELL))
|
||||
disable_shell = $(eval SHELL:=true)
|
||||
|
||||
#
|
||||
# Disable shell calls so the content.mk file will not evaluate something like
|
||||
# $(shell find $(PORT_DIR) ...) while 'PORT_DIR' is empty because we have
|
||||
# overridden the port_dir function.
|
||||
#
|
||||
$(disable_shell)
|
||||
|
||||
#
|
||||
# If a port is missing, append its name to the missing ports file
|
||||
#
|
||||
_assert = $(if $1,$1,$(shell echo $2 >> $(MISSING_PORTS_FILE)))
|
||||
|
||||
#
|
||||
# Utility to query the port directory for a given path to a port description.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# $(call port_dir,$(GENODE_DIR)/repos/libports/ports/libpng)
|
||||
#
|
||||
_port_hash = $(shell cat $(call _assert,$(wildcard $1.hash),$(notdir $1)))
|
||||
_port_dir = $(wildcard $(CONTRIB_DIR)/$(notdir $1)-$(call _port_hash,$1))
|
||||
port_dir = $(call enable_shell)$(call _assert,$(call _port_dir,$1),$(notdir $1))$(call disable_shell)
|
||||
|
||||
#
|
||||
# Prevent the evaluation of mirror_from_rep_dir in content.mk
|
||||
#
|
||||
mirror_from_rep_dir = $(error mirror_from_rep_dir called outside of target)
|
||||
|
||||
#
|
||||
# Prevent the evaluation of the first target in the content.mk file
|
||||
#
|
||||
prevent_execution_of_content_targets:
|
||||
|
||||
#
|
||||
# Include the content.mk file to evaluate all calls to the port_dir function
|
||||
#
|
||||
include $(CONTENT_MK)
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
#
|
||||
# Obtain version information from recipes
|
||||
#
|
||||
# The 'archive_curr_version' function takes the archive type and name as
|
||||
# arguments and returns the version identifier as present in the corresponding
|
||||
# recipe. The nested foreach loop populates 'ARCHIVE_VERSION' with the version
|
||||
# identifier for each archive.
|
||||
#
|
||||
# If an archive is given with a complete (versioned) name, we don't need to
|
||||
# consult any recipe but only check if the corresponding archive exists within
|
||||
# the depot. For binary archives, it suffices that the corresponding source
|
||||
# archive is present.
|
||||
#
|
||||
|
||||
$(foreach TYPE,api src raw pkg,\
|
||||
$(foreach PATH,${ARCHIVES(${TYPE})},\
|
||||
$(eval ARCHIVE_VERSION(${PATH}) := $(call archive_curr_version,$(PATH)))))
|
||||
|
||||
archive_exists_in_depot = $(wildcard $(DEPOT_DIR)/$1)
|
||||
|
||||
ARCHIVES_WITH_NO_VERSION := $(sort \
|
||||
$(foreach TYPE,api src raw pkg,\
|
||||
$(foreach A,${ARCHIVES(${TYPE})},\
|
||||
$(if $(call archive_exists_in_depot,$A),,\
|
||||
$(if ${ARCHIVE_VERSION($A)},,$A)))))
|
||||
|
||||
checked_versions_defined:
|
||||
ifneq ($(ARCHIVES_WITH_NO_VERSION),)
|
||||
@echo "Error: incomplete or missing recipe ($(sort $(ARCHIVES_WITH_NO_VERSION)))"; false
|
||||
endif
|
||||
|
||||
#
|
||||
# Read content of a file as list, sort it and remove duplicates
|
||||
#
|
||||
# \param $1 absolute file path
|
||||
#
|
||||
sorted_file_content = $(if $(wildcard $1),$(shell cat $1 | sort -u),\
|
||||
$(error Failed to read file $1))
|
||||
|
||||
#
|
||||
# Absolute path to content.mk file for given archive
|
||||
#
|
||||
# \param $1 archive type, can be 'src', 'api' or 'raw'
|
||||
# \param $2 api, src or raw archive name in the form 'genodelabs/api/base'
|
||||
#
|
||||
content_mk_file = $(addsuffix /content.mk,$(call recipe_dir,$1/$(call archive_recipe,$2)))
|
||||
|
||||
#
|
||||
# Return a given string minus another given string
|
||||
#
|
||||
# \param $1 string that shall be returned minus the other one
|
||||
# \param $2 string that shall be removed from the other one
|
||||
#
|
||||
remove_from_string=$(1:$2=)
|
||||
|
||||
#
|
||||
# Repository directory for a given recipe name
|
||||
#
|
||||
# \param $1 recipe type in the form 'api'
|
||||
# \param $2 recipe name in the form 'base'
|
||||
#
|
||||
rep_dir_of_recipe=$(call remove_from_string,$(call recipe_dir,$1/$2),/recipes/$1/$2)
|
||||
|
||||
#
|
||||
# Repository directory for a given archive
|
||||
#
|
||||
# \param $1 archive type in the form 'api'
|
||||
# \param $2 archive name in the form 'genodelabs/api/base'
|
||||
#
|
||||
rep_dir_of_archive=$(call rep_dir_of_recipe,$1,$(call archive_recipe,$2))
|
||||
|
||||
# path to temporary file that is used to buffer the names of missing ports
|
||||
MISSING_PORTS_FILE := $(DEPOT_DIR)/var/missing_ports
|
||||
|
||||
# path to temporary make file that is created to fill the missing-ports file
|
||||
GEN_MISSING_PORTS_MK := $(DEPOT_DIR)/var/gen_missing_ports.mk
|
||||
|
||||
# wether to invoke sub-makes with '--quiet'
|
||||
QUIET = $(if $(VERBOSE),--quiet)
|
||||
|
||||
#
|
||||
# Invoke sub-make to create or update missing-ports file
|
||||
#
|
||||
update_missing_ports_file: checked_versions_defined checked_no_uncategorized
|
||||
$(VERBOSE)mkdir -p $(dir $(GEN_MISSING_PORTS_MK))
|
||||
$(VERBOSE)( echo -e "all:\n"; \
|
||||
echo -e "MAKE := $(MAKE)\n"; \
|
||||
$(foreach TYPE,api src raw pkg,\
|
||||
$(foreach A,${ARCHIVES(${TYPE})},\
|
||||
target=$(call versioned_archive,$A); \
|
||||
content_mk=$(call content_mk_file,$(TYPE),$A); \
|
||||
rep_dir=$(call rep_dir_of_archive,$(TYPE),$A); \
|
||||
echo "ARCHIVES(${TYPE}) += $$target"; \
|
||||
echo "CONTENT_MK($$target) := $$content_mk"; \
|
||||
echo "REP_DIR($$target) := $$rep_dir"; \
|
||||
) ) \
|
||||
echo -e ""; \
|
||||
$(foreach A,${ARCHIVES(pkg)},\
|
||||
$(foreach DEP,$(call pkg_pkg_archives,$A),\
|
||||
echo -e "$(call versioned_archive,$A) :" \
|
||||
"$(call versioned_archive,$(DEP))";)) \
|
||||
echo -e ""; \
|
||||
echo -e "\$${ARCHIVES(src)} : \$${ARCHIVES(api)}"; \
|
||||
echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(api)}"; \
|
||||
echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(src)}"; \
|
||||
echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(raw)}"; \
|
||||
echo -e "\nTARGETS := \$$(foreach T,api src raw,\$${ARCHIVES(\$$T)})"; \
|
||||
echo -e "\nall: \$$(TARGETS)"; \
|
||||
echo -e "\n\$$(TARGETS):"; \
|
||||
echo -e "\t$(VERBOSE)\$$(MAKE) \\"; \
|
||||
echo -e "\t $(QUIET) \\"; \
|
||||
echo -e "\t -f $(GENODE_DIR)/tool/depot/mk/content_env_missing_ports.mk \\"; \
|
||||
echo -e "\t GENODE_DIR=$(GENODE_DIR) \\"; \
|
||||
echo -e "\t CONTRIB_DIR=$(GENODE_DIR)/contrib \\"; \
|
||||
echo -e "\t CONTENT_MK=\$${CONTENT_MK(\$$@)} \\"; \
|
||||
echo -e "\t REP_DIR=\$${REP_DIR(\$$@)} \\"; \
|
||||
echo -e "\t MISSING_PORTS_FILE=$(MISSING_PORTS_FILE) \\"; \
|
||||
echo -e "\t VERBOSE=$(VERBOSE)"; \
|
||||
) > $(GEN_MISSING_PORTS_MK)
|
||||
$(VERBOSE)mkdir -p $(dir $(MISSING_PORTS_FILE))
|
||||
$(VERBOSE)rm -f $(MISSING_PORTS_FILE)
|
||||
$(VERBOSE)touch $(MISSING_PORTS_FILE)
|
||||
$(VERBOSE)$(MAKE) $(QUIET) -C $(DEPOT_DIR) -f $(GEN_MISSING_PORTS_MK);
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
#
|
||||
# Collect dependencies for all specified arguments
|
||||
#
|
||||
# The following accessor functions used by 'mk/dependencies.inc'. The
|
||||
# information found in the 'archives' file of a package recipe has the
|
||||
# placeholder '_' for the user. Only archives with this placeholder are
|
||||
# considered. The '_user_pkg_archives' function transforms those archive paths
|
||||
# into user-specific archive paths.
|
||||
#
|
||||
|
||||
_file_in_depot = $(wildcard $(DEPOT_DIR)/$(call archive_user,$1)/src/$(call archive_recipe,$1)/$2)
|
||||
_file_in_recipe = $(addsuffix /$2,$(call recipe_dir,src/$(call archive_recipe,$1)))
|
||||
|
||||
_file_in_depot_or_recipe = $(if $(call _file_in_depot,$1,$2),\
|
||||
$(call _file_in_depot,$1,$2),\
|
||||
$(call _file_in_recipe,$1,$2))
|
||||
|
||||
api_file = $(call _file_in_depot_or_recipe,$1,api)
|
||||
used_apis_file = $(call _file_in_depot_or_recipe,$1,used_apis)
|
||||
|
||||
_pkg_archives_file = $(call recipe_dir,pkg/$(call archive_recipe,$1))/archives
|
||||
_user_pkg_archives = $(patsubst _/%,$(call archive_user,$1)/%,\
|
||||
$(call grep_archive_user,_,\
|
||||
$(call file_content,$(call _pkg_archives_file,$1))))
|
||||
|
||||
pkg_src_archives = $(call grep_archive_type,src,$(call _user_pkg_archives,$1))
|
||||
pkg_raw_archives = $(call grep_archive_type,raw,$(call _user_pkg_archives,$1))
|
||||
pkg_pkg_archives = $(call grep_archive_type,pkg,$(call _user_pkg_archives,$1))
|
||||
Loading…
Reference in New Issue