diff --git a/.gitignore b/.gitignore index 01c7226e4..134de28b1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,7 @@ /build /contrib +/depot +/public /repos/world diff --git a/tool/depot/build b/tool/depot/build new file mode 100755 index 000000000..f00c55965 --- /dev/null +++ b/tool/depot/build @@ -0,0 +1,146 @@ +#!/usr/bin/make -f + +# +# \brief Build binary archives from source +# \author Norman Feske +# \date 2017-03-16 +# + +define HELP_MESSAGE + + Build binary archives from source archives stored in the depot + + usage: + + $(firstword $(MAKEFILE_LIST)) ... + + The argument denotes the archive to create in the + form of a path. The first path element correponds to the identity + of the archive creator, the second element corresponds to the type + of the archive (bin or pkg), the third element specifies the target + architectures (e.g., x86_64), and the fourth element is the name + of the corresponding source archive including the version. + + E.g., the user 'alan' may build the following archives: + + alan/bin/x86_64/zlib- - a binary archive of the zlib + library with the specified + version, built for the 64-bit + x86 architecture + + alan/pkg/x86_32/wm- - all binary archives needed by + the 'wm' package archive, built + for the 32-bit x86 architecture + + The following arguments tweak the operation of the tool: + + FORCE=1 Replace existing archives with freshly created + ones. + + VERBOSE= Show individual operations. + + -j Enable the parallel creation of packages where + denotes the level of parallelism. + + KEEP_BUILD_DIR=1 Do not remove build directories of built binary + packages. This is useful for debugging build + problems. +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..) + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc +include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc + + +# +# Collect dependencies for all specified arguments +# +# The following accessor functions used by 'mk/dependencies.inc'. +# + +_file_within_archive = $(wildcard $(DEPOT_DIR)/$1/$2) + +api_file = $(call _file_within_archive,$1,api) +used_apis_file = $(call _file_within_archive,$1,used_apis) + +_pkg_archives_content = $(call file_content,$(call _file_within_archive,$1,archives)) + +pkg_src_archives = $(call grep_archive_type,src,$(call _pkg_archives_content,$1)) +pkg_raw_archives = $(call grep_archive_type,raw,$(call _pkg_archives_content,$1)) +pkg_pkg_archives = $(call grep_archive_type,pkg,$(call _pkg_archives_content,$1)) + +include $(GENODE_DIR)/tool/depot/mk/dependencies.inc + + +# +# Detect missing source archives +# + +archive_exists_in_depot = $(wildcard $(DEPOT_DIR)/$1) + +MISSING_ARCHIVES := $(sort \ + $(foreach A,${ARCHIVES(bin)},\ + $(if $(call archive_exists_in_depot,$(call src_of_bin,$A)),,$A))) + +checked_source_archives_exist: +ifneq ($(MISSING_ARCHIVES),) + @echo "Error: archives missing in the depot ($(MISSING_ARCHIVES))"; false +endif + + +# +# Generate makefile for archive-build stage +# + +bin_archive_spec = $(word 3,$(subst /, ,$1)) +bin_archive_recipe = $(word 4,$(subst /, ,$1)) + +# determine binary-archive path within the depot +_api_subdir = $(addsuffix /,$(call file_content,$(call api_file,$(call src_of_bin,$1)))) +_dst_bin_spec_path = $(call archive_user,$1)/bin/$(call bin_archive_spec,$1)/ +dst_archive_path = $(call _dst_bin_spec_path,$1)$(call _api_subdir,$1)$(call bin_archive_recipe,$1) + +BUILD_MK_FILE := $(DEPOT_DIR)/var/build.mk + +.PHONY: $(BUILD_MK_FILE) + +wipe_existing_archives: + $(VERBOSE)rm -rf $(addprefix $(DEPOT_DIR)/,\ + $(foreach A,${ARCHIVES(bin)},$(call dst_archive_path,$A))) + +$(BUILD_MK_FILE): checked_source_archives_exist checked_no_uncategorized + $(VERBOSE)mkdir -p $(dir $@) + $(VERBOSE)( echo -e "all:\n"; \ + echo "TOOL_DIR := $(GENODE_DIR)/tool"; \ + $(foreach A,${ARCHIVES(bin)},\ + target=$(call dst_archive_path,$A); \ + user=$(call archive_user,$A); \ + recipe=$(call bin_archive_recipe,$A); \ + spec=$(call bin_archive_spec,$A); \ + echo ""; \ + echo "TARGETS += $$target"; \ + echo "TOOL($$target) := build_bin_archive"; \ + echo "ARGS($$target) := $$recipe USER=$$user SPEC=$$spec"; \ + ) \ + echo -e "\nall: \$$(TARGETS)"; \ + echo -e "\n\$$(TARGETS):"; \ + echo -e "\t\$$(MAKE) -f \$$(TOOL_DIR)/depot/mk/\$${TOOL(\$$@)}" \ + "\$${ARGS(\$$@)} VERBOSE=\$$(VERBOSE)\n"; \ + ) > $@ + + +# +# Invoke sub make to process generated makefile +# +execute_generated_build_mk_file: $(BUILD_MK_FILE) + $(VERBOSE)$(MAKE) $(if $(VERBOSE),--quiet) -f $(BUILD_MK_FILE) \ + -C $(DEPOT_DIR) VERBOSE=$(VERBOSE) + +ifneq ($(FORCE),) +execute_generated_build_mk_file: wipe_existing_archives +endif + +$(MAKECMDGOALS): execute_generated_build_mk_file + @true + diff --git a/tool/depot/create b/tool/depot/create new file mode 100755 index 000000000..3bcefba24 --- /dev/null +++ b/tool/depot/create @@ -0,0 +1,61 @@ +#!/usr/bin/make -f + +# +# \brief Tool for assembling API/source/binary archives +# \author Norman Feske +# \date 2017-03-16 +# + +define HELP_MESSAGE + + Populate depot with source and binary archives based of the current + version of the Genode source tree + + usage: + + $(firstword $(MAKEFILE_LIST)) ... + + This tool is a front end to the 'extract' and 'build' tools. + It accepts an arbitrary number of archives without their version + suffix as arguments. Furthermore, it supports the supplemental + arguments of those tools (like VERBOSE, FORCE, -j). + + The 'create' tool first invokes the 'extract' tool to create the + API/source/package/raw archives needed for the specified archives. + This step is followed by the invokation of the 'build' tool with + archive arguments that match their current versions. Combined + with the 'UPDATE_VERSIONS=1' argument, it thereby allows for the + source-archive creation, version updating, and building of binary + archives via a single command. + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..) + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc + + +.PHONY: extract build + +extract: + $(VERBOSE)$(MAKE) -f $(GENODE_DIR)/tool/depot/extract $(MAKECMDGOALS) \ + VERBOSE=$(VERBOSE) FORCE=$(FORCE) \ + UPDATE_VERSIONS=$(UPDATE_VERSIONS) \ + + +_versioned_src_of_bin = $1-$(call recipe_version,src/$(call bin_archive_recipe,$1)) +_versioned_pkg = $1-$(call recipe_version,pkg/$(call bin_archive_recipe,$1)) + +versioned_archive = $(if $(call archive_has_type,$1,bin),$(call _versioned_src_of_bin,$1),\ + $(if $(call archive_has_type,$1,pkg),$(call _versioned_pkg,$1))) + +build: extract + $(VERBOSE)$(MAKE) -f $(GENODE_DIR)/tool/depot/build \ + $(foreach A,$(MAKECMDGOALS),$(call versioned_archive,$A))\ + VERBOSE=$(VERBOSE) FORCE=$(FORCE) \ + KEEP_BUILD_DIR=$(KEEP_BUILD_DIR) + + +$(MAKECMDGOALS): build + @true + diff --git a/tool/depot/dependencies b/tool/depot/dependencies new file mode 100755 index 000000000..06800b32b --- /dev/null +++ b/tool/depot/dependencies @@ -0,0 +1,63 @@ +#!/usr/bin/make -f + +# +# \brief Tool for determining the dependencies of depot content +# \author Norman Feske +# \date 2017-03-17 +# + +define HELP_MESSAGE + + Show the dependencies of depot content + + usage: + + $(firstword $(MAKEFILE_LIST)) ... + + This tool operates solely on the depot content. It prints the + result only if all dependencies are present within the depot. + Otherwise, the missing archives are given as an error message. + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..) + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc +include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc + + +# +# Collect dependencies for all specified arguments +# + +api_file = $(addsuffix /api,$(addprefix $(DEPOT_DIR)/,$1)) + +used_apis_file = $(addsuffix /used_apis,$(addprefix $(DEPOT_DIR)/,$1)) + +_pkg_archives_of_type = $(call grep_archive_type,$1,\ + $(call file_content,\ + $(addsuffix /archives,$(addprefix $(DEPOT_DIR)/,$2)))) + +pkg_src_archives = $(call _pkg_archives_of_type,src,$1) +pkg_raw_archives = $(call _pkg_archives_of_type,raw,$1) +pkg_pkg_archives = $(call _pkg_archives_of_type,pkg,$1) + +include $(GENODE_DIR)/tool/depot/mk/dependencies.inc + + +# +# Print gathered information +# + +NEEDED_ARCHIVES := $(foreach TYPE,pkg src raw api bin,${ARCHIVES(${TYPE})}) +MISSING_ARCHIVES := $(sort $(foreach A,$(NEEDED_ARCHIVES),\ + $(if $(wildcard $(addprefix $(DEPOT_DIR)/,$A)),,$A))) +checked_completeness: +ifneq ($(MISSING_ARCHIVES),) + @echo "Error: incomplete or missing archives:"; \ + for i in $(MISSING_ARCHIVES); do echo " $$i"; done; false +endif + +$(MAKECMDGOALS): checked_completeness + @for i in $(NEEDED_ARCHIVES); do echo $$i; done + diff --git a/tool/depot/download b/tool/depot/download new file mode 100755 index 000000000..c48665c61 --- /dev/null +++ b/tool/depot/download @@ -0,0 +1,42 @@ +#!/usr/bin/make -f + +# +# \brief Download packages +# \author Norman Feske +# \date 2017-03-23 +# + +define HELP_MESSAGE + + Download, verify, and uncompress depot content + + usage: + + $(firstword $(MAKEFILE_LIST)) {PUBLIC=} + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..) + +PUBLIC_DIR ?= $(GENODE_DIR)/public + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc + +# sanitize arguments +ARGS := $(subst ..,__,$(MAKECMDGOALS)) + +DEPENDENCIES_CMD = $(GENODE_DIR)/tool/depot/dependencies $(ARGS) +DOWNLOAD_CMD = $(GENODE_DIR)/tool/depot/mk/cp_downloader VERBOSE=$(VERBOSE) + +.PHONY: download +download: + $(VERBOSE)\ + while true; do \ + if $(DEPENDENCIES_CMD) > /dev/null 2> /dev/null; then break; fi; \ + missing_deps=`$(DEPENDENCIES_CMD) 2> /dev/null | sed -n "/^ /s/ *//p"`; \ + $(DOWNLOAD_CMD) $$missing_deps || break; \ + done; + +$(MAKECMDGOALS): download + @true + diff --git a/tool/depot/extract b/tool/depot/extract new file mode 100755 index 000000000..307c5a245 --- /dev/null +++ b/tool/depot/extract @@ -0,0 +1,182 @@ +#!/usr/bin/make -f + +# +# \brief Extract API/source/binary archives from Genode source tree +# \author Norman Feske +# \date 2017-03-16 +# + +define HELP_MESSAGE + + Extract API/source/raw archives from the Genode source tree + + usage: + + $(firstword $(MAKEFILE_LIST)) ... + + The argument denotes the archive to extract in the + form of a path. The first path element correponds to the identity + of the archive creator, the second element corresponds to the type + of the archive, and the third element refers to the recipe of + the archive description. + + E.g., the user 'alan' may create the following archives: + + alan/api/libc - an API archive for the libc + alan/src/zlib - a source archive for the zlib library + + The following arguments tweak the operation of the tool: + + FORCE=1 Replace existing archives with freshly created + ones. This is useful during the development of + recipes. + + VERBOSE= Show individual operations. + + -j Enable the parallel creation of packages where + denotes the level of parallelism. + + UPDATE_VERSIONS=1 Automatically increase the version of recipe + hash files whenever their respective archive + content has changed. The versions are named + after the current date, suffixed with a single + letter to differentiate multiple versions + created on the same day. + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..) + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc +include $(GENODE_DIR)/tool/depot/mk/categorize_args.inc + + +# +# Collect dependencies for all specified arguments +# +# The following accessor functions used by 'mk/dependencies.inc'. The +# information found in the 'archives' file of a package recipe has the +# placeholder '_' for the user. Only archives with this placeholder are +# considered. The '_user_pkg_archives' function transforms those archive paths +# into user-specific archive paths. +# + +_file_in_depot = $(wildcard $(DEPOT_DIR)/$(call archive_user,$1)/src/$(call archive_recipe,$1)/$2) +_file_in_recipe = $(addsuffix /$2,$(call recipe_dir,src/$(call archive_recipe,$1))) + +_file_in_depot_or_recipe = $(if $(call _file_in_depot,$1,$2),\ + $(call _file_in_depot,$1,$2),\ + $(call _file_in_recipe,$1,$2)) + +api_file = $(call _file_in_depot_or_recipe,$1,api) +used_apis_file = $(call _file_in_depot_or_recipe,$1,used_apis) + +_pkg_archives_file = $(call recipe_dir,pkg/$(call archive_recipe,$1))/archives +_user_pkg_archives = $(patsubst _/%,$(call archive_user,$1)/%,\ + $(call grep_archive_user,_,\ + $(call file_content,$(call _pkg_archives_file,$1)))) + +pkg_src_archives = $(call grep_archive_type,src,$(call _user_pkg_archives,$1)) +pkg_raw_archives = $(call grep_archive_type,raw,$(call _user_pkg_archives,$1)) +pkg_pkg_archives = $(call grep_archive_type,pkg,$(call _user_pkg_archives,$1)) + +include $(GENODE_DIR)/tool/depot/mk/dependencies.inc + + +# +# Obtain version information from recipes +# +# The 'archive_version' function takes the archive type and name as arguments +# and returns the version identifier as present in the corresponding recipe. +# The nested foreach loop populates 'ARCHIVE_VERSION' with the version +# identifier for each archive. +# +# If an archive is given with a complete (versioned) name, we don't need to +# consult any recipe but only check if the corresponding archive exists within +# the depot. For binary archives, it suffices that the corresponding source +# archive is present. +# + +$(foreach TYPE,api src raw pkg,\ + $(foreach PATH,${ARCHIVES(${TYPE})},\ + $(eval ARCHIVE_VERSION(${PATH}) := $(call archive_version,$(PATH))))) + +archive_exists_in_depot = $(wildcard $(DEPOT_DIR)/$1) + +ARCHIVES_WITH_NO_VERSION := $(sort \ + $(foreach TYPE,api src raw pkg,\ + $(foreach A,${ARCHIVES(${TYPE})},\ + $(if $(call archive_exists_in_depot,$A),,\ + $(if ${ARCHIVE_VERSION($A)},,$A))))) + +checked_versions_defined: +ifneq ($(ARCHIVES_WITH_NO_VERSION),) + @echo "Error: incomplete or missing recipe ($(sort $(ARCHIVES_WITH_NO_VERSION)))"; false +endif + + +# +# Generate makefile for archive-extraction stage +# + +# return versioned archive path, if 'ARCHIVE_VERSION' is undefined, assume +# that the argument is already a versiond path +versioned_archive = $(if $(ARCHIVE_VERSION($1)),$(addsuffix -${ARCHIVE_VERSION($1)},$1),$1) + +EXTRACT_MK_FILE := $(DEPOT_DIR)/var/extract.mk + +.PHONY: $(EXTRACT_MK_FILE) + +wipe_existing_archives: + $(VERBOSE)rm -rf $(addprefix $(DEPOT_DIR)/, $(foreach TYPE,api src raw pkg,\ + $(foreach A,${ARCHIVES(${TYPE})},\ + $(call versioned_archive,$A)))) + +$(EXTRACT_MK_FILE): checked_versions_defined checked_no_uncategorized + $(VERBOSE)mkdir -p $(dir $@) + $(VERBOSE)( echo -e "all:\n"; \ + echo "TOOL_DIR := $(GENODE_DIR)/tool"; \ + $(foreach TYPE,api src raw pkg,\ + $(foreach A,${ARCHIVES(${TYPE})},\ + target=$(call versioned_archive,$A); \ + user=$(call archive_user,$A); \ + recipe=$(call archive_recipe,$A); \ + echo ""; \ + echo "ARCHIVES(${TYPE}) += $$target"; \ + echo "TOOL($$target) := extract_$(TYPE)_archive"; \ + echo "ARGS($$target) := $$recipe USER=$$user"; \ + ) ) \ + echo -e ""; \ + $(foreach A,${ARCHIVES(pkg)},\ + $(foreach DEP,$(call pkg_pkg_archives,$A),\ + echo -e "$(call versioned_archive,$A) :" \ + "$(call versioned_archive,$(DEP))";)) \ + echo -e ""; \ + echo -e "\$${ARCHIVES(src)} : \$${ARCHIVES(api)}"; \ + echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(api)}"; \ + echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(src)}"; \ + echo -e "\$${ARCHIVES(pkg)} : \$${ARCHIVES(raw)}"; \ + echo -e "\nTARGETS := \$$(foreach T,api src raw pkg,\$${ARCHIVES(\$$T)})"; \ + echo -e "\nall: \$$(TARGETS)"; \ + echo -e "\n\$$(TARGETS):"; \ + echo -e "\t\$$(MAKE) -f \$$(TOOL_DIR)/depot/mk/\$${TOOL(\$$@)}" \ + "\$${ARGS(\$$@)}" \ + "VERBOSE=\$$(VERBOSE)" \ + "UPDATE_VERSIONS=\$$(UPDATE_VERSIONS)\n"; \ + ) > $@ + +# +# Invoke sub make to process generated makefile +# +execute_generated_extract_mk_file: $(EXTRACT_MK_FILE) + $(VERBOSE)$(MAKE) $(if $(VERBOSE),--quiet) -f $(EXTRACT_MK_FILE) \ + -C $(DEPOT_DIR) VERBOSE=$(VERBOSE) \ + UPDATE_VERSIONS=$(UPDATE_VERSIONS) + +ifneq ($(FORCE),) +execute_generated_extract_mk_file: wipe_existing_archives +endif + +$(MAKECMDGOALS): execute_generated_extract_mk_file + @true + diff --git a/tool/depot/mk/build_bin_archive b/tool/depot/mk/build_bin_archive new file mode 100755 index 000000000..f18b360e9 --- /dev/null +++ b/tool/depot/mk/build_bin_archive @@ -0,0 +1,219 @@ +#!/usr/bin/make -f + +# +# \brief Tool for building a binary archive from source +# \author Norman Feske +# \date 2016-05-17 +# + +define HELP_MESSAGE + + Build a binary archive from source + + usage: + + $(firstword $(MAKEFILE_LIST)) SPEC= USER= + + name of the source archive to build + build spec, e.g., x86_32, x86_64 + identity of the archive creator + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../../..) + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc + + +# +# The target is the name of the archive +# +ARCHIVE := $(TARGET) + +# +# Define location of source archive +# +RECIPE_DIR := $(call recipe_dir,src/$(ARCHIVE)) +REP_DIR := $(RECIPE_DIR:/recipes/src/$(ARCHIVE)=) +DEPOT_API_DIR := $(DEPOT_DIR)/$(USER)/api +DEPOT_SRC_DIR := $(DEPOT_DIR)/$(USER)/src +DEPOT_BIN_DIR := $(DEPOT_DIR)/$(USER)/bin + + +# +# Look up hash of the source archive from the src recipe +# + +EXPECTED_SRC_HASH_FILE := $(RECIPE_DIR)/hash +SRC_HASH_FILE := $(wildcard $(EXPECTED_SRC_HASH_FILE)) + +checked_src_hash_file: +ifeq ($(SRC_HASH_FILE),) + @$(ECHO) "Error: source-archive hash file missing," + @$(ECHO) " expected at '$(EXPECTED_SRC_HASH_FILE)'" + @false +else + +SRC_HASH_FILE_CONTENT := $(shell cat $(SRC_HASH_FILE)) +SRC_VERSION := $(firstword $(SRC_HASH_FILE_CONTENT)) +endif + + +# +# Look for source archive +# +# First try to locate the source archive in the depot, which succeeds only +# if the archive is given with a versioned name. The archive version is +# unspecified, we look up the version information from the archive recipe. +# + +VERSIONED_ARCHIVE := $(ARCHIVE) + +SRC_DIR := $(DEPOT_SRC_DIR)/$(VERSIONED_ARCHIVE) +SRC_DIR := $(if $(wildcard $(SRC_DIR)),$(SRC_DIR),) + +checked_src_archive: + @true + +ifeq ($(SRC_DIR),) +VERSIONED_ARCHIVE := $(ARCHIVE)-$(SRC_VERSION) +SRC_DIR := $(DEPOT_SRC_DIR)/$(VERSIONED_ARCHIVE) +checked_src_archive: checked_src_hash_file +endif + +ifeq ($(wildcard $(SRC_DIR)),) +checked_src_archive: error_missing_source_archive +endif + +error_missing_source_archive: + @$(ECHO) "Error: missing source archive $(SRC_DIR)" + @false + + +# +# Check for missing SPEC argument +# + +checked_spec_argument: +ifeq ($(SPEC),) + @$(ECHO) "Error: missing SPEC argument" + @false +endif + + +# +# Look for src/api to determine whether to build a library or a target. +# If building a library, concatenate archive dir as /. +# Otherwise use has archive dir. +# + +API_FILE := $(wildcard $(addsuffix /api,$(SRC_DIR))) + +ifneq ($(API_FILE),) +API := $(call file_content,$(API_FILE)) +DEPOT_ARCHIVE_DIR := $(DEPOT_BIN_DIR)/$(SPEC)/$(API)/$(VERSIONED_ARCHIVE) +else +DEPOT_ARCHIVE_DIR := $(DEPOT_BIN_DIR)/$(SPEC)/$(VERSIONED_ARCHIVE) +endif + +DEPOT_ARCHIVE_BUILD_DIR := $(addsuffix .build,$(DEPOT_ARCHIVE_DIR)) + + +# +# Create archive build directory, which corresponds to a Genode build directory +# +# etc/build.conf: REPOSITORIES point to the source archive and all used api +# archive. The list of used api archive comes from the '/used_apis' +# file. +# + +# if building a library, always incorporate the API implemented by the library +USED_APIS += $(API) + +# incorporate all APIs used by the source archive +USED_APIS_FILE := $(SRC_DIR)/used_apis +ifneq ($(wildcard $(USED_APIS_FILE)),) +USED_APIS += $(shell cat $(USED_APIS_FILE)) +endif + +BUILD_CONF := $(DEPOT_ARCHIVE_BUILD_DIR)/etc/build.conf +SPECS_CONF := $(DEPOT_ARCHIVE_BUILD_DIR)/etc/specs.conf +TOOLS_CONF := $(DEPOT_ARCHIVE_BUILD_DIR)/etc/tools.conf +BUILD_MK := $(DEPOT_ARCHIVE_BUILD_DIR)/Makefile + +# validate that all API archives exist +USED_API_DIRS := $(addprefix $(DEPOT_API_DIR)/, $(USED_APIS)) +MISSING_API_DIRS := $(filter-out $(wildcard $(USED_API_DIRS)), $(USED_API_DIRS)) + +checked_api_archives: +ifneq ($(MISSING_API_DIRS),) + @($(ECHO) "Error: The following API archives are missing:"; \ + for api in $(MISSING_API_DIRS); do $(ECHO) " " $$api; done); + @false +endif + +$(BUILD_CONF): checked_src_archive checked_api_archives + $(VERBOSE)mkdir -p $(dir $@) + $(VERBOSE) \ + ( echo "GENODE_DIR := $(GENODE_DIR)"; \ + echo "BASE_DIR := $(GENODE_DIR)/repos/base"; \ + echo "REPOSITORIES := $(SRC_DIR)"; \ + for api in $(USED_APIS); do \ + echo "REPOSITORIES += $(DEPOT_API_DIR)/$$api"; done \ + ) > $(BUILD_CONF) + +$(SPECS_CONF): checked_spec_argument + $(VERBOSE)mkdir -p $(dir $@) + $(VERBOSE)echo "SPECS += genode $(SPEC)" > $@ + +$(TOOLS_CONF): + $(VERBOSE)mkdir -p $(dir $@) + $(VERBOSE)cp $(GENODE_DIR)/repos/base/etc/tools.conf $@ + +$(BUILD_MK): + $(VERBOSE)ln -sf $(GENODE_DIR)/tool/builddir/build.mk $@ + +$(BUILD_CONF) $(SPECS_CONF) $(TOOLS_CONF) $(BUILD_MK): wiped_build_dir + +wiped_build_dir: + $(VERBOSE)rm -rf $(DEPOT_ARCHIVE_BUILD_DIR) + + +# +# Invoke the Genode build system in the build directory +# + +ifeq ($(KEEP_BUILD_DIR),) +RM_BUILD_DIR_CMD := rm -rf $(DEPOT_ARCHIVE_BUILD_DIR) +else +RM_BUILD_DIR_CMD := true +endif + +$(DEPOT_ARCHIVE_BUILD_DIR)/bin: $(BUILD_CONF) $(SPECS_CONF) $(TOOLS_CONF) $(BUILD_MK) + $(VERBOSE)$(MAKE) -C $(DEPOT_ARCHIVE_BUILD_DIR) $(BUILD_ARG) ||\ + ( $(RM_BUILD_DIR_CMD); false ) + + +# +# Extract build results from build directory into binary-archive directory +# + +$(DEPOT_ARCHIVE_DIR): $(DEPOT_ARCHIVE_BUILD_DIR)/bin + $(VERBOSE)mkdir -p $@ + $(VERBOSE)find $< -maxdepth 0 -not -empty -exec cp -rL $)' variables +# with the matching paths. +# +# + +categorize_archives = $(foreach PATH,$(ARCHIVE_PATHS),\ + $(if $(call archive_has_type,$(PATH),$1),\ + $(eval ARCHIVES($1) += $(PATH)))) + +$(foreach TYPE,$(ARCHIVE_TYPES),$(call categorize_archives,$(TYPE))) + +CATEGORIZED := $(foreach TYPE,$(ARCHIVE_TYPES),${ARCHIVES(${TYPE})}) +UNCATEGORIZED := $(filter-out $(CATEGORIZED),$(ARCHIVE_PATHS)) + +checked_no_uncategorized: +ifneq ($(UNCATEGORIZED),) + @echo "Error: unknown archive type ($(UNCATEGORIZED))"; false +endif + + +# +# Sub-categorize source-pkg archives (/pkg/) from binary-pkg +# archives (/pkg//) so that 'ARCHIVES(pkg)' contains source +# pkgs only, and 'ARCHIVES(binpkg)' contains binary pkgs. +# +# If the path contains 4 elements, it refers to a binary pkg where the third +# element is the build spec. Otherwise, the path refers to a source pkg. +# + +_src_pkg = $(if $(word 4,$(subst /, ,$1)),,$1) +_bin_pkg = $(if $(word 4,$(subst /, ,$1)),$1,) + +ARCHIVES(binpkg) := $(strip $(foreach PKG,${ARCHIVES(pkg)},$(call _bin_pkg,$(PKG)))) +ARCHIVES(pkg) := $(strip $(foreach PKG,${ARCHIVES(pkg)},$(call _src_pkg,$(PKG)))) diff --git a/tool/depot/mk/common.inc b/tool/depot/mk/common.inc new file mode 100644 index 000000000..9b1c38ffc --- /dev/null +++ b/tool/depot/mk/common.inc @@ -0,0 +1,32 @@ +# +# \brief Common environment +# \author Norman Feske +# \date 2014-05-27 +# + +SHELL := bash +VERBOSE ?= @ +ECHO := echo -e +HASHSUM := sha1sum + +MAKEFLAGS += --no-print-directory + +BRIGHT_COL ?= \x1b[01;33m +DARK_COL ?= \x1b[00;33m +DEFAULT_COL ?= \x1b[0m + +MSG_PREFIX_TXT := $(DARK_COL)$(TARGET) $(DEFAULT_COL) +MSG_PREFIX := $(ECHO) "$(MSG_PREFIX_TXT)" + +define NEWLINE + + +endef + +EMPTY := + +# +# Utility to read content from a file if it exists and the given file name +# is not empty. +# +file_content = $(if $(wildcard $1),$(shell cat $1),) diff --git a/tool/depot/mk/content_env.mk b/tool/depot/mk/content_env.mk new file mode 100644 index 000000000..e9483861a --- /dev/null +++ b/tool/depot/mk/content_env.mk @@ -0,0 +1,46 @@ +# +# \brief Environment for executing content.mk files +# \author Norman Feske +# \date 2006-05-13 +# +# The file is executed from within the archive directory. +# The following variables must be defined by the caller: +# +# GENODE_DIR - root directory of the Genode source tree +# CONTRIB_DIR - collection of 3rd-party ports, usually $(GENODE_DIR)/contrib +# CONTENT_MK - content.mk file to execute +# VERBOSE - verbosity +# + + +# +# Check presence of argument $1. Back out with error message $2 if not defined. +# +_assert = $(if $1,$1,$(error Error: $2)) + + +# +# Utility to query the port directory for a given path to a port description. +# +# Example: +# +# $(call port_dir,$(GENODE_DIR)/repos/libports/ports/libpng) +# +_hash_of_port = $(shell cat $(call _assert,$(wildcard $1.hash),$(notdir $1) port does not exist)) +_port_dir = $(wildcard $(CONTRIB_DIR)/$(notdir $1)-$(call _hash_of_port,$1)) +_checked_port_dir = $(call _assert,$(call _port_dir,$1),$(notdir $1) port is not prepared or outdated) + +port_dir = $(call _checked_port_dir,$1) + + +# +# Handy shortcuts to be used in content.mk files +# +mirror_from_rep_dir = mkdir -p $(dir $@); cp -r $(REP_DIR)/$@ $(dir $@) + + +# +# Execute recipe's content.mk rules for populating the archive directory +# +include $(CONTENT_MK) + diff --git a/tool/depot/mk/cp_downloader b/tool/depot/mk/cp_downloader new file mode 100755 index 000000000..ce0e9fdee --- /dev/null +++ b/tool/depot/mk/cp_downloader @@ -0,0 +1,71 @@ +#!/usr/bin/make -f + +# +# \brief Simulate the download of files by copying content from a directory +# \author Norman Feske +# \date 2017-03-23 +# + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../../..) + +REMOTE_DIR ?= $(GENODE_DIR)/remote +PUBLIC_DIR ?= $(GENODE_DIR)/public +DEPOT_DIR ?= $(GENODE_DIR)/depot + +define HELP_MESSAGE + + Simulate the download of files by copying content from a directory + + usage: + + $(firstword $(MAKEFILE_LIST)) ... + +endef + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc + +TARGETS := $(addprefix $(DEPOT_DIR)/,$(MAKECMDGOALS)) + + +# +# Unpack after checking signature against public key as stored in the depot +# +# Unfortunately, gpg does not allow us to specify the armored public-key +# file directly as keyring for the verify operation. So we need to create a +# temporary dearmored version. +# + +ARCHIVES := $(MAKECMDGOALS) + +include $(GENODE_DIR)/tool/depot/mk/gpg.inc + +$(DEPOT_DIR)/% : $(PUBLIC_DIR)/%.tgz $(PUBLIC_DIR)/%.tgz.gpg + $(VERBOSE)pubkey_file=$(DEPOT_DIR)/$(call archive_user,$*)/pubkey; \ + gpg --yes -o $$pubkey_file.dearmored --dearmor $$pubkey_file; \ + ( gpg --no-tty --no-default-keyring \ + --keyring $$pubkey_file.dearmored \ + --verify $(PUBLIC_DIR)/$*.tgz.gpg 2> /dev/null; retval=$$?; \ + rm -f $$pubkey_file.dearmored; \ + exit $$retval \ + ) || ( echo -e "Error: could not verify '$*', signature does not match\n" \ + " public key '$$pubkey_file'"; \ + false ) + $(VERBOSE)mkdir -p $(dir $@) + $(VERBOSE)tar xfz $(PUBLIC_DIR)/$*.tgz -C $(dir $@) + +DOWNLOADED_FILES := $(addprefix $(PUBLIC_DIR)/,$(MAKECMDGOALS:=.tgz)) \ + $(addprefix $(PUBLIC_DIR)/,$(MAKECMDGOALS:=.tgz.gpg)) + +.PRECIOUS: $(DOWNLOADED_FILES) + +ifneq ($(MISSING_PUBKEY_FILES),) +$(DOWNLOADED_FILES): missing_pubkey_files +endif + +$(PUBLIC_DIR)/%: + @$(ECHO) "$(DARK_COL)download$(DEFAULT_COL) $*" + $(VERBOSE)mkdir -p $(dir $@) + $(VERBOSE)cp $(REMOTE_DIR)/$* $@ + +$(MAKECMDGOALS): $(TARGETS) + @true diff --git a/tool/depot/mk/dependencies.inc b/tool/depot/mk/dependencies.inc new file mode 100644 index 000000000..fd9d5c675 --- /dev/null +++ b/tool/depot/mk/dependencies.inc @@ -0,0 +1,112 @@ +# +# \brief Dependency-resolution functions +# \author Norman Feske +# \date 2017-03-22 +# +# The following accessor functions must be defined when including this file: +# +# api_file return path to 'api' file of a library source archive +# used_apis_file return path to 'used_apis' file for a source archive +# pkg_src_archives return source archives of a package +# pkg_raw_archives return raw-data archives of a package +# pkg_pkg_archives return package-archive dependencies of a package +# + +# +# Extend ARCHIVES(pkg) with package dependencies +# + +_pkg_archives = $(call pkg_pkg_archives,$1) + +_check_unvisited = $(if $(filter $(VISITED_PKGS),$1),\ + $(error recursive package dependency: $(VISITED_PKGS)),\ + $(eval VISITED_PKGS += $1)) + +_pkg_deps_rec = $(call _check_unvisited,$1) $1 \ + $(foreach PKG,$(call _pkg_archives,$1),\ + $(call _pkg_deps_rec,$(PKG))) + +_pkg_deps = $(foreach PKG,$1,\ + $(eval VISITED_PKGS :=) $(call _pkg_deps_rec,$(PKG))) + +ARCHIVES(pkg) := $(sort ${ARCHIVES(pkg)} $(call _pkg_deps,${ARCHIVES(pkg)})) + + +# +# Extend ARCHIVES(binpkg) with binary-package dependencies +# + +# return list of source archives needed by a given binary package +_binpkg_src_archives = $(call pkg_src_archives,$(call pkg_of_binpkg,$1)) + +# return list of binary archives contained in a binary package +_binpkg_bin_archives = $(foreach S,$(call _binpkg_src_archives,$1),\ + $(call bin_for_src,$(call bin_archive_spec,$1),$S)) + +# return list of package archives contained in a binary package +_binpkg_pkg_archives = $(foreach P,$(call pkg_pkg_archives,$(call pkg_of_binpkg,$1)),\ + $(call binpkg_for_pkg,$(call bin_archive_spec,$1),$P)) + +# override behavior of '_pkg_archives' to work on binary packages +_pkg_archives = $(call _binpkg_pkg_archives,$1) + +ARCHIVES(binpkg) := $(sort ${ARCHIVES(binpkg)} $(call _pkg_deps,${ARCHIVES(binpkg)})) + + +# +# Extend ARCHIVES(pkg) with binary-package dependencies +# + +ARCHIVES(pkg) := $(sort ${ARCHIVES(pkg)} $(foreach P,${ARCHIVES(binpkg)},\ + $(call pkg_of_binpkg,$P))) + +# +# Extend ARCHIVES(bin) with binary-package dependencies +# + +ARCHIVES(bin) := $(sort ${ARCHIVES(bin)} $(foreach P,${ARCHIVES(binpkg)},\ + $(call _binpkg_bin_archives,$P))) + +# +# Extend ARCHIVES(src) with package dependencies +# + +ARCHIVES(src) := $(sort ${ARCHIVES(src)} $(foreach P,${ARCHIVES(pkg)},\ + $(call pkg_src_archives,$P))) + +# +# Extend ARCHIVES(raw) with package dependencies +# + +ARCHIVES(raw) := $(sort ${ARCHIVES(raw)} $(foreach P,${ARCHIVES(pkg)},\ + $(call pkg_raw_archives,$P))) + +# +# Extend ARCHIVES(src) with binary dependencies +# + +ARCHIVES(src) := $(sort ${ARCHIVES(src)}\ + $(foreach B,${ARCHIVES(bin)},\ + $(call src_of_bin,$B))) + +# +# Extend ARCHIVES(api) with the APIs used by ARCHIVES(src) +# + +_used_apis = $(call file_content,$(call used_apis_file,$1)) + +src_api_archives = $(addprefix $(call archive_user,$1)/api/,$(call _used_apis,$1)) + +ARCHIVES(api) := $(sort ${ARCHIVES(api)} $(foreach A,${ARCHIVES(src)},\ + $(call src_api_archives,$A))) + +# +# Extend ARCHIVES(api) with the APIs implemented by library ARCHIVES(src) +# + +_lib_api = $(call file_content,$(call api_file,$1)) + +_lib_api_archive = $(addprefix $(call archive_user,$1)/api/,$(call _lib_api,$1)) + +ARCHIVES(api) := $(sort ${ARCHIVES(api)} $(foreach A,${ARCHIVES(src)},\ + $(call _lib_api_archive,$A))) diff --git a/tool/depot/mk/extract.inc b/tool/depot/mk/extract.inc new file mode 100644 index 000000000..36251e56a --- /dev/null +++ b/tool/depot/mk/extract.inc @@ -0,0 +1,213 @@ +# +# \brief Common steps of creating an archive within the depot +# \author Norman Feske +# \date 2017-03-17 +# +# Variables that must be defined when including this file: +# +# ARCHIVE - archive name, corresponds to recipe name +# RECIPE_DIR - location of the archive recipe within the source tree +# DEPOT_SUB_DIR - archive-type-dependent destination within the depot +# TAG_FILE - file within the archive used as tag for make dependencies +# UPDATE_VERSIONS - update recipe hash file if archive content changed +# + + +## +## Obtain information about the user-specified archive recipe +## +## Validate 'RECIPE_DIR', determine the version and content hash as given in +## the recipe, and define 'RECIPE_HASH_VALUE' and 'RECIPE_VERSION'. Rules using +## those definitions should depend on 'checked_recipe_hash_value_exists'. +## + +checked_recipe_dir_specified: +ifeq ($(RECIPE_DIR),) + @$(ECHO) "Error: recipe for '$(TARGET)' is missing"; false +endif + +$(TARGET): checked_recipe_dir_specified + +checked_recipe_is_unique: checked_recipe_dir_specified +ifneq ($(RECIPE_DIR),$(firstword $(RECIPE_DIR))) + @echo "Error: recipe for '$(TARGET)' is ambiguous, candidates are:";\ + for dir in $(RECIPE_DIR_CANDIDATES); do \ + echo " $$dir"; done; \ + false +RECIPE_DIR_CANDIDATES := $(subst %,&,$(RECIPE_DIR)) +RECIPE_DIR := +endif + +$(TARGET): checked_recipe_is_unique + +# +# Determine hash file of current archive version as defined by the recipe +# + +ifneq ($(RECIPE_DIR),) +EXPECTED_RECIPE_HASH_FILE := $(RECIPE_DIR)/hash +RECIPE_HASH_FILE := $(wildcard $(EXPECTED_RECIPE_HASH_FILE)) +endif + +checked_recipe_hash_file_exists: checked_recipe_is_unique +ifeq ($(RECIPE_HASH_FILE),) + @$(ECHO) "Error: Recipe hash file is missing,\n" \ + " expected at '$(EXPECTED_RECIPE_HASH_FILE)'"; false +endif + +$(TARGET): checked_recipe_hash_file_exists + +# +# Obtain hash value and version identifier from hash file. If no version +# identifier is present, the hash value is taken as version ('lastword' is +# the same as 'firstword'). This is the normal case for source archives. +# + +ifneq ($(RECIPE_HASH_FILE),) +_RECIPE_HASH_FILE_CONTENT = $(call file_content,$(RECIPE_HASH_FILE)) +RECIPE_HASH_VALUE = $(lastword $(_RECIPE_HASH_FILE_CONTENT)) +RECIPE_VERSION = $(firstword $(_RECIPE_HASH_FILE_CONTENT)) +endif + +checked_recipe_hash_value_exists: +ifeq ($(RECIPE_HASH_VALUE),) + @$(ECHO) "Error: archive hash is undefined"; false +endif + +# +# Remember content hash at invocation time, to detect a potential update caused +# by the 'UPDATE_VERSIONS' feature. +# +ORIG_RECIPE_HASH_VALUE := $(RECIPE_HASH_VALUE) + +# +# Define name of temporary archive directory that we use until we know the +# archive hash. +# + +DEPOT_ARCHIVE_DIR := $(DEPOT_SUB_DIR)/$(ARCHIVE).incomplete + +reset_stale_temporary_archive_dir: +ifneq ($(wildcard $(DEPOT_ARCHIVE_DIR)),) + $(VERBOSE)rm -rf $(DEPOT_ARCHIVE_DIR); mkdir -p $(DEPOT_ARCHIVE_DIR) +endif + +$(DEPOT_ARCHIVE_DIR)/$(TAG_FILE): reset_stale_temporary_archive_dir + + +## +## Create archive +## + +# +# Rename archive to the hashed name after successful completion +# +$(ARCHIVE): _rename_to_final_archive + +# +# Rename archive name from the temporary name to the hashed name. However, +# if the hashed archive name already exists, keep the existing one and +# discard the just-built archive. +# +_rename_to_final_archive: _check_hash + @$(VERBOSE)final_name=$(ARCHIVE)-$(RECIPE_VERSION); \ + rm -rf $(DEPOT_SUB_DIR)/$$final_name; \ + mv $(DEPOT_ARCHIVE_DIR) $(DEPOT_SUB_DIR)/$$final_name; \ + hash=$$(< $(DEPOT_ARCHIVE_DIR).hash); hint=""; \ + test $$hash = $(ORIG_RECIPE_HASH_VALUE) ||\ + hint=" $(BRIGHT_COL)(new version)$(DEFAULT_COL)"; \ + rm -f $(DEPOT_ARCHIVE_DIR).hash; \ + $(ECHO) "$(DARK_COL)created$(DEFAULT_COL)" \ + "$(USER)/$(notdir $(DEPOT_SUB_DIR))/$$final_name$$hint"; \ + true; + +# +# Generate suggested version name for 'HASH_OUT_OF_DATE_MESSAGE' +# +# We suggest to use the current date as version. If there is already a version +# with the current date, we add a single-letter suffix to distinguish the new +# version. If we run out of letters or if the old recipe's version cannot be +# compared with the current date, we suggest to append a '-x' to the old +# recipe's version. In the latter case (e.g., when using a versioning scheme +# not based on dates), the package creator may want to manually adjust the +# version identifier anyway. +# +suffix_from_list = $(subst $(EMPTY) $(EMPTY),,$(strip $1)) +suffixed_version = $1$(if $(call suffix_from_list,$2),-$(call suffix_from_list,$2),) + +_string_higher_than = $(filter-out $(firstword $(sort $1 $2)),$1) +_higher_string = $(if $(call _string_higher_than,$1,$2),$1,$2) + +CURRENT_DATE = $(strip $(shell date --iso-8601)) + +_version_higher_than_recipe = $(call _string_higher_than,\ + $(call suffixed_version,$1,$2),$(RECIPE_VERSION)) + +_next_version = $(eval FOUND := $(if $(call _version_higher_than_recipe,$1,$2),\ + $(call suffixed_version,$1,$2)))\ + $(foreach C,a b c d e f g h i j k l m n o p q r s t u v w x y z,\ + $(if $(FOUND),,\ + $(if $(call _version_higher_than_recipe,$1,$2 $C),\ + $(eval FOUND := $(call suffixed_version,$1,$2 $C)))))\ + $(if $(FOUND),$(FOUND),$(addsuffix -x,$(RECIPE_VERSION))) + +next_version = $(strip $(call _next_version,$(CURRENT_DATE))) + +# +# Message presented to the user whenever a recipe hash is out of date +# +define HASH_OUT_OF_DATE_MESSAGE + +Error: $(RECIPE_HASH_FILE) is out of date. + +This error indicates that the archived source code has changed, which should +be reflected by incrementing the archive version and updating the hash of +the recipe. You may update the recipe hash via the following command: + + echo" "$(next_version)" $$hash "> $(RECIPE_HASH_FILE) + +The above command takes the current date as version identifier. Should this +not be your intention, please adjust the hash file manually. + +endef + + +ifeq ($(UPDATE_VERSIONS),) +HASH_MISMATCH_CMD = $(ECHO) "$(subst $(NEWLINE),\n,$(HASH_OUT_OF_DATE_MESSAGE))"; false +else +HASH_MISMATCH_CMD = echo "$(next_version) $$hash" > $(RECIPE_HASH_FILE); +endif + +# +# Check the consistency between the hash of the archive recipe and the actual +# hash of the generated archive. If both hash values differ, we print the +# expected hash value and remove the archive. The user is expected to +# update the recipe hash before attempting the archive creation again. +# +_check_hash: $(DEPOT_ARCHIVE_DIR).hash checked_recipe_hash_value_exists + $(VERBOSE)hash=$$(< $(DEPOT_ARCHIVE_DIR).hash); \ + test $$hash = $(RECIPE_HASH_VALUE) || ($(HASH_MISMATCH_CMD)) + +# +# Shell command used to calculate the hash of an archive +# +# The command is invoked from within the archive directory within the depot. +# +# The 'echo' right before 'cat' is needed to handle the special case where +# 'find' is executed in an empty directory (yielding an empty result), which +# would otherwise prompt 'cat' to block for standard input. +# +HASH_CMD := cd $(DEPOT_ARCHIVE_DIR); \ + echo | cat `find . -type f | LC_COLLATE=C sort` | $(HASHSUM) | sed "s/ .*//" \ + +# +# Generate the hash from the archive content +# +$(DEPOT_ARCHIVE_DIR).hash: $(DEPOT_ARCHIVE_DIR)/$(TAG_FILE) + $(VERBOSE)$(HASH_CMD) > $@ + +$(DEPOT_ARCHIVE_DIR): + $(VERBOSE)mkdir -p $@ + +$(DEPOT_ARCHIVE_DIR)/$(TAG_FILE): $(DEPOT_ARCHIVE_DIR) + diff --git a/tool/depot/mk/extract_api_archive b/tool/depot/mk/extract_api_archive new file mode 100755 index 000000000..248c2db11 --- /dev/null +++ b/tool/depot/mk/extract_api_archive @@ -0,0 +1,54 @@ +#!/usr/bin/make -f + +# +# \brief Tool for assembling an API archive +# \author Norman Feske +# \date 2016-05-13 +# + +define HELP_MESSAGE + + Tool for assembling an API archive + + usage: + + $(firstword $(MAKEFILE_LIST)) USER= + + name of API, usually the name of the library that + implements the API + identity of the archive creator + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../../..) + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc + +# +# The target is the name of the archive +# +ARCHIVE := $(TARGET) +TAG_FILE := LICENSE + +# +# Define location of recipe and the exported archive +# +RECIPE_DIR := $(call recipe_dir,api/$(ARCHIVE)) +REP_DIR := $(RECIPE_DIR:/recipes/api/$(ARCHIVE)=) +DEPOT_SUB_DIR := $(DEPOT_DIR)/$(USER)/api + +# +# Include common archive-creation steps +# +include $(GENODE_DIR)/tool/depot/mk/extract.inc +include $(GENODE_DIR)/tool/depot/mk/extract_content.inc + +# +# Add suplements that are specific for API archives +# +$(DEPOT_ARCHIVE_DIR).hash: $(DEPOT_ARCHIVE_DIR)/lib/mk/$(ARCHIVE).mk + +$(DEPOT_ARCHIVE_DIR)/lib/mk/$(ARCHIVE).mk: + $(VERBOSE)mkdir -p $(dir $@) + $(VERBOSE)touch $@ + diff --git a/tool/depot/mk/extract_content.inc b/tool/depot/mk/extract_content.inc new file mode 100644 index 000000000..002930c6e --- /dev/null +++ b/tool/depot/mk/extract_content.inc @@ -0,0 +1,62 @@ +# +# \brief Rule for populating an archive by evaluating a content.mk file +# \author Norman Feske +# \date 2017-03-17 +# +# This file complements 'create.inc' for the creation of API and source +# archives. +# +# Arguments: +# +# RECIPE_DIR - location of the recipe for the source or API archive +# DEPOT_ARCHIVE_DIR - archive destination within the depot +# TAG_FILE - file within archive used as tag for make dependencies +# GENODE_DIR - root of the Genode source tree +# REP_DIR - source repository where 'RECIPE_DIR' is located +# + +# +# Validate existance of a content.mk file for the archive recipe +# +ifneq ($(RECIPE_DIR),) +EXPECTED_CONTENT_MK_FILE := $(RECIPE_DIR)/content.mk +CONTENT_MK_FILE := $(wildcard $(EXPECTED_CONTENT_MK_FILE)) +endif + +checked_content_mk_exists: checked_recipe_is_unique +ifeq ($(CONTENT_MK_FILE),) + @$(ECHO) "Error: Recipe misses content.mk file,\n" \ + " expected at '$(EXPECTED_CONTENT_MK_FILE)'"; false +endif + +$(TARGET) $(DEPOT_ARCHIVE_DIR)/$(TAG_FILE): checked_content_mk_exists + +# +# Handle the creation of the TAG_FILE (and the population of the archive +# directory as its side effect) as an atomic step. Otherwise a dependent +# rule - in particular the rule for generating the hash - could be +# triggered as soon as the specific tag file appears but before the entire +# sub make is finished with populating the archive directory. +# +.NOTPARALLEL: $(DEPOT_ARCHIVE_DIR)/$(TAG_FILE) + +# +# Assemble archive content by invoking the recipe's content.mk file +# +# If an error (such as a missing installation of a port) occurs during this +# step, remove the incomplete archive before returning an error. +# +QUIET := $(if $(VERBOSE),--quiet) +$(DEPOT_ARCHIVE_DIR)/$(TAG_FILE): $(CONTENT_MK_FILE) + $(VERBOSE)$(MAKE) $(QUIET) -f $(GENODE_DIR)/tool/depot/mk/content_env.mk \ + -C $(DEPOT_ARCHIVE_DIR) \ + CONTENT_MK=$< \ + GENODE_DIR=$(GENODE_DIR) \ + REP_DIR=$(REP_DIR) \ + CONTRIB_DIR=$(GENODE_DIR)/contrib || \ + ( rm -r $(DEPOT_ARCHIVE_DIR); false ) + $(VERBOSE)find $(DEPOT_ARCHIVE_DIR) \( -name "*~" \ + -or -name "*.rej" \ + -or -name "*.orig" \ + -or -name "*.swp" \) -delete + diff --git a/tool/depot/mk/extract_pkg_archive b/tool/depot/mk/extract_pkg_archive new file mode 100755 index 000000000..a0f68273f --- /dev/null +++ b/tool/depot/mk/extract_pkg_archive @@ -0,0 +1,84 @@ +#!/usr/bin/make -f + +# +# \brief Tool for assembling a package archive +# \author Norman Feske +# \date 2017-03-17 +# + +define HELP_MESSAGE + + Tool for assembling a package archive + + usage: + + $(firstword $(MAKEFILE_LIST)) USER= + + name of the package + identity of the archive creator + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../../..) + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc + +# +# The target is the name of the archive +# +ARCHIVE := $(TARGET) +TAG_FILE := README + +# +# Define location of recipe and the exported archive +# +RECIPE_DIR := $(call recipe_dir,pkg/$(ARCHIVE)) +DEPOT_SUB_DIR := $(DEPOT_DIR)/$(USER)/pkg + +# +# Include common archive-creation steps +# +include $(GENODE_DIR)/tool/depot/mk/extract.inc + +# +# Generate 'archives' list with version information +# +_version = $(call recipe_version,$(call archive_type,$1)/$(call archive_recipe,$1)) + +_versioned_entry = _/$(call archive_type,$1)/$(call archive_recipe,$1)-$(call _version,$1) + +VERSIONED_ARCHIVES := $(foreach A,$(call file_content,$(RECIPE_DIR)/archives),\ + $(if $(call archive_has_user,$A,_),$(call _versioned_entry,$A),$A)) + +$(DEPOT_ARCHIVE_DIR).hash: $(DEPOT_ARCHIVE_DIR)/_archives + +$(DEPOT_ARCHIVE_DIR)/_archives: checked_recipe_hash_value_exists + $(VERBOSE)( $(foreach A,$(VERSIONED_ARCHIVES),echo "$A";) ) > $@ + +# +# Copy remaining recipe content to archive as is, except for files with a +# special meaning or backup files. +# +RECIPE_FILES := $(notdir $(wildcard $(RECIPE_DIR)/*)) +RECIPE_FILES := $(patsubst %~,,$(RECIPE_FILES)) +RECIPE_FILES := $(filter-out archives hash,$(RECIPE_FILES)) + +checked_readme_exists: +ifneq ($(filter-out $(RECIPE_FILES),README),) + @$(ECHO) "Error: missing README in package recipe, expected:\n" \ + " $(RECIPE_DIR)/README"; false +endif + +.NOTPARALLEL: $(DEPOT_ARCHIVE_DIR)/$(TAG_FILE) + +$(DEPOT_ARCHIVE_DIR)/$(TAG_FILE): checked_readme_exists + $(VERBOSE)cp $(addprefix $(RECIPE_DIR)/,$(RECIPE_FILES)) $(DEPOT_ARCHIVE_DIR)/ + +# +# Replace the '_' marker in the 'archives' list with the actual user name +# +$(DEPOT_ARCHIVE_DIR)/archives: $(DEPOT_ARCHIVE_DIR).hash $(DEPOT_ARCHIVE_DIR)/_archives + $(VERBOSE)sed "s/^_/$(USER)/" $(DEPOT_ARCHIVE_DIR)/_archives > $@ + $(VERBOSE)rm -f $(DEPOT_ARCHIVE_DIR)/_archives + +_rename_to_final_archive: $(DEPOT_ARCHIVE_DIR)/archives diff --git a/tool/depot/mk/extract_raw_archive b/tool/depot/mk/extract_raw_archive new file mode 100755 index 000000000..58cfd53f3 --- /dev/null +++ b/tool/depot/mk/extract_raw_archive @@ -0,0 +1,48 @@ +#!/usr/bin/make -f + +# +# \brief Tool for assembling a raw-data archive +# \author Norman Feske +# \date 2016-05-13 +# + +define HELP_MESSAGE + + Tool for assembling a raw-data archive + + usage: + + $(firstword $(MAKEFILE_LIST)) USER= + + name of the raw-data archive + identity of the archive creator + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../../..) + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc + +# +# The target is the name of the archive +# +ARCHIVE := $(TARGET) +TAG_FILE := none + +ifeq ($(USER),) +$(error USER undefined) +endif + +# +# Define location of recipe and the exported archive +# +RECIPE_DIR := $(call recipe_dir,raw/$(ARCHIVE)) +REP_DIR := $(RECIPE_DIR:/recipes/raw/$(ARCHIVE)=) +DEPOT_SUB_DIR := $(DEPOT_DIR)/$(USER)/raw + +# +# Include common archive-creation steps +# +include $(GENODE_DIR)/tool/depot/mk/extract.inc +include $(GENODE_DIR)/tool/depot/mk/extract_content.inc + diff --git a/tool/depot/mk/extract_src_archive b/tool/depot/mk/extract_src_archive new file mode 100755 index 000000000..c8254c79d --- /dev/null +++ b/tool/depot/mk/extract_src_archive @@ -0,0 +1,93 @@ +#!/usr/bin/make -f + +# +# \brief Tool for assembling a source archive +# \author Norman Feske +# \date 2016-05-13 +# + +define HELP_MESSAGE + + Tool for assembling a source archive + + usage: + + $(firstword $(MAKEFILE_LIST)) USER= + + name of the source archive + identity of the archive creator + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../../..) + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc + +# +# The target is the name of the archive +# +ARCHIVE := $(TARGET) +TAG_FILE := LICENSE + +ifeq ($(USER),) +$(error USER undefined) +endif + +# +# Define location of recipe and the exported archive +# +RECIPE_DIR := $(call recipe_dir,src/$(ARCHIVE)) +REP_DIR := $(RECIPE_DIR:/recipes/src/$(ARCHIVE)=) +DEPOT_SUB_DIR := $(DEPOT_DIR)/$(USER)/src + +# +# Include common archive-creation steps +# +include $(GENODE_DIR)/tool/depot/mk/extract.inc +include $(GENODE_DIR)/tool/depot/mk/extract_content.inc + +# +# Add used_apis information, supplemented with the current API hashes +# +ifneq ($(wildcard $(RECIPE_DIR)/used_apis),) +$(DEPOT_ARCHIVE_DIR).hash: $(DEPOT_ARCHIVE_DIR)/used_apis +endif + +$(DEPOT_ARCHIVE_DIR)/used_apis: $(DEPOT_ARCHIVE_DIR) +$(DEPOT_ARCHIVE_DIR)/used_apis: $(RECIPE_DIR)/used_apis + $(VERBOSE) \ + for api in $(shell cat $<); do \ + hash_file=$(GENODE_DIR)/repos/**/recipes/api/$$api/hash; \ + if [ ! -f $$hash_file ]; then \ + echo "Error: archive $(ARCHIVE) depends on nonexisting API '$$api',"; \ + echo " expected $$hash_file"; \ + rm -r $(DEPOT_ARCHIVE_DIR); \ + result=false; \ + break; \ + fi; \ + hash_file_content=$$(< $$hash_file); \ + version=$${hash_file_content%% *}; \ + echo "$$api-$$version" >> $@; \ + done; $$result + +# +# If the archive is a library, add the hash of its implemented API +# +ifneq ($(wildcard $(RECIPE_DIR)/api),) +$(DEPOT_ARCHIVE_DIR).hash: $(DEPOT_ARCHIVE_DIR)/api +endif +$(DEPOT_ARCHIVE_DIR)/api: $(DEPOT_ARCHIVE_DIR) +$(DEPOT_ARCHIVE_DIR)/api: $(RECIPE_DIR)/api + $(VERBOSE) \ + api=$$(< $<); \ + hash_file=$(GENODE_DIR)/repos/**/recipes/api/$$api/hash; \ + if [ ! -f $$hash_file ]; then \ + echo "Error: library '$(ARCHIVE)' implements unknown API '$$api',"; \ + echo " expected $$hash_file"; \ + rm -r $(DEPOT_ARCHIVE_DIR); \ + exit -1; \ + fi; \ + hash_file_content=$$(< $$hash_file); \ + version=$${hash_file_content%% *}; \ + echo "$$api-$$version" >> $@; + diff --git a/tool/depot/mk/front_end.inc b/tool/depot/mk/front_end.inc new file mode 100644 index 000000000..b0b1e05c4 --- /dev/null +++ b/tool/depot/mk/front_end.inc @@ -0,0 +1,67 @@ +# +# 'GENODE_DIR' must be defined before including this file +# + +TARGET := $(firstword $(sort $(MAKECMDGOALS))) +.PHONY: $(TARGET) + +include $(GENODE_DIR)/tool/depot/mk/common.inc + +# list of all repositories located at '/repos/' +REPOSITORIES ?= $(shell find $(GENODE_DIR)/repos -follow -mindepth 1 -maxdepth 1 -type d) + +# list of all repositories that contain depot recipes +REP_RECIPES_DIRS := $(wildcard $(addsuffix /recipes,$(REPOSITORIES))) + +DEPOT_DIR := $(GENODE_DIR)/depot + +usage: + @$(ECHO) "$(subst $(NEWLINE),\n,$(HELP_MESSAGE))"; + + +# +# Helper functions +# + +# function for looking up a recipe directory from one of the repositories +recipe_dir = $(wildcard $(addsuffix /$1,$(REP_RECIPES_DIRS))) + +# function for returning the archive version as given in the recipe +recipe_version = $(firstword $(call file_content,$(addsuffix /hash,$(call recipe_dir,$1)))) + + +# +# Accessor functions for various elements of archive paths +# + +sanitized = $(subst ..,__,$1) + +path_element = $(call sanitized,$(word $1,$(subst /, ,$2))) + +archive_user = $(call path_element,1,$1) +archive_type = $(call path_element,2,$1) +archive_recipe = $(call path_element,3,$1) +archive_has_type = $(filter $(call archive_type,$1),$2) +archive_has_user = $(filter $(call archive_user,$1),$2) +archive_version = $(call recipe_version,$(addprefix $(call archive_type,$1)/,$(call archive_recipe,$1))) + +# binary archives have the form /bin//{-} +bin_archive_spec = $(call path_element,3,$1) +bin_archive_recipe = $(call path_element,4,$1) +bin_archive_version = $(call recipe_version,src/$(call bin_archive_recipe,$1)) + +grep_archive_type = $(foreach A,$2,$(if $(call archive_has_type,$A,$1),$A,)) +grep_archive_user = $(foreach A,$2,$(if $(call archive_has_user,$A,$1),$A,)) + +# return pkg-archive path of given binary-pkg archive path +pkg_of_binpkg = $(call archive_user,$1)/pkg/$(call bin_archive_recipe,$1) + +# return binary-archive path for architecture $1 and source archive $2 +bin_for_src = $(call archive_user,$2)/bin/$1/$(call archive_recipe,$2) + +# return source-archive path for given binary-archive path +src_of_bin = $(call archive_user,$1)/src/$(call bin_archive_recipe,$1) + +# return binary-package archive path for architecture $1 and package archive $2 +binpkg_for_pkg = $(call archive_user,$2)/bin/$1/$(call archive_recipe,$2) + diff --git a/tool/depot/mk/gpg.inc b/tool/depot/mk/gpg.inc new file mode 100644 index 000000000..a432ded9d --- /dev/null +++ b/tool/depot/mk/gpg.inc @@ -0,0 +1,21 @@ +# +# \brief Helper for using the GNU privacy guard +# \author Norman Feske +# \date 2017-03-27 +# + +pubkey_filename = $(call archive_user,$1)/pubkey +pubkey_path = $(wildcard $(DEPOT_DIR)/$(call pubkey_filename,$1)) + +# obtain key ID of 'depot//pubkey' to be used to select signing key +pubkey_id = $(shell gpg --with-colon < $(call pubkey_path,$1) | head -1 | cut -d: -f5 ) + +MISSING_PUBKEY_FILES := $(sort \ + $(foreach A,$(ARCHIVES),\ + $(if $(call pubkey_path,$A),,\ + $(DEPOT_DIR)/$(call pubkey_filename,$A)))) + +missing_pubkey_files: + @echo "Error: missing public-key files:";\ + for i in $(MISSING_PUBKEY_FILES); do echo " $$i"; done; false + diff --git a/tool/depot/publish b/tool/depot/publish new file mode 100755 index 000000000..6b9d0db24 --- /dev/null +++ b/tool/depot/publish @@ -0,0 +1,87 @@ +#!/usr/bin/make -f + +# +# \brief Tool for assembling a package archive +# \author Norman Feske +# \date 2017-03-17 +# + +define HELP_MESSAGE + + Compress and sign depot content for publishing + + usage: + + $(firstword $(MAKEFILE_LIST)) {PUBLIC=} + + The denotes the archives (and implicitly their + dependencies) to publish from the depot to the public directory. + It must be given including the version number of the package archive. + + This tool does not touch any Genode source repository. It solely + reads from the depot and writes to the public directory. + + The optional 'PUBLIC' argument defines the location of the public + directory. If not specified, '/public/' is used. + +endef + +export GENODE_DIR := $(realpath $(dir $(MAKEFILE_LIST))/../..) + +PUBLIC_DIR ?= $(GENODE_DIR)/public + +include $(GENODE_DIR)/tool/depot/mk/front_end.inc + + +# +# Determine dependencies, check for completeness +# + +ifneq ($(MAKECMDGOALS),) +DEPENDENCIES_CMD := $(GENODE_DIR)/tool/depot/dependencies $(MAKECMDGOALS) +DEPENDENCIES_RESULT := $(shell $(DEPENDENCIES_CMD) 2> /dev/null || true) +endif + +ifeq ($(filter Error:,$(DEPENDENCIES_RESULT)),) +ARCHIVES := $(DEPENDENCIES_RESULT) +else +ARCHIVES := +$(MAKECMDGOALS): dependencies_error +endif + +# re-execute the dependencies command to present the error to the user +dependencies_error: + @$(DEPENDENCIES_CMD) + + +# +# Generate compressed and signed archives +# + +include $(GENODE_DIR)/tool/depot/mk/gpg.inc + +MISSING_PUBKEY_FILES := $(sort \ + $(foreach A,$(ARCHIVES),\ + $(if $(call pubkey_path,$A),,\ + $(DEPOT_DIR)/$(call pubkey_filename,$A)))) + +TARGETS := $(addsuffix .tgz.gpg,$(addprefix $(PUBLIC_DIR)/,$(ARCHIVES))) + +$(PUBLIC_DIR)/%.tgz.gpg : $(PUBLIC_DIR)/%.tgz + $(VERBOSE)rm -f $@; + $(VERBOSE)gpg --sign --no-tty --use-agent --local-user $(call pubkey_id,$*) $< + +.PRECIOUS: $(TARGETS:.tgz.gpg=.tgz) + +$(PUBLIC_DIR)/%.tgz: $(DEPOT_DIR)/% + @$(ECHO) "$(DARK_COL)publish$(DEFAULT_COL) $@" + $(VERBOSE)test -e $(dir $@) || mkdir -p $(dir $@) + $(VERBOSE)tar czf $@ -C $(dir $<) $(notdir $<) + +ifneq ($(MISSING_PUBKEY_FILES),) +$(MAKECMDGOALS) $(TARGETS): missing_pubkey_files +endif + +$(MAKECMDGOALS): $(TARGETS) + @true +