From 015d319fa74b3ea7f51db94780f8e7e43b3b9f82 Mon Sep 17 00:00:00 2001
From: Norman Feske <norman.feske@genode-labs.com>
Date: Mon, 3 Apr 2017 15:39:36 +0200
Subject: [PATCH] run: link all files in [run_dir]/genode to core

This way, files copied from the depot are incorporated in addition to
the files explicitly specified as boot modules.

The patch also adds an automatic check for the validity of the XML
syntax of boot modules ending with '.config'.

Issue #2339
---
 tool/run/boot_dir/fiasco    | 29 ++++++++++++++++---
 tool/run/boot_dir/foc       | 29 ++++++++++++++++---
 tool/run/boot_dir/hw        | 54 ++++++++++++++++++++++++++---------
 tool/run/boot_dir/linux     |  6 +++-
 tool/run/boot_dir/nova      | 56 ++++++++++++++++---------------------
 tool/run/boot_dir/okl4      | 35 ++++++++---------------
 tool/run/boot_dir/pistachio | 31 +++++++++++++++++---
 tool/run/boot_dir/sel4      | 25 +++++++++++++++--
 tool/run/run                | 56 ++++++++++++++++++++++---------------
 9 files changed, 214 insertions(+), 107 deletions(-)

diff --git a/tool/run/boot_dir/fiasco b/tool/run/boot_dir/fiasco
index 4e4fc60d3..74e28a07d 100644
--- a/tool/run/boot_dir/fiasco
+++ b/tool/run/boot_dir/fiasco
@@ -14,13 +14,34 @@ proc core_link_address     { } { return "0x01000000" }
 #
 proc run_boot_dir {binaries} {
 
-	build { lib/ld/fiasco kernel/fiasco }
+	#
+	# Build kernel and dynamic linker on demand, if not yet present in
+	# '[run_dir]/genode/'
+	#
+	set kernel_arg ""
+	set ld_arg     ""
+	if {![file exists [run_dir]/genode/fiasco]}    { set kernel_arg "kernel/fiasco" }
+	if {![file exists [run_dir]/genode/ld.lib.so]} { set ld_arg     "lib/ld/fiasco" }
+	set targets "$kernel_arg $ld_arg"
+
+	if {[llength $targets]} { build $targets }
+
+	if {$kernel_arg != ""} {
+		file copy -force [pwd]/bin/fiasco           [run_dir]/genode/fiasco
+		file copy -force [pwd]/bin/sigma0-fiasco    [run_dir]/genode/sigma0-fiasco
+		file copy -force [pwd]/bin/bootstrap-fiasco [run_dir]/genode/bootstrap-fiasco
+	}
 
 	build_core_image $binaries
 
-	file copy -force kernel/fiasco/kernel    [run_dir]/kernel
-	file copy -force kernel/fiasco/sigma0    [run_dir]/sigma0
-	file copy -force kernel/fiasco/bootstrap [run_dir]/bootstrap
+	#
+	# Move kernel files to distinct location within the boot directory so that
+	# we can remove [run_dir]/genode after the core image has been built.
+	#
+	file copy -force [run_dir]/genode/fiasco           [run_dir]/kernel
+	file copy -force [run_dir]/genode/sigma0-fiasco    [run_dir]/sigma0
+	file copy -force [run_dir]/genode/bootstrap-fiasco [run_dir]/bootstrap
+	exec rm -r [run_dir]/genode
 
 	if {[have_include "image/iso"] || [have_include "image/disk"]} {
 
diff --git a/tool/run/boot_dir/foc b/tool/run/boot_dir/foc
index a93372f47..38be9ca4c 100644
--- a/tool/run/boot_dir/foc
+++ b/tool/run/boot_dir/foc
@@ -57,13 +57,34 @@ proc l4_bin_dir { } {
 
 proc run_boot_dir_x86 {binaries} {
 
-	build { lib/ld/foc kernel/foc }
+	#
+	# Build kernel and dynamic linker on demand, if not yet present in
+	# '[run_dir]/genode/'
+	#
+	set kernel_arg ""
+	set ld_arg     ""
+	if {![file exists [run_dir]/genode/foc]}       { set kernel_arg "kernel/foc" }
+	if {![file exists [run_dir]/genode/ld.lib.so]} { set ld_arg     "lib/ld/foc" }
+	set targets "$kernel_arg $ld_arg"
+
+	if {[llength $targets]} { build $targets }
+
+	if {$kernel_arg != ""} {
+		file copy -force [pwd]/bin/foc           [run_dir]/genode/foc
+		file copy -force [pwd]/bin/sigma0-foc    [run_dir]/genode/sigma0-foc
+		file copy -force [pwd]/bin/bootstrap-foc [run_dir]/genode/bootstrap-foc
+	}
 
 	build_core_image $binaries
 
-	file copy -force [pwd]/bin/foc           [run_dir]/kernel
-	file copy -force [pwd]/bin/sigma0-foc    [run_dir]/sigma0
-	file copy -force [pwd]/bin/bootstrap-foc [run_dir]/bootstrap
+	#
+	# Move kernel files to distinct location within the boot directory so that
+	# we can remove [run_dir]/genode after the core image has been built.
+	#
+	file copy -force [run_dir]/genode/foc           [run_dir]/kernel
+	file copy -force [run_dir]/genode/sigma0-foc    [run_dir]/sigma0
+	file copy -force [run_dir]/genode/bootstrap-foc [run_dir]/bootstrap
+	exec rm -r [run_dir]/genode
 
 	if {[have_include "image/iso"] || [have_include "image/disk"]} {
 
diff --git a/tool/run/boot_dir/hw b/tool/run/boot_dir/hw
index 8c011845c..f17625332 100644
--- a/tool/run/boot_dir/hw
+++ b/tool/run/boot_dir/hw
@@ -41,30 +41,58 @@ proc run_boot_dir {binaries} {
 		close $fh
 	}
 
-	build { lib/ld/hw bootstrap }
+	#
+	# Build bootstrap, the core object, and the dynamic linker on demand
+	#
+	# If those parts were imported from the depot, the build step is skipped.
+	#
+	set bootstrap_arg ""
+	set core_arg      ""
+	set ld_arg        ""
 
-	# boot module list without core
+	if {![file exists [run_dir]/genode/ld.lib.so]}      { set ld_arg        lib/ld/hw    }
+	if {![file exists [run_dir]/genode/core-hw.o]}      { set core_arg      core/hw      }
+	if {![file exists [run_dir]/genode/bootstrap-hw.o]} { set bootstrap_arg bootstrap/hw }
+
+	set build_args "$bootstrap_arg $core_arg $ld_arg"
+	if {[llength $build_args]} { build $build_args }
+
+	if {$ld_arg        != ""} { file copy -force bin/ld-hw.lib.so   [run_dir]/genode/ld.lib.so }
+	if {$core_arg      != ""} { file copy -force bin/core-hw.o      [run_dir]/genode/core-hw.o }
+	if {$bootstrap_arg != ""} { file copy -force bin/bootstrap-hw.o [run_dir]/genode/bootstrap-hw.o }
+
+	#
+	# Copy specified modules to the run directory, excluding core.
+	#
 	set idx [lsearch $binaries "core"]
 	set modules [lreplace $binaries $idx $idx]
-
-	# strip binaries
-	copy_and_strip_genode_binaries_to_run_dir $modules
-	lappend modules "config"
+	copy_genode_binaries_to_run_dir $modules
 
 	puts "core link address is [core_link_address]"
 
-	set core_obj core/hw/core-hw.o
-	set bootstrap_obj bootstrap/hw/bootstrap.o
+	set core_obj      core-hw.o
+	set bootstrap_obj bootstrap-hw.o
 
 	# create core and bootstrap binary without modules for debugging
-	build_core $core_obj      {} [run_dir].core      [core_link_address]
-	build_core $bootstrap_obj {} [run_dir].bootstrap [bootstrap_link_address]
+	build_core [run_dir]/genode/$core_obj      {} [run_dir].core      [core_link_address]
+	build_core [run_dir]/genode/$bootstrap_obj {} [run_dir].bootstrap [bootstrap_link_address]
+
+	# determine modules to be incorporated into the core image
+	set modules [glob -tails -directory [run_dir]/genode/ *]
+	set excluded_modules {}
+	lappend excluded_modules $core_obj $bootstrap_obj
+	foreach excluded $excluded_modules {
+		set modules [lsearch -inline -not -all $modules $excluded] }
+
+	# check syntax of all boot modules named *.config
+	foreach file [glob -nocomplain [run_dir]/genode/*.config] {
+		check_xml_syntax $file }
 
 	# create core binary containing the boot modules
-	build_core $core_obj $modules [run_dir]/genode/core.elf [core_link_address]
+	build_core [run_dir]/genode/$core_obj $modules [run_dir]/genode/core.elf [core_link_address]
 	exec [cross_dev_prefix]strip [run_dir]/genode/core.elf
-	build_core $bootstrap_obj { core.elf } [run_dir]/image.elf [bootstrap_link_address]
-	exec rm -f [run_dir]/genode/core.elf
+	build_core [run_dir]/genode/$bootstrap_obj { core.elf } [run_dir]/image.elf [bootstrap_link_address]
+	exec rm -fr [run_dir]/genode
 	exec [cross_dev_prefix]strip [run_dir]/image.elf
 
 	if {[have_include "image/iso"] || [have_include "image/disk"]} {
diff --git a/tool/run/boot_dir/linux b/tool/run/boot_dir/linux
index e2da4430d..4bb6e0f57 100644
--- a/tool/run/boot_dir/linux
+++ b/tool/run/boot_dir/linux
@@ -8,11 +8,15 @@ proc binary_name_timer     { } { return "linux_timer_drv" }
 #
 proc run_boot_dir {binaries} {
 
-	build { lib/ld/linux }
+	if {![file exists [run_dir]/genode/ld.lib.so]} { build { lib/ld/linux } }
 
 	foreach binary $binaries {
 		set src_binary_path "../../../../bin/[kernel_specific_binary $binary]"
 		exec ln -sf $src_binary_path [run_dir]/genode/$binary }
+
+	# check syntax of all boot modules named *.config
+	foreach file [glob -nocomplain [run_dir]/genode/*.config] {
+		check_xml_syntax $file }
 }
 
 
diff --git a/tool/run/boot_dir/nova b/tool/run/boot_dir/nova
index a55e090fd..32221cbfc 100644
--- a/tool/run/boot_dir/nova
+++ b/tool/run/boot_dir/nova
@@ -2,31 +2,7 @@ proc binary_name_ld_lib_so { } { return "ld-nova.lib.so" }
 proc binary_name_core_o    { } { return "core-nova.o" }
 proc binary_name_timer     { } { return "nova_timer_drv" }
 
-
-##
-# Read the location of the NOVA kernel directory from 'etc/nova.conf'
-#
-proc nova_kernel { } {
-	global _nova_kernel
-
-	if {![info exists _nova_kernel]} {
-		if {[file exists etc/nova.conf]} {
-			set _nova_kernel [exec sed -n "/^NOVA_KERNEL/s/^.*=\\s*//p" etc/nova.conf]
-		} else {
-		        set _nova_kernel "[pwd]/kernel/nova/hypervisor"
-		}
-	}
-	return $_nova_kernel
-}
-
-
-##
-# Return whether nova is provided from the outside
-#
-proc nova_external { } {
-	if {[nova_kernel] == "[pwd]/kernel/nova/hypervisor"} { return 0 }
-	return 1
-}
+proc kernel_files { } { return hypervisor }
 
 
 proc run_boot_string { } {
@@ -36,6 +12,7 @@ proc run_boot_string { } {
 
 proc core_link_address { } { return "0x100000" }
 
+
 proc core_ld_opts { } {
 	set ret { -Wl,-T }
 	lappend ret "-Wl,[genode_dir]/repos/base/src/ld/genode.ld"
@@ -48,21 +25,36 @@ proc core_ld_opts { } {
 #
 proc run_boot_dir {binaries} {
 
-	if {![nova_external]} {
-		build { lib/ld/nova kernel }
-	} else {
-		build { lib/ld/nova }
-	}
+	#
+	# Build kernel-specific targets if needed
+	#
+	# If the run scripts imports the base-nova binary archive, [run_dir] is
+	# already populated, so we can skip the build.
+	#
+	set kernel_arg ""
+	set core_arg   ""
+	set ld_arg     ""
+	if {![file exists [run_dir]/genode/hypervisor]}  { set kernel_arg kernel }
+	if {![file exists [run_dir]/genode/core-nova.o]} { set core_arg   core/nova }
+	if {![file exists [run_dir]/genode/ld.lib.so]}   { set ld_arg     lib/ld/nova }
+
+	set targets "$kernel_arg $core_arg $ld_arg"
+	if {[llength $targets]} { build $targets }
+
+	if {$kernel_arg != ""} { file copy -force bin/hypervisor     [run_dir]/genode/hypervisor }
+	if {$core_arg   != ""} { file copy -force bin/core-nova.o    [run_dir]/genode/core.o }
+	if {$ld_arg     != ""} { file copy -force bin/ld-nova.lib.so [run_dir]/genode/ld.lib.so }
 
 	#
 	# Collect contents of the ISO image
 	#
 	build_core_image $binaries
 
-	puts "using NOVA kernel at [nova_kernel]"
-	exec [cross_dev_prefix]objcopy -O elf32-i386 [nova_kernel] [run_dir]/hypervisor
+	exec [cross_dev_prefix]objcopy -O elf32-i386 [run_dir]/genode/hypervisor [run_dir]/hypervisor
 	exec [cross_dev_prefix]strip [run_dir]/hypervisor
 
+	exec rm -rf [run_dir]/genode
+
 	if {[have_include "image/iso"] || [have_include "image/disk"]} {
 		#
 		# Install isolinux/GRUB files and bender
diff --git a/tool/run/boot_dir/okl4 b/tool/run/boot_dir/okl4
index cb3adbc46..1c9c0c41a 100644
--- a/tool/run/boot_dir/okl4
+++ b/tool/run/boot_dir/okl4
@@ -2,6 +2,8 @@ proc binary_name_ld_lib_so { } { return "ld-okl4.lib.so" }
 proc binary_name_core_o    { } { return "core-okl4.o"    }
 proc binary_name_timer     { } { return "pit_timer_drv"  }
 
+proc kernel_files { } { return okl4 }
+
 
 ##
 # Get the base-okl4 repository
@@ -28,15 +30,6 @@ proc okl4_dir { } {
 }
 
 
-##
-# Return whether okl4 kernel is provided from the outside
-#
-proc okl4_external { } {
-	if {"[okl4_dir]" == "[base_okl4_dir]/contrib/okl4"} { return 0 }
-	return 1
-}
-
-
 set weaver_xml_template {
 	<machine>
 		<word_size size="0x20" />
@@ -113,25 +106,19 @@ proc run_boot_dir {binaries} {
 	global weaver_xml_template
 
 	#
-	# Build kernel if needed
+	# Build kernel and dynamic linker if needed
 	#
-	# Once the kernel is exists, it gets never revisited automatically.
-	# Consequently, when changing the kernel sources, the kernel build must be
-	# issued explicitly via 'make kernel'. This way, the rare case of changing
-	# the kernel does not stand in the way of the everyday's work flow of
-	# executing run scripts as quick as possible.
-	#
-	if {![okl4_external] && ![file exists bin/okl4]} {
-		build { lib/ld/okl4 kernel/okl4 }
-	} else {
-		build { lib/ld/okl4 }
-	}
+	set kernel_arg ""
+	set ld_arg     ""
+	if {![file exists [run_dir]/genode/okl4]}      { set kernel_arg kernel/okl4 }
+	if {![file exists [run_dir]/genode/ld.lib.so]} { set ld_arg     lib/ld/okl4 }
+	set targets "$kernel_arg $ld_arg"
+	if {[llength $targets]} { build $targets }
+	if {$kernel_arg != ""} { file copy -force bin/okl4 [run_dir]/genode/okl4 }
 
 	build_core_image $binaries
 	exec mv [run_dir]/image.elf [run_dir].image
 
-	exec cp bin/okl4 [run_dir]/kernel
-
 	#
 	# Generate ELF weaver config
 	#
@@ -139,7 +126,7 @@ proc run_boot_dir {binaries} {
 	puts $fh {<?xml version="1.0"?>}
 	puts $fh {<!DOCTYPE image SYSTEM "weaver-1.1.dtd">}
 	puts $fh {<image>}
-	regsub okl4_kernel $weaver_xml_template "[run_dir]/kernel" weaver_xml_template
+	regsub okl4_kernel $weaver_xml_template "[run_dir]/genode/okl4" weaver_xml_template
 	regsub core $weaver_xml_template "[run_dir].image" weaver_xml_template
 	puts $fh $weaver_xml_template
 	puts $fh {</image>}
diff --git a/tool/run/boot_dir/pistachio b/tool/run/boot_dir/pistachio
index 7807dc8c8..d2fdbeb36 100644
--- a/tool/run/boot_dir/pistachio
+++ b/tool/run/boot_dir/pistachio
@@ -2,6 +2,7 @@ proc binary_name_ld_lib_so { } { return "ld-pistachio.lib.so" }
 proc binary_name_core_o    { } { return "core-pistachio.o"    }
 proc binary_name_timer     { } { return "pit_timer_drv" }
 
+proc kernel_files { } { return { pistachio sigma0-pistachio kickstart-pistachio } }
 
 proc run_boot_string   { } { return "\n\r\033\\\[1m\033\\\[33mL4Ka::Pistachio -" }
 proc core_link_address { } { return "0x02000000" }
@@ -12,13 +13,35 @@ proc core_link_address { } { return "0x02000000" }
 #
 proc run_boot_dir {binaries} {
 
-	build { lib/ld/pistachio kernel/pistachio }
+	#
+	# Build kernel and dynamic linker on demand, if not yet present in
+	# '[run_dir]/genode/'
+	#
+	set kernel_arg ""
+	set ld_arg     ""
+	if {![file exists [run_dir]/genode/pistachio]} { set kernel_arg "kernel/pistachio" }
+	if {![file exists [run_dir]/genode/ld.lib.so]} { set ld_arg     "lib/ld/pistachio" }
+	set targets "$kernel_arg $ld_arg"
+
+	if {[llength $targets]} { build $targets }
+
+	if {$kernel_arg != ""} {
+		file copy -force bin/pistachio           [run_dir]/genode/pistachio
+		file copy -force bin/sigma0-pistachio    [run_dir]/genode/sigma0-pistachio
+		file copy -force bin/kickstart-pistachio [run_dir]/genode/kickstart-pistachio
+	}
 
 	build_core_image $binaries
 
-	exec cp bin/pistachio           [run_dir]/kernel
-	exec cp bin/sigma0-pistachio    [run_dir]/sigma0
-	exec cp bin/kickstart-pistachio [run_dir]/kickstart
+	#
+	# Move kernel files to distinct location within the boot directory so that
+	# we can remove [run_dir]/genode after the core image has been built.
+	#
+	exec mkdir -p [run_dir]/pistachio
+	file copy -force [run_dir]/genode/pistachio           [run_dir]/kernel
+	file copy -force [run_dir]/genode/sigma0-pistachio    [run_dir]/sigma0
+	file copy -force [run_dir]/genode/kickstart-pistachio [run_dir]/kickstart
+	exec rm -r [run_dir]/genode
 
 	if {[have_include "image/iso"] || [have_include "image/disk"]} {
 
diff --git a/tool/run/boot_dir/sel4 b/tool/run/boot_dir/sel4
index 3abcc0756..ff126fc38 100644
--- a/tool/run/boot_dir/sel4
+++ b/tool/run/boot_dir/sel4
@@ -12,12 +12,31 @@ proc core_link_address { } { return "0x02000000" }
 #
 proc run_boot_dir {binaries} {
 
-	# build sel4 kernel
-	build { lib/ld/sel4 kernel/sel4 }
-	exec cp bin/sel4 [run_dir]/sel4
+	#
+	# Build kernel and dynamic linker on demand, if not yet present in
+	# '[run_dir]/genode/'
+	#
+	set kernel_arg ""
+	set ld_arg     ""
+	if {![file exists [run_dir]/genode/sel4]}      { set kernel_arg "kernel/sel4" }
+	if {![file exists [run_dir]/genode/ld.lib.so]} { set ld_arg     "lib/ld/sel4" }
+	set targets "$kernel_arg $ld_arg"
+
+	if {[llength $targets]} { build $targets }
+
+	if {$kernel_arg != ""} {
+		file copy -force [pwd]/bin/sel4 [run_dir]/genode/sel4
+	}
 
 	build_core_image $binaries
 
+	#
+	# Move kernel files to distinct location within the boot directory so that
+	# we can remove [run_dir]/genode after the core image has been built.
+	#
+	file copy -force [run_dir]/genode/sel4 [run_dir]/sel4
+	exec rm -r [run_dir]/genode
+
 	if {[have_include "image/iso"] || [have_include "image/disk"]} {
 		#
 		# Install isolinux/GRUB files and bender
diff --git a/tool/run/run b/tool/run/run
index c43f09e54..b90989d93 100755
--- a/tool/run/run
+++ b/tool/run/run
@@ -466,16 +466,10 @@ proc kernel_specific_binary { binary } {
 }
 
 
-##
-# Copy the specified binaries from the 'bin/' directory to the run
-# directory and try to strip executables.
-#
-proc copy_and_strip_genode_binaries_to_run_dir { binaries } {
+proc copy_genode_binaries_to_run_dir { binaries } {
 
 	foreach binary $binaries {
-		exec cp bin/[kernel_specific_binary $binary] [run_dir]/genode/$binary
-		catch {
-			exec [cross_dev_prefix]strip [run_dir]/genode/$binary || true }
+		file copy -force bin/[kernel_specific_binary $binary] [run_dir]/genode/$binary
 	}
 }
 
@@ -520,13 +514,16 @@ proc wait_for_output { wait_for_re timeout_value running_spawn_id } {
 
 ##
 # Dummy boot_string procedure
+#
 proc run_boot_string { } { return ""; }
 
+
 ##
 # Fall-back boot_dir module
 #
 # If this function is called someone forgot to include an appropriate boot_dir
 # module. So, we exit with an error.
+#
 proc run_boot_dir { binaries } {
 	puts stderr "Error: boot_dir module missing, e.g., '--include boot_dir/hw'"
 	exit 1
@@ -764,33 +761,48 @@ proc build_core {lib modules target link_address} {
 }
 
 
+##
+# Return kernel-specific files to be excluded from the core image
+#
+proc kernel_files { } { return { } }
+
+
 ##
 # Generate bootable core image containing all boot-modules
 #
-proc build_core_image {binaries} {
+proc build_core_image { modules } {
 
-	# boot module list without core
-	set idx [lsearch $binaries "core"]
-	set modules [lreplace $binaries $idx $idx]
+	set core_obj [kernel_specific_binary core.o]
 
-	# strip binaries
-	copy_and_strip_genode_binaries_to_run_dir $modules
-	lappend modules "config"
+	# replace 'core' with actual core-object name in 'modules' list
+	if {[lsearch $modules "core"] != -1} {
+		set idx [lsearch $modules "core"]
+		set modules [lreplace $modules $idx $idx]
+		lappend modules $core_obj
+	}
 
-	set core_obj bin/[kernel_specific_binary core.o]
+	copy_genode_binaries_to_run_dir $modules
 
 	# create core binary without modules for debugging
-	build_core $core_obj {} [run_dir].core [core_link_address]
+	build_core [run_dir]/genode/$core_obj {} [run_dir].core [core_link_address]
+
+	# determine modules to be incorporated into the core image
+	set modules [glob -nocomplain -tails -directory [run_dir]/genode/ *]
+	set excluded_modules [kernel_files]
+	lappend excluded_modules $core_obj
+	foreach excluded $excluded_modules {
+		set modules [lsearch -inline -not -all $modules $excluded] }
+
+	# check syntax of all boot modules named *.config
+	foreach file [glob -nocomplain [run_dir]/genode/*.config] {
+		check_xml_syntax $file }
 
 	# create core binary containing the boot modules
-	build_core $core_obj $modules [run_dir]/image.elf [core_link_address]
+	build_core [run_dir]/genode/$core_obj $modules [run_dir]/image.elf [core_link_address]
 	exec [cross_dev_prefix]strip [run_dir]/image.elf
 
 	# Save config part of the image.elf for easy inspection
-	exec cp -f  [run_dir]/genode/config [run_dir].config
-
-	# remove individual binaries, which are now contained in 'image.elf'
-	exec rm -rf [run_dir]/genode
+	exec cp -f [run_dir]/genode/config [run_dir].config
 }
 
 source [genode_dir]/tool/run/depot.inc