nova: create core threads on boot cpu

The boot CPU is not necessarily 0 as currently assumed for base-nova.
Replace all hard coded values by the actual boot cpu number.

Issue #814

base-nova/src/base/server/server.cc

@@ -253,9 +253,8 @@ Rpc_entrypoint::Rpc_entrypoint(Cap_session *cap_session, size_t stack_size,
 		_tid.ec_sel = ec_cap.local_name();
 	}
 	else {
-		enum { CPU_NO = 0 }; //XXX find out the boot cpu
+		/* tell thread starting code to use boot CPU */
-		/* tell thread starting code on which CPU to let run the server thread */
+		*reinterpret_cast<addr_t *>(stack_top()) = ~0UL;
-		*reinterpret_cast<addr_t *>(stack_top()) = CPU_NO;
 
 		/*
 		 * Required for core threads (creates local EC)

base-nova/src/core/echo.cc

@@ -20,10 +20,10 @@
 
 /* local includes */
 #include <echo.h>
+#include <nova_util.h>
 
 enum {
 	ECHO_STACK_SIZE = 1024,
-	ECHO_CPU_NO     = 0,
 	ECHO_GLOBAL     = false,
 	ECHO_EXC_BASE   = 0
 };
@@ -72,8 +72,9 @@ Echo::Echo(Genode::addr_t utcb_addr)
 
 	/* create echo EC */
 	Genode::addr_t pd_sel = Genode::Platform_pd::pd_core_sel();
-	uint8_t res = create_ec(_ec_sel, pd_sel, ECHO_CPU_NO, utcb_addr,
+	uint8_t res = create_ec(_ec_sel, pd_sel, boot_cpu(), utcb_addr,
-	                        (mword_t)echo_stack_top(), ECHO_EXC_BASE, ECHO_GLOBAL);
+	                        reinterpret_cast<mword_t>(echo_stack_top()),
+	                        ECHO_EXC_BASE, ECHO_GLOBAL);
 
 	/* make error condition visible by raising an unhandled page fault */
 	if (res) { ((void (*)())(res*0x10000UL))(); }

base-nova/src/core/include/nova_util.h

@@ -28,6 +28,24 @@
 
 enum { verbose_local_map = false };
 
+/**
+ * Return boot CPU number. It is required if threads in core should be placed
+ * on the same CPU as the main thread.
+ */
+inline Genode::addr_t boot_cpu()
+{
+	/**
+	 * Initial value of ax and di register, saved by the crt0 startup code
+	 * and SOLELY VALID in 'core' !!!
+	 *
+	 * For x86_32 - __initial_ax contains the number of the boot CPU.
+	 * For x86_64 - __initial_di contains the number of the boot CPU.
+	 */
+	extern Genode::addr_t __initial_ax;
+	extern Genode::addr_t __initial_di;
+
+	return (sizeof(void *) > 4) ? __initial_di : __initial_ax;
+}
 
 /**
  * Establish a mapping

base-nova/src/core/irq_session_component.cc

@@ -67,8 +67,8 @@ class Irq_thread : public Thread_base
 			*sp = reinterpret_cast<addr_t>(_thread_start);
 	
 			/* create global EC */
-			enum { CPU_NO = 0, GLOBAL = true };
+			enum { GLOBAL = true };
-			uint8_t res = create_ec(_tid.ec_sel, pd_sel, CPU_NO,
+			uint8_t res = create_ec(_tid.ec_sel, pd_sel, boot_cpu(),
 			                        utcb, (mword_t)sp, _tid.exc_pt_sel, GLOBAL);
 			if (res != NOVA_OK) {
 				PERR("%p - create_ec returned %d", this, res);
@@ -139,11 +139,10 @@ class Genode::Irq_proxy_component : public Irq_proxy<Irq_thread>
 			}
 
 			/* assign IRQ to CPU */
-			enum { CPU = 0 };
 			addr_t msi_addr = 0;
 			addr_t msi_data = 0;
-			uint8_t res = Nova::assign_gsi(_irq_sel, _dev_mem, CPU, msi_addr, msi_data);
+			uint8_t res = Nova::assign_gsi(_irq_sel, _dev_mem, boot_cpu(),
-
+			                               msi_addr, msi_data);
 			if (res != Nova::NOVA_OK)
 				PERR("Error: assign_pci failed -irq:dev:msi_addr:msi_data "
 				     "%lx:%lx:%lx:%lx", _irq_number, _dev_mem, msi_addr,

base-nova/src/core/platform.cc

@@ -38,8 +38,7 @@ Native_utcb *main_thread_utcb();
 
 
 /**
- * Initial value of esp register, saved by the crt0 startup code
+ * Initial value of esp register, saved by the crt0 startup code.
- *
  * This value contains the address of the hypervisor information page.
  */
 extern addr_t __initial_sp;
@@ -255,14 +254,13 @@ static void init_core_page_fault_handler()
 {
 	/* create echo EC */
 	enum {
-		CPU_NO     = 0,
 		GLOBAL     = false,
 		EXC_BASE   = 0
 	};
 
 	addr_t ec_sel = cap_selector_allocator()->alloc();
 
-	uint8_t ret = create_ec(ec_sel, __core_pd_sel, CPU_NO,
+	uint8_t ret = create_ec(ec_sel, __core_pd_sel, boot_cpu(),
 	                        CORE_PAGER_UTCB_ADDR, core_pager_stack_top(),
 	                        EXC_BASE, GLOBAL);
 	if (ret)
@@ -341,7 +339,8 @@ Platform::Platform() :
 	if (verbose_boot_info) {
 		printf("Hypervisor %s VMX\n", hip->has_feature_vmx() ? "features" : "does not feature");
 		printf("Hypervisor %s SVM\n", hip->has_feature_svm() ? "features" : "does not feature");
-		printf("Hypervisor reports %u CPU%c\n", _cpus, _cpus > 1 ? 's' : ' ');
+		printf("Hypervisor reports %u CPU%c - boot CPU is %lu\n",
+		       _cpus, _cpus > 1 ? 's' : ' ', boot_cpu());
 	}
 
 	/* initialize core allocators */

base-nova/src/core/platform_thread.cc

@@ -326,7 +326,7 @@ Weak_ptr<Address_space> Platform_thread::address_space()
 Platform_thread::Platform_thread(const char *name, unsigned, int thread_id)
 :
 	_pd(0), _pager(0), _id_base(cap_selector_allocator()->alloc(1)),
-	_sel_exc_base(Native_thread::INVALID_INDEX), _cpu_no(0), //XXX find out boot CPU
+	_sel_exc_base(Native_thread::INVALID_INDEX), _cpu_no(boot_cpu()),
 	_is_main_thread(false), _is_vcpu(false)
 {
 	strncpy(_name, name, sizeof(_name));

base-nova/src/core/thread_start.cc

@@ -84,6 +84,10 @@ void Thread_base::start()
 	addr_t pd_sel   = Platform_pd::pd_core_sel();
 	addr_t cpu_no   = *reinterpret_cast<addr_t *>(stack_top());
 
+	/* server code sets this value */
+	if (cpu_no == ~0UL)
+		cpu_no = boot_cpu();
+
 	/* create local EC */
 	enum { LOCAL_THREAD = false };
 	uint8_t res = create_ec(_tid.ec_sel, pd_sel, cpu_no,