vbox5: check memory configuration stricter

The required memory of the VMM depends on the VM memory configuration (vbox)
and the available screen resolution. Check for insane configurations in the
beginning to avoid late surprises.

Issue #3216

repos/ports/src/virtualbox5/accloff/sup.cc

@@ -127,7 +127,14 @@ void genode_update_tsc(void (*update_func)(void), unsigned long update_us)
 
 HRESULT genode_setup_machine(ComObjPtr<Machine> machine)
 {
-	return genode_check_memory_config(machine);
+	ULONG memory_vbox;
+	HRESULT rc = machine->COMGETTER(MemorySize)(&memory_vbox);
+	if (FAILED(rc))
+		return rc;
+
+	size_t const vmm_memory = 1024ULL * 1024 * (memory_vbox + 16);
+
+	return genode_check_memory_config(machine, vmm_memory);
 }
 
 

repos/ports/src/virtualbox5/frontend/main.cc

@@ -167,8 +167,7 @@ HRESULT setupmachine(Genode::Env &env)
 
 	static Bstr gaFramebufferId[64];
 
-	unsigned uScreenId;
-	for (uScreenId = 0; uScreenId < cMonitors; uScreenId++)
+	for (unsigned uScreenId = 0; uScreenId < cMonitors; uScreenId++)
 	{
 		Genodefb *fb = new Genodefb(env);
 		HRESULT rc = display->AttachFramebuffer(uScreenId, fb, gaFramebufferId[uScreenId].asOutParam());
@@ -182,6 +181,12 @@ HRESULT setupmachine(Genode::Env &env)
 	if (FAILED(rc))
 		return rc;
 
+	/* check whether enough memory is available for VM + VMM */
+	ULONG const required_memory_vm = (13 * 1024 + 6 * memory_vbox) << 10;
+	rc = genode_check_memory_config(machine, required_memory_vm);
+	if (FAILED(rc))
+		return rc;
+
 	/* wait until VM is up */
 	MachineState_T machineState = MachineState_Null;
 	do {
@@ -209,6 +214,25 @@ HRESULT setupmachine(Genode::Env &env)
 
 	genodeConsole->init_backends(gKeyboard, gMouse);
 
+	/* check whether enough memory for the fb is available */
+	ULONG required_memory_fb = 0;
+	for (unsigned uScreenId = 0; uScreenId < cMonitors; uScreenId++)
+	{
+		IFramebuffer * aFramebuffer = nullptr;
+		HRESULT rc = display->QueryFramebuffer(uScreenId, &aFramebuffer);
+		Genodefb * fb = dynamic_cast<Genodefb *>(aFramebuffer);
+		if (FAILED(rc) || !fb)
+			continue;
+
+		required_memory_fb += fb->w() * fb->h() * 4;
+	}
+	if (!required_memory_fb)
+		required_memory_fb = 4096 * 2160 * 4;
+
+	rc = genode_check_memory_config(machine, required_memory_fb);
+	if (FAILED(rc))
+		return rc;
+
 	return rc;
 }
 

repos/ports/src/virtualbox5/spec/nova/sup.cc

@@ -170,10 +170,11 @@ HRESULT genode_setup_machine(ComObjPtr<Machine> machine)
 	 * - second chunkid (1..2) is reserved for handy pages allocation
 	 * - another chunkid is used additional for handy pages but as large page
 	 */
-	HRESULT ret = genode_check_memory_config(machine);
-
+	size_t const vmm_memory = 1024ULL * 1024 * (memory_vbox + 16) +
+	                          (CHUNKID_START + 1) * GMM_CHUNK_SIZE;
+	HRESULT ret = genode_check_memory_config(machine, vmm_memory);
 	if (ret == VINF_SUCCESS)
-		vm_memory(1024ULL * 1024 * memory_vbox + (CHUNKID_START + 1) * GMM_CHUNK_SIZE);
+		vm_memory(vmm_memory);
 
 	return ret;
 };

repos/ports/src/virtualbox5/sup.cc

@@ -376,30 +376,16 @@ void genode_VMMR0_DO_GVMM_REGISTER_VMCPU(PVMR0 pVMR0, VMCPUID idCpu)
 }
 
 
-HRESULT genode_check_memory_config(ComObjPtr<Machine> machine)
+HRESULT genode_check_memory_config(ComObjPtr<Machine>,
+                                   size_t const memory_vmm)
 {
-	HRESULT rc;
-
-	/* Validate configured memory of vbox file and Genode config */
-	ULONG memory_vbox;
-	rc = machine->COMGETTER(MemorySize)(&memory_vbox);
-	if (FAILED(rc))
-		return rc;
-
 	/* Request max available memory */
-	size_t memory_genode = genode_env().pd().avail_ram().value >> 20;
-	size_t memory_vmm    = 28;
+	size_t const memory_available = genode_env().pd().avail_ram().value;
 
-	if (memory_vbox + memory_vmm > memory_genode) {
-		using Genode::error;
-		error("Configured memory ", memory_vbox, " MB (vbox file) is insufficient.");
-		error(memory_genode,              " MB (1) - ",
-		      memory_vmm,                 " MB (2) = ",
-		      memory_genode - memory_vmm, " MB (3)");
-		error("(1) available memory based defined by Genode config");
-		error("(2) minimum memory required for VBox VMM");
-		error("(3) maximal available memory to VM");
-		return E_FAIL;
-	}
-	return S_OK;
+	if (memory_vmm <= memory_available)
+		return S_OK;
+
+	Genode::error("Available memory too low to start the VM - available: ",
+	              memory_vmm, "MB < ", memory_available, "MB requested");
+	return E_FAIL;
 }

repos/ports/src/virtualbox5/sup.h

@@ -26,7 +26,7 @@
 #include "MachineImpl.h"
 HRESULT genode_setup_machine(ComObjPtr<Machine> machine);
 
-HRESULT genode_check_memory_config(ComObjPtr<Machine> machine);
+HRESULT genode_check_memory_config(ComObjPtr<Machine> machine, size_t);
 
 /**
  * Returns true if a vCPU could be started. If false we run without