pthread: handle self destruction better

Defer destruction of threads which tries to self-destruct. Check an perform
cleanup of such threads during pthread_cancel and pthread_create.

Issue #1687

repos/libports/run/pthread.run

@@ -20,7 +20,7 @@ install_config {
 		<any-service> <parent/> <any-child/> </any-service>
 	</default-route>
 	<start name="test-pthread">
-		<resource name="RAM" quantum="2M"/>
+		<resource name="RAM" quantum="64M"/>
 		<config>
 			<libc stdout="/dev/log">
 				<vfs> <dir name="dev"> <log/> </dir> </vfs>
@@ -35,6 +35,6 @@ build_boot_image {
 	ld.lib.so libc.lib.so pthread.lib.so
 }
 
-append qemu_args " -nographic -m 64 "
+append qemu_args " -nographic -m 128 "
 
 run_genode_until {--- returning from main ---.*\n} 10

repos/libports/src/lib/pthread/thread.cc

@@ -25,6 +25,25 @@
 
 using namespace Genode;
 
+/*
+ * Structure to handle self-destructing pthreads.
+ */
+struct thread_cleanup : List<thread_cleanup>::Element
+{
+	pthread_t thread;
+
+	thread_cleanup(pthread_t t) : thread(t) { }
+
+	~thread_cleanup() {
+		if (thread)
+			destroy(env()->heap(), thread);
+	}
+};
+
+static Lock pthread_cleanup_list_lock;
+static List<thread_cleanup> pthread_cleanup_list;
+
+
 extern "C" {
 
 	/* Thread */
@@ -53,12 +72,31 @@ extern "C" {
 	}
 
 
-	int pthread_cancel(pthread_t thread)
+	void pthread_cleanup()
 	{
-		destroy(env()->heap(), thread);
+		{
-		return 0;
+			Lock_guard<Lock> lock_guard(pthread_cleanup_list_lock);
+
+			while (thread_cleanup * t = pthread_cleanup_list.first()) {
+				pthread_cleanup_list.remove(t);
+				destroy(env()->heap(), t);
+			}
+		}
 	}
 
+	int pthread_cancel(pthread_t thread)
+	{
+		/* cleanup threads which tried to self-destruct */
+		pthread_cleanup();
+
+		if (pthread_equal(pthread_self(), thread)) {
+			Lock_guard<Lock> lock_guard(pthread_cleanup_list_lock);
+			pthread_cleanup_list.insert(new (env()->heap()) thread_cleanup(thread));
+		} else
+			destroy(env()->heap(), thread);
+
+		return 0;
+	}
 
 	void pthread_exit(void *value_ptr)
 	{

repos/libports/src/lib/pthread/thread.h

@@ -70,6 +70,8 @@ extern "C" {
 			pthread_exit(exit_status);
 		}
 	};
+
+	void pthread_cleanup();
 }
 
 #endif /* _INCLUDE__SRC_LIB_PTHREAD_THREAD_H_ */

repos/libports/src/lib/pthread/thread_create.cc

@@ -27,6 +27,9 @@ extern "C"
 	int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
 	                   void *(*start_routine) (void *), void *arg)
 	{
+		/* cleanup threads which tried to self-destruct */
+		pthread_cleanup();
+
 		pthread_t thread_obj = new (Genode::env()->heap())
 		                           pthread(attr ? *attr : 0, start_routine,
 		                           arg, STACK_SIZE, "pthread", nullptr);

repos/libports/src/test/pthread/main.cc

@@ -52,6 +52,8 @@ void *thread_func(void *arg)
 	return 0;
 }
 
+void *thread_func_self_destruct(void *arg) { return 0; }
+
 static inline void compare_semaphore_values(int reported_value, int expected_value)
 {
     if (reported_value != expected_value) {
@@ -60,7 +62,6 @@ static inline void compare_semaphore_values(int reported_value, int expected_val
     }
 }
 
-
 int main(int argc, char **argv)
 {
 	printf("--- pthread test ---\n");
@@ -131,6 +132,16 @@ int main(int argc, char **argv)
 	for (int i = 0; i < NUM_THREADS; i++)
 		sem_destroy(&thread[i].thread_args.thread_finished_sem);
 
+	printf("main thread: create pthreads which self de-struct\n");
+
+	for (unsigned i = 0 ; i < 100; i++) {
+		pthread_t t;
+		if (pthread_create(&t, 0, thread_func_self_destruct, 0) != 0) {
+			printf("error: pthread_create() failed\n");
+			return -1;
+		}
+	}
+
 	printf("--- returning from main ---\n");
 	return 0;
 }

repos/ports/src/virtualbox/thread.cc

@@ -118,6 +118,9 @@ static int create_thread(pthread_t *thread, const pthread_attr_t *attr,
 extern "C" int pthread_create(pthread_t *thread, const pthread_attr_t *attr,
                               void *(*start_routine) (void *), void *arg)
 {
+	/* cleanup threads which tried to self-destruct */
+	pthread_cleanup();
+
 	PRTTHREADINT rtthread = reinterpret_cast<PRTTHREADINT>(arg);
 
 	/* retry thread creation once after CPU session upgrade */