Fix for failure to reset runtimeKeepaliveCounter (#21526)

Failure to reset this counter means that we can leak workers under
the following circumstances:

1. Thread A calls exit_with_live_runtime causing to to have its
   keelalive counter set.
2. Thread A is cancelled via pthead_cancel, returning the worker to the
   pool.
3. Thread B inherits the worker from the pool, with the keelalive
   counter still in a non-zero state.
4. Thread B exits normally, but is kept alive unnecessarily (it also
   cannot be join()'d).

Each time steps 1-4 is repeated we effectively leak the worker since it
can never be re-used.

Author: sbc100

src/library.js

@@ -3332,11 +3332,7 @@ addToLibrary({
       }
     }
 #endif
-#if MINIMAL_RUNTIME
-    throw e;
-#else
     quit_(1, e);
-#endif
   },
 
   $runtimeKeepaliveCounter__internal: true,

src/library_pthread.js

@@ -1101,16 +1101,14 @@ var LibraryPThread = {
     '_emscripten_thread_exit',
 #if !MINIMAL_RUNTIME
     '$keepRuntimeAlive',
-#endif
-#if EXIT_RUNTIME && !MINIMAL_RUNTIME
     '$runtimeKeepaliveCounter',
 #endif
   ],
   $invokeEntryPoint: (ptr, arg) => {
 #if PTHREADS_DEBUG
     dbg(`invokeEntryPoint: ${ptrToString(ptr)}`);
 #endif
-#if EXIT_RUNTIME && !MINIMAL_RUNTIME
+#if !MINIMAL_RUNTIME
     // An old thread on this worker may have been canceled without returning the
     // `runtimeKeepaliveCounter` to zero. Reset it now so the new thread won't
     // be affected.

test/other/test_pthread_reuse.c

@@ -0,0 +1,57 @@
+#include <emscripten/emscripten.h>
+#include <emscripten/eventloop.h>
+#include <emscripten/console.h>
+
+#include <assert.h>
+#include <pthread.h>
+#include <stdio.h>
+
+_Atomic int ready;
+
+void markReady(void* arg) {
+  ready = 1;
+}
+
+void* thread_main(void* arg) {
+  int keepalive = (intptr_t)arg;
+  emscripten_outf("in thread_main: %d", keepalive);
+  if (keepalive) {
+    // Exit with live runtime.  Once we are in the event loop call markReady.
+    emscripten_async_call(markReady, NULL, 0);
+    emscripten_exit_with_live_runtime();
+  }
+  return NULL;
+}
+
+void checkThreadPool() {
+  int running = EM_ASM_INT(return PThread.runningWorkers.length);
+  int unused = EM_ASM_INT(return PThread.unusedWorkers.length);
+  printf("running=%d unused=%d\n", running, unused);
+  assert(running == 0);
+  assert(unused == 1);
+}
+
+int main() {
+  printf("in main\n");
+  checkThreadPool();
+  pthread_t t, t2;
+
+  for (int i = 0; i < 3; i++) {
+    ready = 0;
+
+    // Create a thread that exit's with a non-zero keepalive counter
+    pthread_create(&t, NULL, thread_main, (void*)1);
+    while (!ready) {}
+    pthread_cancel(t);
+    pthread_join(t, NULL);
+    checkThreadPool();
+
+    // Create a second thread that should re-use the same worker.
+    // This thread should reset its keepalive counter on startup and
+    // be able to exit normally.
+    pthread_create(&t2, NULL, thread_main, NULL);
+    pthread_join(t2, NULL);
+    checkThreadPool();
+  }
+  printf("done\n");
+}

test/other/test_pthread_reuse.out

@@ -0,0 +1,15 @@
+in main
+running=0 unused=1
+in thread_main: 1
+running=0 unused=1
+in thread_main: 0
+running=0 unused=1
+in thread_main: 1
+running=0 unused=1
+in thread_main: 0
+running=0 unused=1
+in thread_main: 1
+running=0 unused=1
+in thread_main: 0
+running=0 unused=1
+done

test/test_other.py

@@ -11803,6 +11803,11 @@ def test_pthread_asyncify(self):
     self.set_setting('EXIT_RUNTIME')
     self.do_run_in_out_file_test('other/test_pthread_asyncify.c')
 
+  @node_pthreads
+  def test_pthread_reuse(self):
+    self.set_setting('PTHREAD_POOL_SIZE', 1)
+    self.do_run_in_out_file_test('other/test_pthread_reuse.c')
+
   def test_stdin_preprocess(self):
     create_file('temp.h', '#include <string>')
     outputStdin = self.run_process([EMCC, '-x', 'c++', '-dM', '-E', '-'], input="#include <string>", stdout=PIPE).stdout