Refactor stub pthread library and enable unconditionally (#602)

This PR changes the layout of the stub pthread library to match that of
libc-top-half: splits it into (mostly) one file per function, and
ensures the same API is exposed from both libraries, adding stub
functions as necessary.

This PR also removes `-lwasi-emulated-pthread` as well as
`-D_WASI_EMULATED_PTHREAD` and enables this functionality
unconditionally, seeing as this is necessary for building libc++ with
threading enabled.

It adds a flag `-D_WASI_STRICT_PTHREAD` that causes thread creation
functions (`pthread_create`, `pthread_detach`, `pthread_join`) to be
defined as a macro causing a compile error, which can be used to locate
the parts of a codebase requiring changes for the single thread model.

Specifically, after this commit, the two targets `wasm32-wasip1` and
`wasm32-wasip1-threads` differ as follows (cleaned up for clarity):

```diff
--- expected/wasm32-wasip1/defined-symbols.txt
+++ expected/wasm32-wasip1-threads/defined-symbols.txt
+flockfile
+ftrylockfile
+funlockfile
+sem_destroy
+sem_getvalue
+sem_init
+sem_post
+sem_timedwait
+sem_trywait
+sem_wait
+wasi_thread_start
```

```diff
--- expected/wasm32-wasip1/predefined-macros.txt
+++ expected/wasm32-wasip1-threads/predefined-macros.txt
+#define _REENTRANT 1
+#define SEM_NSEMS_MAX 256
+#define SEM_VALUE_MAX 0x7fffffff
+#define __wasm_atomics__ 1
+#define __wasm_bulk_memory__ 1
```

Author: whitequark

Makefile

@@ -71,7 +71,7 @@ DLMALLOC_SOURCES = $(DLMALLOC_SRC_DIR)/dlmalloc.c
 DLMALLOC_INC = $(DLMALLOC_DIR)/include
 EMMALLOC_DIR = emmalloc
 EMMALLOC_SOURCES = $(EMMALLOC_DIR)/emmalloc.c
-STUB_PTHREADS_DIR = stub-pthreads
+THREAD_STUB_DIR = thread-stub
 LIBC_BOTTOM_HALF_DIR = libc-bottom-half
 LIBC_BOTTOM_HALF_CLOUDLIBC_SRC = $(LIBC_BOTTOM_HALF_DIR)/cloudlibc/src
 LIBC_BOTTOM_HALF_CLOUDLIBC_SRC_INC = $(LIBC_BOTTOM_HALF_CLOUDLIBC_SRC)/include
@@ -138,8 +138,6 @@ LIBWASI_EMULATED_SIGNAL_SOURCES = \
 LIBWASI_EMULATED_SIGNAL_MUSL_SOURCES = \
     $(LIBC_TOP_HALF_MUSL_SRC_DIR)/signal/psignal.c \
     $(LIBC_TOP_HALF_MUSL_SRC_DIR)/string/strsignal.c
-LIBWASI_EMULATED_PTHREAD_SOURCES = \
-    $(STUB_PTHREADS_DIR)/stub-pthreads-emulated.c
 LIBDL_SOURCES = $(LIBC_TOP_HALF_MUSL_SRC_DIR)/misc/dl.c
 LIBSETJMP_SOURCES = $(LIBC_TOP_HALF_MUSL_SRC_DIR)/setjmp/wasm32/rt.c
 LIBC_BOTTOM_HALF_CRT_SOURCES = $(wildcard $(LIBC_BOTTOM_HALF_DIR)/crt/*.c)
@@ -373,12 +371,38 @@ endif
 ifeq ($(THREAD_MODEL), single)
 # pthreads stubs for single-threaded environment
 LIBC_TOP_HALF_MUSL_SOURCES += \
-    $(STUB_PTHREADS_DIR)/barrier.c \
-    $(STUB_PTHREADS_DIR)/condvar.c \
-    $(STUB_PTHREADS_DIR)/mutex.c \
-    $(STUB_PTHREADS_DIR)/rwlock.c \
-    $(STUB_PTHREADS_DIR)/spinlock.c \
-    $(STUB_PTHREADS_DIR)/stub-pthreads-good.c
+    $(addprefix $(THREAD_STUB_DIR)/, \
+        pthread_barrier_destroy.c \
+        pthread_barrier_init.c \
+        pthread_barrier_wait.c \
+        pthread_cond_broadcast.c \
+        pthread_cond_destroy.c \
+        pthread_cond_init.c \
+        pthread_cond_signal.c \
+        pthread_cond_timedwait.c \
+        pthread_cond_wait.c \
+        pthread_create.c \
+        pthread_detach.c \
+        pthread_getattr_np.c \
+        pthread_join.c \
+        pthread_mutex_consistent.c \
+        pthread_mutex_getprioceiling.c \
+        pthread_mutex_lock.c \
+        pthread_mutex_timedlock.c \
+        pthread_mutex_trylock.c \
+        pthread_mutex_unlock.c \
+        pthread_once.c \
+        pthread_rwlock_rdlock.c \
+        pthread_rwlock_timedrdlock.c \
+        pthread_rwlock_timedwrlock.c \
+        pthread_rwlock_tryrdlock.c \
+        pthread_rwlock_trywrlock.c \
+        pthread_rwlock_unlock.c \
+        pthread_rwlock_wrlock.c \
+        pthread_spin_lock.c \
+        pthread_spin_trylock.c \
+        pthread_spin_unlock.c \
+    )
 endif
 
 MUSL_PRINTSCAN_SOURCES = \
@@ -505,7 +529,6 @@ LIBWASI_EMULATED_PROCESS_CLOCKS_OBJS = $(call objs,$(LIBWASI_EMULATED_PROCESS_CL
 LIBWASI_EMULATED_GETPID_OBJS = $(call objs,$(LIBWASI_EMULATED_GETPID_SOURCES))
 LIBWASI_EMULATED_SIGNAL_OBJS = $(call objs,$(LIBWASI_EMULATED_SIGNAL_SOURCES))
 LIBWASI_EMULATED_SIGNAL_MUSL_OBJS = $(call objs,$(LIBWASI_EMULATED_SIGNAL_MUSL_SOURCES))
-LIBWASI_EMULATED_PTHREAD_OBJS = $(call objs,$(LIBWASI_EMULATED_PTHREAD_SOURCES))
 LIBDL_OBJS = $(call objs,$(LIBDL_SOURCES))
 LIBSETJMP_OBJS = $(call objs,$(LIBSETJMP_SOURCES))
 LIBC_BOTTOM_HALF_CRT_OBJS = $(call objs,$(LIBC_BOTTOM_HALF_CRT_SOURCES))
@@ -529,7 +552,6 @@ LIBWASI_EMULATED_PROCESS_CLOCKS_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBWASI_EMULA
 LIBWASI_EMULATED_GETPID_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBWASI_EMULATED_GETPID_OBJS))
 LIBWASI_EMULATED_SIGNAL_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBWASI_EMULATED_SIGNAL_OBJS))
 LIBWASI_EMULATED_SIGNAL_MUSL_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBWASI_EMULATED_SIGNAL_MUSL_OBJS))
-LIBWASI_EMULATED_PTHREAD_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBWASI_EMULATED_PTHREAD_OBJS))
 LIBDL_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBDL_OBJS))
 LIBSETJMP_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBSETJMP_OBJS))
 BULK_MEMORY_SO_OBJS = $(patsubst %.o,%.pic.o,$(BULK_MEMORY_OBJS))
@@ -546,7 +568,6 @@ PIC_OBJS = \
 	$(LIBWASI_EMULATED_GETPID_SO_OBJS) \
 	$(LIBWASI_EMULATED_SIGNAL_SO_OBJS) \
 	$(LIBWASI_EMULATED_SIGNAL_MUSL_SO_OBJS) \
-	$(LIBWASI_EMULATED_PTHREAD_SO_OBJS) \
 	$(LIBDL_SO_OBJS) \
 	$(LIBSETJMP_SO_OBJS) \
 	$(BULK_MEMORY_SO_OBJS) \
@@ -646,8 +667,6 @@ $(OBJDIR)/libwasi-emulated-getpid.so.a: $(LIBWASI_EMULATED_GETPID_SO_OBJS)
 
 $(OBJDIR)/libwasi-emulated-signal.so.a: $(LIBWASI_EMULATED_SIGNAL_SO_OBJS) $(LIBWASI_EMULATED_SIGNAL_MUSL_SO_OBJS)
 
-$(OBJDIR)/libwasi-emulated-pthread.so.a: $(LIBWASI_EMULATED_PTHREAD_SO_OBJS)
-
 $(OBJDIR)/libdl.so.a: $(LIBDL_SO_OBJS)
 
 $(OBJDIR)/libsetjmp.so.a: $(LIBSETJMP_SO_OBJS)
@@ -666,8 +685,6 @@ $(SYSROOT_LIB)/libwasi-emulated-getpid.a: $(LIBWASI_EMULATED_GETPID_OBJS)
 
 $(SYSROOT_LIB)/libwasi-emulated-signal.a: $(LIBWASI_EMULATED_SIGNAL_OBJS) $(LIBWASI_EMULATED_SIGNAL_MUSL_OBJS)
 
-$(SYSROOT_LIB)/libwasi-emulated-pthread.a: $(LIBWASI_EMULATED_PTHREAD_OBJS)
-
 $(SYSROOT_LIB)/libdl.a: $(LIBDL_OBJS)
 
 $(SYSROOT_LIB)/libsetjmp.a: $(LIBSETJMP_OBJS)
@@ -770,12 +787,6 @@ $(FTS_OBJS) $(FTS_SO_OBJS): CFLAGS += \
 $(LIBWASI_EMULATED_PROCESS_CLOCKS_OBJS) $(LIBWASI_EMULATED_PROCESS_CLOCKS_SO_OBJS): CFLAGS += \
     -I$(LIBC_BOTTOM_HALF_CLOUDLIBC_SRC)
 
-$(LIBWASI_EMULATED_PTHREAD_OBJS) $(LIBWASI_EMULATED_PTHREAD_SO_OBJS): CFLAGS += \
-    -I$(LIBC_TOP_HALF_MUSL_SRC_DIR)/include \
-    -I$(LIBC_TOP_HALF_MUSL_SRC_DIR)/internal \
-    -I$(LIBC_TOP_HALF_MUSL_DIR)/arch/wasm32 \
-    -D_WASI_EMULATED_PTHREAD
-
 # emmalloc uses a lot of pointer type-punning, which is UB under strict aliasing,
 # and this was found to have real miscompilations in wasi-libc#421.
 $(EMMALLOC_OBJS): CFLAGS += \
@@ -816,7 +827,6 @@ LIBC_SO = \
 	$(SYSROOT_LIB)/libwasi-emulated-process-clocks.so \
 	$(SYSROOT_LIB)/libwasi-emulated-getpid.so \
 	$(SYSROOT_LIB)/libwasi-emulated-signal.so \
-	$(SYSROOT_LIB)/libwasi-emulated-pthread.so \
 	$(SYSROOT_LIB)/libdl.so
 ifeq ($(BUILD_LIBSETJMP),yes)
 LIBC_SO += \
@@ -835,10 +845,6 @@ STATIC_LIBS = \
     $(SYSROOT_LIB)/libwasi-emulated-getpid.a \
     $(SYSROOT_LIB)/libwasi-emulated-signal.a \
     $(SYSROOT_LIB)/libdl.a
-ifneq ($(THREAD_MODEL), posix)
-    STATIC_LIBS += \
-        $(SYSROOT_LIB)/libwasi-emulated-pthread.a
-endif
 ifeq ($(BUILD_LIBSETJMP),yes)
 STATIC_LIBS += \
 	$(SYSROOT_LIB)/libsetjmp.a

expected/wasm32-wasip1/defined-symbols.txt

@@ -182,15 +182,14 @@ __progname
 __progname_full
 __pthread_cond_timedwait
 __pthread_create
-__pthread_detach
 __pthread_join
 __pthread_key_create
 __pthread_key_delete
-__pthread_mutex_consistent
 __pthread_mutex_lock
 __pthread_mutex_timedlock
 __pthread_mutex_trylock
 __pthread_mutex_unlock
+__pthread_once
 __pthread_rwlock_rdlock
 __pthread_rwlock_timedrdlock
 __pthread_rwlock_timedwrlock
@@ -991,13 +990,14 @@ pthread_condattr_setpshared
 pthread_create
 pthread_detach
 pthread_equal
-pthread_exit
+pthread_getattr_np
 pthread_getspecific
 pthread_join
 pthread_key_create
 pthread_key_delete
 pthread_mutex_consistent
 pthread_mutex_destroy
+pthread_mutex_getprioceiling
 pthread_mutex_init
 pthread_mutex_lock
 pthread_mutex_timedlock
@@ -1221,6 +1221,7 @@ tgamma
 tgammaf
 tgammal
 thrd_current
+thrd_detach
 thrd_equal
 thrd_sleep
 time

expected/wasm32-wasip1/predefined-macros.txt

@@ -2252,8 +2252,12 @@
 #define _POSIX_STREAM_MAX 8
 #define _POSIX_SYMLINK_MAX 255
 #define _POSIX_SYMLOOP_MAX 8
+#define _POSIX_THREADS _POSIX_VERSION
+#define _POSIX_THREAD_ATTR_STACKSIZE _POSIX_VERSION
 #define _POSIX_THREAD_DESTRUCTOR_ITERATIONS 4
 #define _POSIX_THREAD_KEYS_MAX 128
+#define _POSIX_THREAD_PROCESS_SHARED _POSIX_VERSION
+#define _POSIX_THREAD_SAFE_FUNCTIONS _POSIX_VERSION
 #define _POSIX_THREAD_THREADS_MAX 64
 #define _POSIX_TIMEOUTS _POSIX_VERSION
 #define _POSIX_TIMERS _POSIX_VERSION
@@ -3430,12 +3434,7 @@
 #define preadv64 preadv
 #define pthread_cleanup_pop(r) _pthread_cleanup_pop(&__cb, (r)); } while(0)
 #define pthread_cleanup_push(f,x) do { struct __ptcb __cb; _pthread_cleanup_push(&__cb, f, x);
-#define pthread_create(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; to enable stub functions which always fail, compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
-#define pthread_detach(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; to enable stub functions which always fail, compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
 #define pthread_equal(x,y) ((x)==(y))
-#define pthread_join(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; to enable stub functions which always fail, compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
-#define pthread_timedjoin_np(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; to enable stub functions which always fail, compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
-#define pthread_tryjoin_np(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; to enable stub functions which always fail, compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
 #define pwrite64 pwrite
 #define pwritev64 pwritev
 #define readdir64 readdir

expected/wasm32-wasip2/defined-symbols.txt

@@ -185,15 +185,14 @@ __progname
 __progname_full
 __pthread_cond_timedwait
 __pthread_create
-__pthread_detach
 __pthread_join
 __pthread_key_create
 __pthread_key_delete
-__pthread_mutex_consistent
 __pthread_mutex_lock
 __pthread_mutex_timedlock
 __pthread_mutex_trylock
 __pthread_mutex_unlock
+__pthread_once
 __pthread_rwlock_rdlock
 __pthread_rwlock_timedrdlock
 __pthread_rwlock_timedwrlock
@@ -1127,13 +1126,14 @@ pthread_condattr_setpshared
 pthread_create
 pthread_detach
 pthread_equal
-pthread_exit
+pthread_getattr_np
 pthread_getspecific
 pthread_join
 pthread_key_create
 pthread_key_delete
 pthread_mutex_consistent
 pthread_mutex_destroy
+pthread_mutex_getprioceiling
 pthread_mutex_init
 pthread_mutex_lock
 pthread_mutex_timedlock
@@ -1457,6 +1457,7 @@ tgamma
 tgammaf
 tgammal
 thrd_current
+thrd_detach
 thrd_equal
 thrd_sleep
 time

expected/wasm32-wasip2/predefined-macros.txt

@@ -2403,8 +2403,12 @@
 #define _POSIX_STREAM_MAX 8
 #define _POSIX_SYMLINK_MAX 255
 #define _POSIX_SYMLOOP_MAX 8
+#define _POSIX_THREADS _POSIX_VERSION
+#define _POSIX_THREAD_ATTR_STACKSIZE _POSIX_VERSION
 #define _POSIX_THREAD_DESTRUCTOR_ITERATIONS 4
 #define _POSIX_THREAD_KEYS_MAX 128
+#define _POSIX_THREAD_PROCESS_SHARED _POSIX_VERSION
+#define _POSIX_THREAD_SAFE_FUNCTIONS _POSIX_VERSION
 #define _POSIX_THREAD_THREADS_MAX 64
 #define _POSIX_TIMEOUTS _POSIX_VERSION
 #define _POSIX_TIMERS _POSIX_VERSION
@@ -3585,12 +3589,7 @@
 #define preadv64 preadv
 #define pthread_cleanup_pop(r) _pthread_cleanup_pop(&__cb, (r)); } while(0)
 #define pthread_cleanup_push(f,x) do { struct __ptcb __cb; _pthread_cleanup_push(&__cb, f, x);
-#define pthread_create(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; to enable stub functions which always fail, compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
-#define pthread_detach(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; to enable stub functions which always fail, compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
 #define pthread_equal(x,y) ((x)==(y))
-#define pthread_join(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; to enable stub functions which always fail, compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
-#define pthread_timedjoin_np(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; to enable stub functions which always fail, compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
-#define pthread_tryjoin_np(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; to enable stub functions which always fail, compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
 #define pwrite64 pwrite
 #define pwritev64 pwritev
 #define readdir64 readdir

libc-top-half/musl/include/pthread.h

@@ -83,21 +83,14 @@ int pthread_detach(pthread_t);
 _Noreturn void pthread_exit(void *);
 int pthread_join(pthread_t, void **);
 #else
-#if defined(_WASI_EMULATED_PTHREAD) || defined(_REENTRANT)
+#if defined(_REENTRANT) || !defined(_WASI_STRICT_PTHREAD)
 int pthread_create(pthread_t *__restrict, const pthread_attr_t *__restrict, void *(*)(void *), void *__restrict);
 int pthread_detach(pthread_t);
 int pthread_join(pthread_t, void **);
 #else
-#include <assert.h>
-#define pthread_create(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; \
-to enable stub functions which always fail, \
-compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
-#define pthread_detach(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; \
-to enable stub functions which always fail, \
-compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
-#define pthread_join(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; \
-to enable stub functions which always fail, \
-compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
+#define pthread_create(...) ({ _Static_assert(0, "This function is not available on a single-threaded target; switch to a multi-threaded target with -pthread or enable a stub implementation by undefining _WASI_STRICT_PTHREAD"); 0;})
+#define pthread_detach(...) ({ _Static_assert(0, "This function is not available on a single-threaded target; switch to a multi-threaded target with -pthread or enable a stub implementation by undefining _WASI_STRICT_PTHREAD"); 0;})
+#define pthread_join(...) ({ _Static_assert(0, "This function is not available on a single-threaded target; switch to a multi-threaded target with -pthread or enable a stub implementation by undefining _WASI_STRICT_PTHREAD"); 0;})
 #endif
 #endif
 
@@ -249,16 +242,12 @@ int pthread_setname_np(pthread_t, const char *);
 int pthread_getname_np(pthread_t, char *, size_t);
 int pthread_getattr_default_np(pthread_attr_t *);
 int pthread_setattr_default_np(const pthread_attr_t *);
-#if defined(__wasilibc_unmodified_upstream) || defined(_WASI_EMULATED_PTHREAD) || defined(_REENTRANT)
+#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT) || !defined(_WASI_STRICT_PTHREAD)
 int pthread_tryjoin_np(pthread_t, void **);
 int pthread_timedjoin_np(pthread_t, void **, const struct timespec *);
 #else
-#define pthread_tryjoin_np(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; \
-to enable stub functions which always fail, \
-compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
-#define pthread_timedjoin_np(...) ({ _Static_assert(0, "This mode of WASI does not have threads enabled; \
-to enable stub functions which always fail, \
-compile with -D_WASI_EMULATED_PTHREAD and link with -lwasi-emulated-pthread"); 0;})
+#define pthread_tryjoin_np(...) ({ _Static_assert(0, "This function is not available on a single-threaded target; switch to a multi-threaded target with -pthread or enable a stub implementation by undefining _WASI_STRICT_PTHREAD"); 0;})
+#define pthread_timedjoin_np(...) ({ _Static_assert(0, "This function is not available on a single-threaded target; switch to a multi-threaded target with -pthread or enable a stub implementation by undefining _WASI_STRICT_PTHREAD"); 0;})
 #endif
 #endif
 

libc-top-half/musl/include/unistd.h

@@ -336,15 +336,15 @@ pid_t gettid(void);
 #endif
 #define _POSIX_VDISABLE         0
 
-#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
+#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT) || !defined(_WASI_STRICT_PTHREAD)
 #define _POSIX_THREADS          _POSIX_VERSION
 #define _POSIX_THREAD_PROCESS_SHARED _POSIX_VERSION
 #define _POSIX_THREAD_SAFE_FUNCTIONS _POSIX_VERSION
 #endif
 #if defined(__wasilibc_unmodified_upstream) /* wasi-libc doesn't provide pthread_attr_{get,set}stackaddr */
 #define _POSIX_THREAD_ATTR_STACKADDR _POSIX_VERSION
 #endif
-#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT)
+#if defined(__wasilibc_unmodified_upstream) || defined(_REENTRANT) || !defined(_WASI_STRICT_PTHREAD)
 #define _POSIX_THREAD_ATTR_STACKSIZE _POSIX_VERSION
 #endif
 #if defined(__wasilibc_unmodified_upstream) /* WASI has no scheduling control, and wasi-libc doesn't provide pthread_getcpuclockid */