From 0fe645d6fd22a6f57e777a29e65cf9a4ff9785ae Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 27 Jul 2022 20:28:06 +0200 Subject: gh-95174: Add pthread stubs for WASI (GH-95234) Co-authored-by: Brett Cannon --- Doc/library/sys.rst | 2 + Include/cpython/pthread_stubs.h | 88 ++++++++++ Include/cpython/pythread.h | 3 + Lib/test/pythoninfo.py | 4 +- Lib/test/test_sys.py | 2 +- Lib/test/test_threadsignals.py | 2 + Makefile.pre.in | 1 + .../2022-07-25-09-48-43.gh-issue-95145.ZNS3dj.rst | 2 + Python/thread.c | 18 +- Python/thread_pthread.h | 4 +- Python/thread_pthread_stubs.h | 185 +++++++++++++++++++++ Tools/wasm/README.md | 17 +- Tools/wasm/config.site-wasm32-wasi | 8 - Tools/wasm/wasi-env | 7 - configure | 15 +- configure.ac | 12 +- pyconfig.h.in | 3 + 17 files changed, 332 insertions(+), 41 deletions(-) create mode 100644 Include/cpython/pthread_stubs.h create mode 100644 Misc/NEWS.d/next/Build/2022-07-25-09-48-43.gh-issue-95145.ZNS3dj.rst create mode 100644 Python/thread_pthread_stubs.h diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index e10efb1..62b3673 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1658,6 +1658,8 @@ always available. | | | | | * ``'nt'``: Windows threads | | | * ``'pthread'``: POSIX threads | + | | * ``'pthread-stubs'``: stub POSIX threads | + | | (on WebAssembly platforms without threading support) | | | * ``'solaris'``: Solaris threads | +------------------+---------------------------------------------------------+ | :const:`lock` | Name of the lock implementation: | diff --git a/Include/cpython/pthread_stubs.h b/Include/cpython/pthread_stubs.h new file mode 100644 index 0000000..d95ee03 --- /dev/null +++ b/Include/cpython/pthread_stubs.h @@ -0,0 +1,88 @@ +#ifndef Py_CPYTHON_PTRHEAD_STUBS_H +#define Py_CPYTHON_PTRHEAD_STUBS_H + +#if !defined(HAVE_PTHREAD_STUBS) +# error "this header file requires stubbed pthreads." +#endif + +#ifndef _POSIX_THREADS +# define _POSIX_THREADS 1 +#endif + +/* Minimal pthread stubs for CPython. + * + * The stubs implement the minimum pthread API for CPython. + * - pthread_create() fails. + * - pthread_exit() calls exit(0). + * - pthread_key_*() functions implement minimal TSS without destructor. + * - all other functions do nothing and return 0. + */ + +#ifdef __wasi__ +// WASI's bits/alltypes.h provides type definitions when __NEED_ is set. +// The header file can be included multiple times. +# define __NEED_pthread_cond_t 1 +# define __NEED_pthread_condattr_t 1 +# define __NEED_pthread_mutex_t 1 +# define __NEED_pthread_mutexattr_t 1 +# define __NEED_pthread_key_t 1 +# define __NEED_pthread_t 1 +# define __NEED_pthread_attr_t 1 +# include +#else +typedef struct { void *__x; } pthread_cond_t; +typedef struct { unsigned __attr; } pthread_condattr_t; +typedef struct { void *__x; } pthread_mutex_t; +typedef struct { unsigned __attr; } pthread_mutexattr_t; +typedef unsigned pthread_key_t; +typedef unsigned pthread_t; +typedef struct { unsigned __attr; } pthread_attr_t; +#endif + +// mutex +PyAPI_FUNC(int) pthread_mutex_init(pthread_mutex_t *restrict mutex, + const pthread_mutexattr_t *restrict attr); +PyAPI_FUNC(int) pthread_mutex_destroy(pthread_mutex_t *mutex); +PyAPI_FUNC(int) pthread_mutex_trylock(pthread_mutex_t *mutex); +PyAPI_FUNC(int) pthread_mutex_lock(pthread_mutex_t *mutex); +PyAPI_FUNC(int) pthread_mutex_unlock(pthread_mutex_t *mutex); + +// condition +PyAPI_FUNC(int) pthread_cond_init(pthread_cond_t *restrict cond, + const pthread_condattr_t *restrict attr); +PyAPI_FUNC(int) pthread_cond_destroy(pthread_cond_t *cond); +PyAPI_FUNC(int) pthread_cond_wait(pthread_cond_t *restrict cond, + pthread_mutex_t *restrict mutex); +PyAPI_FUNC(int) pthread_cond_timedwait(pthread_cond_t *restrict cond, + pthread_mutex_t *restrict mutex, + const struct timespec *restrict abstime); +PyAPI_FUNC(int) pthread_cond_signal(pthread_cond_t *cond); +PyAPI_FUNC(int) pthread_condattr_init(pthread_condattr_t *attr); +PyAPI_FUNC(int) pthread_condattr_setclock( + pthread_condattr_t *attr, clockid_t clock_id); + +// pthread +PyAPI_FUNC(int) pthread_create(pthread_t *restrict thread, + const pthread_attr_t *restrict attr, + void *(*start_routine)(void *), + void *restrict arg); +PyAPI_FUNC(int) pthread_detach(pthread_t thread); +PyAPI_FUNC(pthread_t) pthread_self(void); +PyAPI_FUNC(int) pthread_exit(void *retval) __attribute__ ((__noreturn__)); +PyAPI_FUNC(int) pthread_attr_init(pthread_attr_t *attr); +PyAPI_FUNC(int) pthread_attr_setstacksize(pthread_attr_t *attr, size_t stacksize); +PyAPI_FUNC(int) pthread_attr_destroy(pthread_attr_t *attr); + + +// pthread_key +#ifndef PTHREAD_KEYS_MAX +# define PTHREAD_KEYS_MAX 128 +#endif + +PyAPI_FUNC(int) pthread_key_create(pthread_key_t *key, + void (*destr_function)(void *)); +PyAPI_FUNC(int) pthread_key_delete(pthread_key_t key); +PyAPI_FUNC(void *) pthread_getspecific(pthread_key_t key); +PyAPI_FUNC(int) pthread_setspecific(pthread_key_t key, const void *value); + +#endif // Py_CPYTHON_PTRHEAD_STUBS_H diff --git a/Include/cpython/pythread.h b/Include/cpython/pythread.h index 1fd86a6..ce4ec8f 100644 --- a/Include/cpython/pythread.h +++ b/Include/cpython/pythread.h @@ -20,6 +20,9 @@ PyAPI_FUNC(int) _PyThread_at_fork_reinit(PyThread_type_lock *lock); but hardcode the unsigned long to avoid errors for include directive. */ # define NATIVE_TSS_KEY_T unsigned long +#elif defined(HAVE_PTHREAD_STUBS) +# include "cpython/pthread_stubs.h" +# define NATIVE_TSS_KEY_T pthread_key_t #else # error "Require native threads. See https://bugs.python.org/issue31370" #endif diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 172c03f..adc211b 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -588,8 +588,8 @@ def collect_socket(info_add): try: hostname = socket.gethostname() - except OSError: - # WASI SDK 15.0 does not have gethostname(2). + except (OSError, AttributeError): + # WASI SDK 16.0 does not have gethostname(2). if sys.platform != "wasi": raise else: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index f629dd5..c4798af 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -626,7 +626,7 @@ class SysModuleTest(unittest.TestCase): def test_thread_info(self): info = sys.thread_info self.assertEqual(len(info), 3) - self.assertIn(info.name, ('nt', 'pthread', 'solaris', None)) + self.assertIn(info.name, ('nt', 'pthread', 'pthread-stubs', 'solaris', None)) self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) @unittest.skipUnless(support.is_emscripten, "only available on Emscripten") diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py index e0ac18c..6a53d65 100644 --- a/Lib/test/test_threadsignals.py +++ b/Lib/test/test_threadsignals.py @@ -36,7 +36,9 @@ def send_signals(): os.kill(process_pid, signal.SIGUSR2) signalled_all.release() + @threading_helper.requires_working_threading() +@unittest.skipUnless(hasattr(signal, "alarm"), "test requires signal.alarm") class ThreadSignals(unittest.TestCase): def test_signals(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index 20e66b8..746ff42 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1544,6 +1544,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/objimpl.h \ $(srcdir)/Include/cpython/odictobject.h \ $(srcdir)/Include/cpython/picklebufobject.h \ + $(srcdir)/Include/cpython/pthread_stubs.h \ $(srcdir)/Include/cpython/pyctype.h \ $(srcdir)/Include/cpython/pydebug.h \ $(srcdir)/Include/cpython/pyerrors.h \ diff --git a/Misc/NEWS.d/next/Build/2022-07-25-09-48-43.gh-issue-95145.ZNS3dj.rst b/Misc/NEWS.d/next/Build/2022-07-25-09-48-43.gh-issue-95145.ZNS3dj.rst new file mode 100644 index 0000000..c751b5e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-07-25-09-48-43.gh-issue-95145.ZNS3dj.rst @@ -0,0 +1,2 @@ +wasm32-wasi builds no longer depend on WASIX's pthread stubs. Python now has +its own stubbed pthread API. diff --git a/Python/thread.c b/Python/thread.c index d321121..e206a69 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -55,8 +55,15 @@ PyThread_init_thread(void) PyThread__init_thread(); } -#if defined(_POSIX_THREADS) -# define PYTHREAD_NAME "pthread" +#if defined(HAVE_PTHREAD_STUBS) +# define PYTHREAD_NAME "pthread-stubs" +# include "thread_pthread_stubs.h" +#elif defined(_POSIX_THREADS) +# if defined(__EMSCRIPTEN__) || !defined(__EMSCRIPTEN_PTHREADS__) +# define PYTHREAD_NAME "pthread-stubs" +# else +# define PYTHREAD_NAME "pthread" +# endif # include "thread_pthread.h" #elif defined(NT_THREADS) # define PYTHREAD_NAME "nt" @@ -171,7 +178,9 @@ PyThread_GetInfo(void) } PyStructSequence_SET_ITEM(threadinfo, pos++, value); -#ifdef _POSIX_THREADS +#ifdef HAVE_PTHREAD_STUBS + value = Py_NewRef(Py_None); +#elif defined(_POSIX_THREADS) #ifdef USE_SEMAPHORES value = PyUnicode_FromString("semaphore"); #else @@ -182,8 +191,7 @@ PyThread_GetInfo(void) return NULL; } #else - Py_INCREF(Py_None); - value = Py_None; + value = Py_NewRef(Py_None); #endif PyStructSequence_SET_ITEM(threadinfo, pos++, value); diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index c310d72..1c5b320 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -7,7 +7,9 @@ #if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR) #define destructor xxdestructor #endif -#include +#ifndef HAVE_PTHREAD_STUBS +# include +#endif #if defined(__APPLE__) || defined(HAVE_PTHREAD_DESTRUCTOR) #undef destructor #endif diff --git a/Python/thread_pthread_stubs.h b/Python/thread_pthread_stubs.h new file mode 100644 index 0000000..8b80c0f --- /dev/null +++ b/Python/thread_pthread_stubs.h @@ -0,0 +1,185 @@ +#include "cpython/pthread_stubs.h" + +// mutex +int +pthread_mutex_init(pthread_mutex_t *restrict mutex, + const pthread_mutexattr_t *restrict attr) +{ + return 0; +} + +int +pthread_mutex_destroy(pthread_mutex_t *mutex) +{ + return 0; +} + +int +pthread_mutex_trylock(pthread_mutex_t *mutex) +{ + return 0; +} + +int +pthread_mutex_lock(pthread_mutex_t *mutex) +{ + return 0; +} + +int +pthread_mutex_unlock(pthread_mutex_t *mutex) +{ + return 0; +} + +// condition +int +pthread_cond_init(pthread_cond_t *restrict cond, + const pthread_condattr_t *restrict attr) +{ + return 0; +} + +PyAPI_FUNC(int)pthread_cond_destroy(pthread_cond_t *cond) +{ + return 0; +} + +int +pthread_cond_wait(pthread_cond_t *restrict cond, + pthread_mutex_t *restrict mutex) +{ + return 0; +} + +int +pthread_cond_timedwait(pthread_cond_t *restrict cond, + pthread_mutex_t *restrict mutex, + const struct timespec *restrict abstime) +{ + return 0; +} + +int +pthread_cond_signal(pthread_cond_t *cond) +{ + return 0; +} + +int +pthread_condattr_init(pthread_condattr_t *attr) +{ + return 0; +} + +int +pthread_condattr_setclock(pthread_condattr_t *attr, clockid_t clock_id) +{ + return 0; +} + +// pthread +int +pthread_create(pthread_t *restrict thread, + const pthread_attr_t *restrict attr, + void *(*start_routine)(void *), + void *restrict arg) +{ + return EAGAIN; +} + +int +pthread_detach(pthread_t thread) +{ + return 0; +} + +PyAPI_FUNC(pthread_t) pthread_self(void) +{ + return 0; +} + +int +pthread_exit(void *retval) +{ + exit(0); +} + +int +pthread_attr_init(pthread_attr_t *attr) +{ + return 0; +} + +int +pthread_attr_setstacksize( + pthread_attr_t *attr, size_t stacksize) +{ + return 0; +} + +int +pthread_attr_destroy(pthread_attr_t *attr) +{ + return 0; +} + +// pthread_key +typedef struct { + bool in_use; + void *value; +} py_tls_entry; + +static py_tls_entry py_tls_entries[PTHREAD_KEYS_MAX] = {0}; + +int +pthread_key_create(pthread_key_t *key, void (*destr_function)(void *)) +{ + if (!key) { + return EINVAL; + } + if (destr_function != NULL) { + Py_FatalError("pthread_key_create destructor is not supported"); + } + for (pthread_key_t idx = 0; idx < PTHREAD_KEYS_MAX; idx++) { + if (!py_tls_entries[idx].in_use) { + py_tls_entries[idx].in_use = true; + *key = idx; + return 0; + } + } + return EAGAIN; +} + +int +pthread_key_delete(pthread_key_t key) +{ + if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) { + return EINVAL; + } + py_tls_entries[key].in_use = false; + py_tls_entries[key].value = NULL; + return 0; +} + + +void * +pthread_getspecific(pthread_key_t key) { + if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) { + return NULL; + } + return py_tls_entries[key].value; +} + +int +pthread_setspecific(pthread_key_t key, const void *value) +{ + if (key < 0 || key >= PTHREAD_KEYS_MAX || !py_tls_entries[key].in_use) { + return EINVAL; + } + py_tls_entries[key].value = (void *)value; + return 0; +} + +// let thread_pthread define the Python API +#include "thread_pthread.h" diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index d08e807..6496a29 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -226,16 +226,13 @@ AddType application/wasm wasm # WASI (wasm32-wasi) -WASI builds require [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 15.0+ -and currently [wasix](https://github.com/singlestore-labs/wasix) for POSIX -compatibility stubs. +WASI builds require [WASI SDK](https://github.com/WebAssembly/wasi-sdk) 16.0+. ## Cross-compile to wasm32-wasi The script ``wasi-env`` sets necessary compiler and linker flags as well as ``pkg-config`` overrides. The script assumes that WASI-SDK is installed in -``/opt/wasi-sdk`` or ``$WASI_SDK_PATH`` and WASIX is installed in -``/opt/wasix`` or ``$WASIX_PATH``. +``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``. ```shell mkdir -p builddir/wasi @@ -434,21 +431,15 @@ rm -f wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz ### Install [wasmtime](https://github.com/bytecodealliance/wasmtime) WASI runtime -**NOTE**: wasmtime 0.37 has a bug. Newer versions should be fine again. +wasmtime 0.38 or newer is required. ```shell curl -sSf -L -o ~/install-wasmtime.sh https://wasmtime.dev/install.sh chmod +x ~/install-wasmtime.sh -~/install-wasmtime.sh --version v0.36.0 +~/install-wasmtime.sh --version v0.38.0 ln -srf -t /usr/local/bin/ ~/.wasmtime/bin/wasmtime ``` -### Install [WASIX](https://github.com/singlestore-labs/wasix) - -```shell -git clone https://github.com/singlestore-labs/wasix.git ~/wasix -make install -C ~/wasix -``` ### WASI debugging diff --git a/Tools/wasm/config.site-wasm32-wasi b/Tools/wasm/config.site-wasm32-wasi index 237fa8b..893a0d1 100644 --- a/Tools/wasm/config.site-wasm32-wasi +++ b/Tools/wasm/config.site-wasm32-wasi @@ -32,14 +32,6 @@ ac_cv_func_makedev=no # OSError: [Errno 28] Invalid argument: '.' ac_cv_func_fdopendir=no -# WASIX stubs we don't want to use. -ac_cv_func_kill=no - -# WASI SDK 15.0 does not have chmod. -# Ignore WASIX stubs for now. -ac_cv_func_chmod=no -ac_cv_func_fchmod=no - # WASI sockets are limited to operations on given socket fd and inet sockets. # Disable AF_UNIX and AF_PACKET support, see socketmodule.h. ac_cv_header_sys_un_h=no diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 06c54e6..6c2d56e 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -31,7 +31,6 @@ fi WASI_SDK_PATH="${WASI_SDK_PATH:-/opt/wasi-sdk}" WASI_SYSROOT="${WASI_SDK_PATH}/share/wasi-sysroot" -WASIX_PATH="${WASIX_PATH:-/opt/wasix}" if ! test -x "${WASI_SDK_PATH}/bin/clang"; then echo "Error: ${WASI_SDK_PATH}/bin/clang does not exist." >&2 @@ -65,12 +64,6 @@ PKG_CONFIG_PATH="" PKG_CONFIG_LIBDIR="${WASI_SYSROOT}/lib/pkgconfig:${WASI_SYSROOT}/share/pkgconfig" PKG_CONFIG_SYSROOT_DIR="${WASI_SYSROOT}" -# add WASIX (POSIX stubs for WASI) if WASIX is installed -if test -f "${WASIX_PATH}/lib/libwasix.a"; then - CFLAGS="${CFLAGS} -isystem ${WASIX_PATH}/include" - LDFLAGS="${LDFLAGS} -L${WASIX_PATH}/lib -lwasix" -fi - PATH="${WASI_SDK_PATH}/bin:${PATH}" export WASI_SDK_PATH WASI_SYSROOT diff --git a/configure b/configure index 21f7f91..5df9f83 100755 --- a/configure +++ b/configure @@ -14748,11 +14748,16 @@ if test "x$ac_cv_lib_cma_pthread_create" = xyes; then : else + case $ac_sys_system in #( + WASI) : + posix_threads=stub ;; #( + *) : as_fn_error $? "could not find pthreads on your system" "$LINENO" 5 + ;; +esac fi - fi fi @@ -14910,6 +14915,14 @@ done fi +if test "x$posix_threads" = xstub; then : + + +$as_echo "#define HAVE_PTHREAD_STUBS 1" >>confdefs.h + + +fi + # Check for enable-ipv6 { $as_echo "$as_me:${as_lineno-$LINENO}: checking if --enable-ipv6 is specified" >&5 diff --git a/configure.ac b/configure.ac index 6b36696..38880fc 100644 --- a/configure.ac +++ b/configure.ac @@ -4207,9 +4207,11 @@ pthread_create (NULL, NULL, start_routine, NULL)]])],[ posix_threads=yes LIBS="$LIBS -lcma" ],[ - AC_MSG_ERROR([could not find pthreads on your system]) - ]) - ])])])])]) + AS_CASE([$ac_sys_system], + [WASI], [posix_threads=stub], + [AC_MSG_ERROR([could not find pthreads on your system])] + ) + ])])])])])]) AC_CHECK_LIB(mpc, usconfig, [ LIBS="$LIBS -lmpc" @@ -4272,6 +4274,10 @@ if test "$posix_threads" = "yes"; then AC_CHECK_FUNCS(pthread_getcpuclockid) fi +AS_VAR_IF([posix_threads], [stub], [ + AC_DEFINE([HAVE_PTHREAD_STUBS], [1], [Define if platform requires stubbed pthreads support]) +]) + # Check for enable-ipv6 AH_TEMPLATE(ENABLE_IPV6, [Define if --enable-ipv6 is specified]) AC_MSG_CHECKING([if --enable-ipv6 is specified]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 91b249e..10e7ad1 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -938,6 +938,9 @@ /* Define to 1 if you have the `pthread_sigmask' function. */ #undef HAVE_PTHREAD_SIGMASK +/* Define if platform requires stubbed pthreads support */ +#undef HAVE_PTHREAD_STUBS + /* Define to 1 if you have the header file. */ #undef HAVE_PTY_H -- cgit v0.12