From c9844cb8aa9615cdc8770d1e43ce6e2ac3efd836 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 4 Apr 2022 20:31:31 +0300 Subject: bpo-40280: Add --enable-wasm-dynamic-linking (GH-32253) --- Doc/using/configure.rst | 29 +++++- .../Build/2022-04-02-17-52-38.bpo-40280.U8Dd0H.rst | 2 + Tools/wasm/README.md | 23 +++-- Tools/wasm/config.site-wasm32-emscripten | 4 - configure | 104 ++++++++++++++++++--- configure.ac | 81 +++++++++++++--- 6 files changed, 194 insertions(+), 49 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2022-04-02-17-52-38.bpo-40280.U8Dd0H.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index debbee7..b46157c 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -52,11 +52,13 @@ General Options Set the Python executable suffix to *SUFFIX*. The default suffix is ``.exe`` on Windows and macOS (``python.exe`` - executable), ``.wasm`` on Emscripten (``python.wasm`` executable), and - an empty string on other platforms (``python`` executable). + executable), ``.js`` on Emscripten node, ``.html`` on Emscripten browser, + ``.wasm`` on WASI, and an empty string on other platforms (``python`` + executable). .. versionchanged:: 3.11 - The default suffix on Emscripten platform is ``.wasm``. + The default suffix on WASM platform is one of ``.js``, ``.html`` + or ``.wasm``. .. cmdoption:: --with-tzpath= @@ -141,6 +143,27 @@ General Options .. versionadded:: 3.11 +WebAssemby Options +------------------ + +.. cmdoption:: --with-emscripten-target=[browser|node] + + Set build flavor for ``wasm32-emscripten``. + + * ``browser`` (default): preload minimal stdlib, default MEMFS. + * ``node``: NODERAWFS and pthread support. + + .. versionadded:: 3.11 + +.. cmdoption:: --enable-wasm-dynamic-linking + + Turn on dynamic linking support for WASM. + + Dynamic linking enables ``dlopen``. File size of the executable + increases due to limited dead code elimination and additional features. + + .. versionadded:: 3.11 + Install Options --------------- diff --git a/Misc/NEWS.d/next/Build/2022-04-02-17-52-38.bpo-40280.U8Dd0H.rst b/Misc/NEWS.d/next/Build/2022-04-02-17-52-38.bpo-40280.U8Dd0H.rst new file mode 100644 index 0000000..74fe5c7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-04-02-17-52-38.bpo-40280.U8Dd0H.rst @@ -0,0 +1,2 @@ +Add configure option :option:`--enable-wasm-dynamic-linking` to enable +``dlopen`` and MAIN_MODULE / SIDE_MODULE on ``wasm32-emscripten``. diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 1cdaa4e..6b1e7b0 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -81,24 +81,31 @@ node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscri ## wasm32-emscripten limitations and issues +- Heap and stack are limited. - Most stdlib modules with a dependency on external libraries are missing: ``ctypes``, ``readline``, ``sqlite3``, ``ssl``, and more. - Shared extension modules are not implemented yet. All extension modules are statically linked into the main binary. + The experimental configure option ``--enable-wasm-dynamic-linking`` enables + dynamic extensions. - Processes are not supported. System calls like fork, popen, and subprocess fail with ``ENOSYS`` or ``ENOSUP``. +- Only ``AF_INET`` and ``AF_INET6`` with ``SOCK_STREAM`` (TCP) or + ``SOCK_DGRAM`` (UDP) are available. ``AF_UNIX`` is not supported. +- ``socketpair`` does not work. - Blocking sockets are not available and non-blocking sockets don't work correctly, e.g. ``socket.accept`` crashes the runtime. ``gethostbyname`` does not resolve to a real IP address. IPv6 is not available. - The ``select`` module is limited. ``select.select()`` crashes the runtime due to lack of exectfd support. -- The ``*at`` variants of functions (e.g. ``openat``) are not available. - The ``dir_fd`` argument of *os* module functions can't be used. - Signal support is limited. ``signal.alarm``, ``itimer``, ``sigaction`` are not available or do not work correctly. ``SIGTERM`` exits the runtime. - Most user, group, and permission related function and modules are not supported or don't work as expected, e.g.``pwd`` module, ``grp`` module, - ``os.setgroups``, ``os.chown``, and so on. + ``os.setgroups``, ``os.chown``, and so on. ``lchown`` and `lchmod`` are + not available. +- ``umask`` is a no-op. +- hard links (``os.link``) are not supported. - Offset and iovec I/O functions (e.g. ``os.pread``, ``os.preadv``) are not available. - ``os.mknod`` and ``os.mkfifo`` @@ -108,17 +115,9 @@ node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscri - ``mmap`` module is unstable. flush (``msync``) can crash the runtime. - Resource-related functions like ``os.nice`` and most functions of the ``resource`` module are not available. -- Some time and datetime features are broken. ``strftime`` and ``strptime`` - have known bugs, e.g. - [%% quoting](https://github.com/emscripten-core/emscripten/issues/16155), - [%U off-by-one](https://github.com/emscripten-core/emscripten/issues/16156). - Extended glibc formatting features are not available. +- glibc extensions for date and time formatting are not available. - ``locales`` module is affected by musl libc issues, [bpo-46390](https://bugs.python.org/issue46390). -- ``uuid`` module is affected by - [memory leak](https://github.com/emscripten-core/emscripten/issues/16081) - and crasher in Emscripten's ``freeaddrinfo``, -- Recursive ``glob`` leaks file descriptors. - Python's object allocator ``obmalloc`` is disabled by default. - ``ensurepip`` is not available. diff --git a/Tools/wasm/config.site-wasm32-emscripten b/Tools/wasm/config.site-wasm32-emscripten index 60ede49..3363664 100644 --- a/Tools/wasm/config.site-wasm32-emscripten +++ b/Tools/wasm/config.site-wasm32-emscripten @@ -91,7 +91,3 @@ ac_cv_func_linkat=no # alarm signal is not delivered, may need a callback into the event loop? ac_cv_func_alarm=no - -# To use dlopen, you need to use Emscripten's linking support, -# see https://emscripten.org/docs/compiling/Dynamic-Linking.html -ac_cv_func_dlopen=no diff --git a/configure b/configure index a06d4c9..72d8880 100755 --- a/configure +++ b/configure @@ -1017,6 +1017,7 @@ with_framework_name enable_framework with_cxx_main with_emscripten_target +enable_wasm_dynamic_linking with_suffix enable_shared with_static_libpython @@ -1730,6 +1731,9 @@ Optional Features: Unix install. optional INSTALLDIR specifies the installation path. see Mac/README.rst (default is no) + --enable-wasm-dynamic-linking + Enable dynamic linking support for WebAssembly + (default is no) --enable-shared enable building a shared Python library (default is no) --enable-profiling enable C-level code profiling with gprof (default is @@ -6290,6 +6294,30 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_sys_emscripten_target" >&5 $as_echo "$ac_sys_emscripten_target" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --enable-wasm-dynamic-linking" >&5 +$as_echo_n "checking for --enable-wasm-dynamic-linking... " >&6; } +# Check whether --enable-wasm-dynamic-linking was given. +if test "${enable_wasm_dynamic_linking+set}" = set; then : + enableval=$enable_wasm_dynamic_linking; + case $ac_sys_system in #( + Emscripten) : + ;; #( + WASI) : + as_fn_error $? "WASI dynamic linking is not implemented yet." "$LINENO" 5 ;; #( + *) : + as_fn_error $? "--enable-wasm-dynamic-linking only applies to Emscripten and WASI" "$LINENO" 5 + ;; +esac + +else + + enable_wasm_dynamic_linking=missing + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_wasm_dynamic_linking" >&5 +$as_echo "$enable_wasm_dynamic_linking" >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-suffix" >&5 $as_echo_n "checking for --with-suffix... " >&6; } @@ -7760,18 +7788,20 @@ then fi # WASM flags -# TODO: Add -s MAIN_MODULE=2 for dlopen() support. -# The option disables code elimination, which increases code size of main -# binary. All objects must be built with -fPIC. case $ac_sys_system/$ac_sys_emscripten_target in #( Emscripten/browser*) : - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1" - LINKFORSHARED="--preload-file \$(WASM_ASSETS_DIR)" + LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH=1" + LINKFORSHARED="--preload-file=\$(WASM_ASSETS_DIR)" + if test "x$enable_wasm_dynamic_linking" = xyes; then : + + as_fn_append LINKFORSHARED " -sMAIN_MODULE=1" + +fi WASM_ASSETS_DIR=".\$(prefix)" WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py" if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "browser-debug"; then - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" + LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS=1" LINKFORSHARED="$LINKFORSHARED -gsource-map --emit-symbol-map" else LINKFORSHARED="$LINKFORSHARED -O2 -g0" @@ -7779,11 +7809,16 @@ case $ac_sys_system/$ac_sys_emscripten_target in #( ;; #( Emscripten/node*) : - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s USE_PTHREADS=1" - LINKFORSHARED="-s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1" + LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH=1 -sNODERAWFS=1 -sUSE_PTHREADS=1" + LINKFORSHARED="-sPROXY_TO_PTHREAD=1 -sEXIT_RUNTIME=1" + if test "x$enable_wasm_dynamic_linking" = xyes; then : + + as_fn_append LINKFORSHARED " -sMAIN_MODULE=1" + +fi CFLAGS_NODIST="$CFLAGS_NODIST -pthread" if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "node-debug"; then - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" + LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS=1" LINKFORSHARED="$LINKFORSHARED -gseparate-dwarf --emit-symbol-map" else LINKFORSHARED="$LINKFORSHARED -O2 -g0" @@ -7808,6 +7843,18 @@ $as_echo "#define _WASI_EMULATED_PROCESS_CLOCKS 1" >>confdefs.h ;; esac +case $enable_wasm_dynamic_linking in #( + yes) : + ac_cv_func_dlopen=yes ;; #( + no) : + ac_cv_func_dlopen=no ;; #( + missing) : + + ;; #( + *) : + ;; +esac + @@ -10469,10 +10516,6 @@ then Linux*|GNU*|QNX*|VxWorks*|Haiku*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; - Emscripten*) - LDSHARED='$(CC) -shared -s SIDE_MODULE=1' - LDCXXSHARED='$(CXX) -shared -s SIDE_MODULE=1' - ;; FreeBSD*) if [ "`$CC -dM -E - &5 $as_echo "$LDSHARED" >&6; } LDCXXSHARED=${LDCXXSHARED-$LDSHARED} + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking BLDSHARED flags" >&5 +$as_echo_n "checking BLDSHARED flags... " >&6; } BLDSHARED=${BLDSHARED-$LDSHARED} +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $BLDSHARED" >&5 +$as_echo "$BLDSHARED" >&6; } + # CCSHARED are the C *flags* used to create objects to go into a shared # library (module) -- this is only needed for a few systems { $as_echo "$as_me:${as_lineno-$LINENO}: checking CCSHARED" >&5 @@ -10540,6 +10599,12 @@ then fi;; Linux-android*) ;; Linux*|GNU*) CCSHARED="-fPIC";; + Emscripten*|WASI*) + if test "x$enable_wasm_dynamic_linking" = xyes; then : + + CCSHARED="-fPIC" + +fi;; FreeBSD*|NetBSD*|OpenBSD*|DragonFly*) CCSHARED="-fPIC";; Haiku*) CCSHARED="-fPIC";; OpenUNIX*|UnixWare*) @@ -10647,6 +10712,13 @@ then CFLAGSFORSHARED='$(CCSHARED)' esac fi + +if test "x$enable_wasm_dynamic_linking" = xyes; then : + + CFLAGSFORSHARED='$(CCSHARED)' + +fi + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CFLAGSFORSHARED" >&5 $as_echo "$CFLAGSFORSHARED" >&6; } @@ -15379,7 +15451,8 @@ $as_echo "yes" >&6; } fi if test "$have_zlib" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$ZLIB_LIBS" = "-lz"; then - ZLIB_LIBS="-s USE_ZLIB=1" + ZLIB_CFLAGS="-sUSE_ZLIB=1" + ZLIB_LIBS="-sUSE_ZLIB=1" fi if test "x$have_zlib" = xyes; then : @@ -15612,7 +15685,8 @@ $as_echo "yes" >&6; } fi if test "$have_bzip2" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$BZIP2_LIBS" = "-lbz2"; then - BZIP2_LIBS="-s USE_BZIP2=1" + BZIP2_CFLAGS="-sUSE_BZIP2=1" + BZIP2_LIBS="-sUSE_BZIP2=1" fi diff --git a/configure.ac b/configure.ac index abcd379..fda2312 100644 --- a/configure.ac +++ b/configure.ac @@ -1107,6 +1107,25 @@ dnl are free to remove them in the future. ]) AC_MSG_RESULT([$ac_sys_emscripten_target]) +dnl On Emscripten dlopen() requires -s MAIN_MODULE and -fPIC. The flags +dnl disables dead code elimination and increases the size of the WASM module +dnl by about 1.5 to 2MB. MAIN_MODULE defines __wasm_mutable_globals__. +dnl See https://emscripten.org/docs/compiling/Dynamic-Linking.html +AC_MSG_CHECKING([for --enable-wasm-dynamic-linking]) +AC_ARG_ENABLE([wasm-dynamic-linking], + [AS_HELP_STRING([--enable-wasm-dynamic-linking], + [Enable dynamic linking support for WebAssembly (default is no)])], +[ + AS_CASE([$ac_sys_system], + [Emscripten], [], + [WASI], [AC_MSG_ERROR([WASI dynamic linking is not implemented yet.])], + [AC_MSG_ERROR([--enable-wasm-dynamic-linking only applies to Emscripten and WASI])] + ) +], [ + enable_wasm_dynamic_linking=missing +]) +AC_MSG_RESULT([$enable_wasm_dynamic_linking]) + AC_MSG_CHECKING([for --with-suffix]) AC_ARG_WITH([suffix], [AS_HELP_STRING([--with-suffix=SUFFIX], [set executable suffix to SUFFIX (default is empty, yes is mapped to '.exe')])], @@ -1890,29 +1909,32 @@ then fi # WASM flags -# TODO: Add -s MAIN_MODULE=2 for dlopen() support. -# The option disables code elimination, which increases code size of main -# binary. All objects must be built with -fPIC. AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], [Emscripten/browser*], [ - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1" - LINKFORSHARED="--preload-file \$(WASM_ASSETS_DIR)" + LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH=1" + LINKFORSHARED="--preload-file=\$(WASM_ASSETS_DIR)" + AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ + AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE=1"]) + ]) WASM_ASSETS_DIR=".\$(prefix)" WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py" dnl separate-dwarf does not seem to work in Chrome DevTools Support. if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "browser-debug"; then - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" + LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS=1" LINKFORSHARED="$LINKFORSHARED -gsource-map --emit-symbol-map" else LINKFORSHARED="$LINKFORSHARED -O2 -g0" fi ], [Emscripten/node*], [ - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s USE_PTHREADS=1" - LINKFORSHARED="-s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1" + LDFLAGS_NODIST="$LDFLAGS_NODIST -sALLOW_MEMORY_GROWTH=1 -sNODERAWFS=1 -sUSE_PTHREADS=1" + LINKFORSHARED="-sPROXY_TO_PTHREAD=1 -sEXIT_RUNTIME=1" + AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ + AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE=1"]) + ]) CFLAGS_NODIST="$CFLAGS_NODIST -pthread" if test "$Py_DEBUG" = 'true' -o "$ac_sys_emscripten_target" = "node-debug"; then - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" + LDFLAGS_NODIST="$LDFLAGS_NODIST -sASSERTIONS=1" LINKFORSHARED="$LINKFORSHARED -gseparate-dwarf --emit-symbol-map" else LINKFORSHARED="$LINKFORSHARED -O2 -g0" @@ -1927,6 +1949,12 @@ AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], ] ) +AS_CASE([$enable_wasm_dynamic_linking], + [yes], [ac_cv_func_dlopen=yes], + [no], [ac_cv_func_dlopen=no], + [missing], [] +) + AC_SUBST(BASECFLAGS) AC_SUBST(CFLAGS_NODIST) AC_SUBST(LDFLAGS_NODIST) @@ -2929,10 +2957,6 @@ then Linux*|GNU*|QNX*|VxWorks*|Haiku*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; - Emscripten*) - LDSHARED='$(CC) -shared -s SIDE_MODULE=1' - LDCXXSHARED='$(CXX) -shared -s SIDE_MODULE=1' - ;; FreeBSD*) if [[ "`$CC -dM -E - = 1.2.0], [ ]) if test "$have_zlib" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$ZLIB_LIBS" = "-lz"; then - ZLIB_LIBS="-s USE_ZLIB=1" + ZLIB_CFLAGS="-sUSE_ZLIB=1" + ZLIB_LIBS="-sUSE_ZLIB=1" fi dnl binascii can use zlib for optimized crc32. @@ -4482,7 +4532,8 @@ PKG_CHECK_MODULES([BZIP2], [bzip2], [have_bzip2=yes], [ ]) if test "$have_bzip2" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$BZIP2_LIBS" = "-lbz2"; then - BZIP2_LIBS="-s USE_BZIP2=1" + BZIP2_CFLAGS="-sUSE_BZIP2=1" + BZIP2_LIBS="-sUSE_BZIP2=1" fi -- cgit v0.12