diff options
| -rw-r--r-- | Makefile.pre.in | 18 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst | 1 | ||||
| -rw-r--r-- | Tools/wasm/README.md | 55 | ||||
| -rw-r--r-- | Tools/wasm/config.site-wasm32-emscripten | 70 | ||||
| -rwxr-xr-x | Tools/wasm/wasm_assets.py | 174 | ||||
| -rwxr-xr-x | configure | 41 | ||||
| -rw-r--r-- | configure.ac | 31 |
7 files changed, 380 insertions, 10 deletions
diff --git a/Makefile.pre.in b/Makefile.pre.in index 59c92a0..ed77beb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -830,6 +830,22 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) else true; \ fi +# wasm32-emscripten build +# wasm assets directory is relative to current build dir, e.g. "./usr/local". +# --preload-file turns a relative asset path into an absolute path. +WASM_ASSETS_DIR=".$(prefix)" +WASM_STDLIB="$(WASM_ASSETS_DIR)/local/lib/python$(VERSION)/os.py" + +$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ + pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py + $(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \ + --builddir . --prefix $(prefix) + +python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB) + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \ + $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ + -s ASSERTIONS=1 --preload-file $(WASM_ASSETS_DIR) + ########################################################################## # Build static libmpdec.a LIBMPDEC_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBMPDEC_CFLAGS@ @@ -938,6 +954,7 @@ Makefile Modules/config.c: Makefile.pre \ $(SHELL) $(MAKESETUP) -c $(srcdir)/Modules/config.c.in \ -s Modules \ Modules/Setup.local \ + @MODULES_SETUP_STDLIB@ \ $(srcdir)/Modules/Setup.bootstrap \ $(srcdir)/Modules/Setup @mv config.c Modules @@ -2379,6 +2396,7 @@ clean-retain-profile: pycremoval -rm -f pybuilddir.txt -rm -f Lib/lib2to3/*Grammar*.pickle -rm -f _bootstrap_python + -rm -f python.html python.js python.data -rm -f Programs/_testembed Programs/_freeze_module -rm -f Python/deepfreeze/*.[co] -rm -f Python/frozen_modules/*.h diff --git a/Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst b/Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst new file mode 100644 index 0000000..905ee44 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst @@ -0,0 +1 @@ +A new directory ``Tools/wasm`` contains WebAssembly-related helpers like ``config.site`` override for wasm32-emscripten, wasm assets generator to bundle the stdlib, and a README. diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md new file mode 100644 index 0000000..93c76b2 --- /dev/null +++ b/Tools/wasm/README.md @@ -0,0 +1,55 @@ +# Python WebAssembly (WASM) build + +This directory contains configuration and helpers to facilitate cross +compilation of CPython to WebAssembly (WASM). + +## wasm32-emscripten build + +Cross compiling to wasm32-emscripten platform needs the [Emscripten](https://emscripten.org/) +tool chain and a build Python interpreter. +All commands below are relative to a repository checkout. + +### Compile a build Python interpreter + +```shell +mkdir -p builddir/build +pushd builddir/build +../../configure -C +make -j$(nproc) +popd +``` + +### Fetch and build additional emscripten ports + +```shell +embuilder build zlib +``` + +### Cross compile to wasm32-emscripten + +```shell +mkdir -p builddir/emscripten +pushd builddir/emscripten + +CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \ + emconfigure ../../configure -C \ + --host=wasm32-unknown-emscripten \ + --build=$(../../config.guess) \ + --with-build-python=$(pwd)/../build/python + +emmake make -j$(nproc) python.html +``` + +### Test in browser + +Serve `python.html` with a local webserver and open the file in a browser. + +```shell +emrun python.html +``` + +or + +```shell +python3 -m http.server +``` diff --git a/Tools/wasm/config.site-wasm32-emscripten b/Tools/wasm/config.site-wasm32-emscripten new file mode 100644 index 0000000..67304be --- /dev/null +++ b/Tools/wasm/config.site-wasm32-emscripten @@ -0,0 +1,70 @@ +# config.site override for cross compiling to wasm32-emscripten platform +# +# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \ +# emconfigure ./configure --host=wasm32-unknown-emscripten --build=... +# +# Written by Christian Heimes <christian@python.org> +# Partly based on pyodide's pyconfig.undefs.h file. +# + +# cannot be detected in cross builds +ac_cv_buggy_getaddrinfo=no + +# Emscripten has no /dev/pt* +ac_cv_file__dev_ptmx=no +ac_cv_file__dev_ptc=no + +# dummy readelf, Emscripten build does not need readelf. +ac_cv_prog_ac_ct_READELF=true + +# new undefined symbols / unsupported features +ac_cv_func_posix_spawn=no +ac_cv_func_posix_spawnp=no +ac_cv_func_eventfd=no +ac_cv_func_memfd_create=no +ac_cv_func_prlimit=no + +# unsupported syscall, https://github.com/emscripten-core/emscripten/issues/13393 +ac_cv_func_shutdown=no + +# breaks build, see https://github.com/ethanhs/python-wasm/issues/16 +ac_cv_lib_bz2_BZ2_bzCompress=no + +# The rest is based on pyodide +# https://github.com/pyodide/pyodide/blob/main/cpython/pyconfig.undefs.h + +ac_cv_func_epoll=no +ac_cv_func_epoll_create1=no +ac_cv_header_linux_vm_sockets_h=no +ac_cv_func_socketpair=no +ac_cv_func_utimensat=no +ac_cv_func_sigaction=no + +# Untested syscalls in emscripten +ac_cv_func_openat=no +ac_cv_func_mkdirat=no +ac_cv_func_fchownat=no +ac_cv_func_renameat=no +ac_cv_func_linkat=no +ac_cv_func_symlinkat=no +ac_cv_func_readlinkat=no +ac_cv_func_fchmodat=no +ac_cv_func_dup3=no + +# Syscalls not implemented in emscripten +ac_cv_func_preadv2=no +ac_cv_func_preadv=no +ac_cv_func_pwritev2=no +ac_cv_func_pwritev=no +ac_cv_func_pipe2=no +ac_cv_func_nice=no + +# Syscalls that resulted in a segfault +ac_cv_func_utimensat=no +ac_cv_header_sys_ioctl_h=no + +# sockets are supported, but only in non-blocking mode +# ac_cv_header_sys_socket_h=no + +# Unsupported functionality +#undef HAVE_PTHREAD_H diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/wasm_assets.py new file mode 100755 index 0000000..6a40271 --- /dev/null +++ b/Tools/wasm/wasm_assets.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +"""Create a WASM asset bundle directory structure. + +The WASM asset bundles are pre-loaded by the final WASM build. The bundle +contains: + +- a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip +- os.py as marker module {PREFIX}/lib/python3.11/os.py +- empty lib-dynload directory, to make sure it is copied into the bundle {PREFIX}/lib/python3.11/lib-dynload/.empty +""" + +import argparse +import pathlib +import shutil +import sys +import zipfile + +# source directory +SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() +SRCDIR_LIB = SRCDIR / "Lib" + +# sysconfig data relative to build dir. +SYSCONFIGDATA_GLOB = "build/lib.*/_sysconfigdata_*.py" + +# Library directory relative to $(prefix). +WASM_LIB = pathlib.PurePath("lib") +WASM_STDLIB_ZIP = ( + WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" +) +WASM_STDLIB = ( + WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" +) +WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" + + +# Don't ship large files / packages that are not particularly useful at +# the moment. +OMIT_FILES = ( + # regression tests + "test/", + # user interfaces: TK, curses + "curses/", + "idlelib/", + "tkinter/", + "turtle.py", + "turtledemo/", + # package management + "ensurepip/", + "venv/", + # build system + "distutils/", + "lib2to3/", + # concurrency + "concurrent/", + "multiprocessing/", + # deprecated + "asyncore.py", + "asynchat.py", + # Synchronous network I/O and protocols are not supported; for example, + # socket.create_connection() raises an exception: + # "BlockingIOError: [Errno 26] Operation in progress". + "cgi.py", + "cgitb.py", + "email/", + "ftplib.py", + "http/", + "imaplib.py", + "nntplib.py", + "poplib.py", + "smtpd.py", + "smtplib.py", + "socketserver.py", + "telnetlib.py", + "urllib/", + "wsgiref/", + "xmlrpc/", + # dbm / gdbm + "dbm/", + # other platforms + "_aix_support.py", + "_bootsubprocess.py", + "_osx_support.py", + # webbrowser + "antigravity.py", + "webbrowser.py", + # ctypes + "ctypes/", + # Pure Python implementations of C extensions + "_pydecimal.py", + "_pyio.py", + # Misc unused or large files + "pydoc_data/", + "msilib/", +) + +# regression test sub directories +OMIT_SUBDIRS = ( + "ctypes/test/", + "tkinter/test/", + "unittest/test/", +) + + +OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES} +OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS) + + +def filterfunc(name: str) -> bool: + return not name.startswith(OMIT_SUBDIRS_ABSOLUTE) + + +def create_stdlib_zip( + args: argparse.Namespace, compression: int = zipfile.ZIP_DEFLATED, *, optimize: int = 0 +) -> None: + sysconfig_data = list(args.builddir.glob(SYSCONFIGDATA_GLOB)) + if not sysconfig_data: + raise ValueError("No sysconfigdata file found") + + with zipfile.PyZipFile( + args.wasm_stdlib_zip, mode="w", compression=compression, optimize=0 + ) as pzf: + for entry in sorted(args.srcdir_lib.iterdir()): + if entry.name == "__pycache__": + continue + if entry in OMIT_ABSOLUTE: + continue + if entry.name.endswith(".py") or entry.is_dir(): + # writepy() writes .pyc files (bytecode). + pzf.writepy(entry, filterfunc=filterfunc) + for entry in sysconfig_data: + pzf.writepy(entry) + + +def path(val: str) -> pathlib.Path: + return pathlib.Path(val).absolute() + + +parser = argparse.ArgumentParser() +parser.add_argument( + "--builddir", + help="absolute build directory", + default=pathlib.Path(".").absolute(), + type=path, +) +parser.add_argument( + "--prefix", help="install prefix", default=pathlib.Path("/usr/local"), type=path +) + + +def main(): + args = parser.parse_args() + + relative_prefix = args.prefix.relative_to(pathlib.Path("/")) + args.srcdir = SRCDIR + args.srcdir_lib = SRCDIR_LIB + args.wasm_root = args.builddir / relative_prefix + args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP + args.wasm_stdlib = args.wasm_root / WASM_STDLIB + args.wasm_dynload = args.wasm_root / WASM_DYNLOAD + + # Empty, unused directory for dynamic libs, but required for site initialization. + args.wasm_dynload.mkdir(parents=True, exist_ok=True) + marker = args.wasm_dynload / ".empty" + marker.touch() + # os.py is a marker for finding the correct lib directory. + shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib) + # The rest of stdlib that's useful in a WASM context. + create_stdlib_zip(args) + size = round(args.wasm_stdlib_zip.stat().st_size / 1024 ** 2, 2) + parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n") + + +if __name__ == "__main__": + main() @@ -772,6 +772,7 @@ MODULE_TIME_FALSE MODULE_TIME_TRUE MODULE__IO_FALSE MODULE__IO_TRUE +MODULES_SETUP_STDLIB MODULE_BUILDTYPE TEST_MODULES LIBRARY_DEPS @@ -13298,7 +13299,13 @@ fi if test -z "$with_pymalloc" then + case $ac_sys_system in #( + Emscripten) : + with_pymalloc="no" ;; #( + *) : with_pymalloc="yes" + ;; +esac fi if test "$with_pymalloc" != "no" then @@ -21165,12 +21172,22 @@ fi if test "$enable_test_modules" = no; then TEST_MODULES=no - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } else + case $ac_sys_system in #( + Emscripten) : + TEST_MODULES=no ;; #( + *) : TEST_MODULES=yes - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 + ;; +esac +fi +if test "x$TEST_MODULES" = xyes; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + fi @@ -21189,7 +21206,7 @@ case $ac_sys_system in #( py_stdlib_not_available="_scproxy spwd" ;; #( Emscripten) : - py_stdlib_not_available="_curses _curses_panel _dbm _gdbm _multiprocessing _posixshmem _posixsubprocess _scproxy _xxsubinterpreters fcntl grp nis ossaudiodev resource spwd syslog termios" + py_stdlib_not_available="_ctypes _curses _curses_panel _dbm _gdbm _multiprocessing _posixshmem _posixsubprocess _scproxy _tkinter _xxsubinterpreters fcntl grp nis ossaudiodev resource readline spwd syslog termios" ;; #( *) : py_stdlib_not_available="_scproxy" @@ -21205,6 +21222,20 @@ case $host_cpu in #( esac +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for additional Modules/Setup files" >&5 +$as_echo_n "checking for additional Modules/Setup files... " >&6; } +case $ac_sys_system in #( + Emscripten) : + MODULES_SETUP_STDLIB=Modules/Setup.stdlib ;; #( + *) : + MODULES_SETUP_STDLIB= + ;; +esac +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $MODULES_SETUP_STDLIB" >&5 +$as_echo "$MODULES_SETUP_STDLIB" >&6; } + + + MODULE_BLOCK= @@ -25100,7 +25131,7 @@ fi $as_echo "$as_me: creating Makefile" >&6;} $SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \ -s Modules \ - Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup + Modules/Setup.local $MODULES_SETUP_STDLIB $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup mv config.c Modules if test -z "$PKG_CONFIG"; then diff --git a/configure.ac b/configure.ac index 86404bc..050b907 100644 --- a/configure.ac +++ b/configure.ac @@ -3865,7 +3865,11 @@ AC_ARG_WITH(pymalloc, if test -z "$with_pymalloc" then - with_pymalloc="yes" + dnl default to yes except for wasm32-emscripten + AS_CASE([$ac_sys_system], + [Emscripten], [with_pymalloc="no"], + [with_pymalloc="yes"] + ) fi if test "$with_pymalloc" != "no" then @@ -6253,11 +6257,15 @@ AC_ARG_ENABLE(test-modules, AS_HELP_STRING([--disable-test-modules], [don't build nor install test modules])) if test "$enable_test_modules" = no; then TEST_MODULES=no - AC_MSG_RESULT(yes) else - TEST_MODULES=yes - AC_MSG_RESULT(no) + AS_CASE([$ac_sys_system], + [Emscripten], [TEST_MODULES=no], + [TEST_MODULES=yes] + ) fi +AS_VAR_IF([TEST_MODULES], [yes], + [AC_MSG_RESULT(no)], [AC_MSG_RESULT(yes)] +) AC_SUBST(TEST_MODULES) dnl Modules that are not available on some platforms @@ -6272,6 +6280,7 @@ AS_CASE([$ac_sys_system], [FreeBSD*], [py_stdlib_not_available="_scproxy spwd"], [Emscripten], [ py_stdlib_not_available="m4_normalize([ + _ctypes _curses _curses_panel _dbm @@ -6280,12 +6289,14 @@ AS_CASE([$ac_sys_system], _posixshmem _posixsubprocess _scproxy + _tkinter _xxsubinterpreters fcntl grp nis ossaudiodev resource + readline spwd syslog termios @@ -6301,6 +6312,16 @@ AS_CASE([$host_cpu], ) AC_SUBST([MODULE_BUILDTYPE]) +dnl Use Modules/Setup.stdlib as additional provider? +AC_MSG_CHECKING([for additional Modules/Setup files]) +AS_CASE([$ac_sys_system], + [Emscripten], [MODULES_SETUP_STDLIB=Modules/Setup.stdlib], + [MODULES_SETUP_STDLIB=] +) +AC_MSG_RESULT([$MODULES_SETUP_STDLIB]) +AC_SUBST([MODULES_SETUP_STDLIB]) + + dnl _MODULE_BLOCK_ADD([VAR], [VALUE]) dnl internal: adds $1=quote($2) to MODULE_BLOCK AC_DEFUN([_MODULE_BLOCK_ADD], [AS_VAR_APPEND([MODULE_BLOCK], ["$1=_AS_QUOTE([$2])$as_nl"])]) @@ -6515,7 +6536,7 @@ fi AC_MSG_NOTICE([creating Makefile]) $SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \ -s Modules \ - Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup + Modules/Setup.local $MODULES_SETUP_STDLIB $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup mv config.c Modules if test -z "$PKG_CONFIG"; then |