gh-95853: Add script to automate WASM build (GH-95828)

Automate WASM build with a new Python script. The script provides
several build profiles with configure flags for Emscripten flavors
and WASI. The script can detect and use Emscripten SDK and WASI SDK from
default locations or env vars.

``configure`` now detects Node arguments and creates HOSTRUNNER
arguments for Node 16. It also sets correct arguments for
``wasm64-emscripten``.

Co-authored-by: Brett Cannon <brett@python.org>

3 files changed, 604 insertions, 8 deletions

diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md
index 6496a29..c4c21b4 100644
--- a/Tools/wasm/README.md
+++ b/Tools/wasm/README.md
@@ -35,7 +35,13 @@ docker run --rm -ti -v $(pwd):/python-wasm/cpython -w /python-wasm/cpython quay.
 
 ### Compile a build Python interpreter
 
-From within the container, run the following commands:
+From within the container, run the following command:
+
+```shell
+./Tools/wasm/wasm_build.py build
+```
+
+The command is roughly equivalent to:
 
 ```shell
 mkdir -p builddir/build
@@ -45,13 +51,13 @@ make -j$(nproc)
 popd
 ```
 
-### Fetch and build additional emscripten ports
+### Cross-compile to wasm32-emscripten for browser
 
 ```shell
-embuilder build zlib bzip2
+./Tools/wasm/wasm_build.py emscripten-browser
 ```
 
-### Cross compile to wasm32-emscripten for browser
+The command is roughly equivalent to:
 
 ```shell
 mkdir -p builddir/emscripten-browser
@@ -85,14 +91,21 @@ and header files with debug builds.
 ### Cross compile to wasm32-emscripten for node
 
 ```shell
-mkdir -p builddir/emscripten-node
-pushd builddir/emscripten-node
+./Tools/wasm/wasm_build.py emscripten-browser-dl
+```
+
+The command is roughly equivalent to:
+
+```shell
+mkdir -p builddir/emscripten-node-dl
+pushd builddir/emscripten-node-dl
 
 CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
   emconfigure ../../configure -C \
     --host=wasm32-unknown-emscripten \
     --build=$(../../config.guess) \
     --with-emscripten-target=node \
+    --enable-wasm-dynamic-linking \
     --with-build-python=$(pwd)/../build/python
 
 emmake make -j$(nproc)
@@ -100,7 +113,7 @@ popd
 ```
 
 ```shell
-node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node/python.js
+node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node-dl/python.js
 ```
 
 (``--experimental-wasm-bigint`` is not needed with recent NodeJS versions)
@@ -199,6 +212,15 @@ Node builds use ``NODERAWFS``.
 - Node RawFS allows direct access to the host file system without need to
   perform ``FS.mount()`` call.
 
+## wasm64-emscripten
+
+- wasm64 requires recent NodeJS and ``--experimental-wasm-memory64``.
+- ``EM_JS`` functions must return ``BigInt()``.
+- ``Py_BuildValue()`` format strings must match size of types. Confusing 32
+  and 64 bits types leads to memory corruption, see
+  [gh-95876](https://github.com/python/cpython/issues/95876) and
+  [gh-95878](https://github.com/python/cpython/issues/95878).
+
 # Hosting Python WASM builds
 
 The simple REPL terminal uses SharedArrayBuffer. For security reasons
@@ -235,6 +257,12 @@ The script ``wasi-env`` sets necessary compiler and linker flags as well as
 ``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``.
 
 ```shell
+./Tools/wasm/wasm_build.py wasi
+```
+
+The command is roughly equivalent to:
+
+```shell
 mkdir -p builddir/wasi
 pushd builddir/wasi
 
diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env
index 6c2d56e..48908b0 100755
--- a/Tools/wasm/wasi-env
+++ b/Tools/wasm/wasi-env
@@ -72,4 +72,5 @@ export CFLAGS LDFLAGS
 export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR
 export PATH
 
-exec "$@"
+# no exec, it makes arvg[0] path absolute.
+"$@"
diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py
new file mode 100755
index 0000000..e7a1f4a
--- /dev/null
+++ b/Tools/wasm/wasm_build.py
@@ -0,0 +1,567 @@
+#!/usr/bin/env python3
+"""Build script for Python on WebAssembly platforms.
+
+  $ ./Tools/wasm/wasm_builder.py emscripten-browser compile
+  $ ./Tools/wasm/wasm_builder.py emscripten-node-dl test
+  $ ./Tools/wasm/wasm_builder.py wasi test
+
+Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking),
+"emscripten-browser", and "wasi".
+
+Emscripten builds require a recent Emscripten SDK. The tools looks for an
+activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages
+(Debian, Homebrew) are not supported.
+
+WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH'
+and falls back to /opt/wasi-sdk.
+"""
+import argparse
+import enum
+import dataclasses
+import os
+import pathlib
+import shlex
+import shutil
+import subprocess
+import sysconfig
+
+# for Python 3.8
+from typing import Any, Dict, Callable, Iterable, List, Optional, Union
+
+SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
+WASMTOOLS = SRCDIR / "Tools" / "wasm"
+BUILDDIR = SRCDIR / "builddir"
+CONFIGURE = SRCDIR / "configure"
+SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local"
+
+HAS_CCACHE = shutil.which("ccache") is not None
+
+# path to WASI-SDK root
+WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk"))
+
+# path to Emscripten SDK config file.
+# auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
+EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten"))
+# 3.1.16 has broken utime()
+EMSDK_MIN_VERSION = (3, 1, 17)
+_MISSING = pathlib.PurePath("MISSING")
+
+# WASM_WEBSERVER = WASMTOOLS / "wasmwebserver.py"
+
+CLEAN_SRCDIR = f"""
+Builds require a clean source directory. Please use a clean checkout or
+run "make clean -C '{SRCDIR}'".
+"""
+
+INSTALL_EMSDK = """
+wasm32-emscripten builds need Emscripten SDK. Please follow instructions at
+https://emscripten.org/docs/getting_started/downloads.html how to install
+Emscripten and how to activate the SDK with ". /path/to/emsdk/emsdk_env.sh".
+
+    git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk
+    cd /path/to/emsdk
+    ./emsdk install latest
+    ./emsdk activate latest
+    source /path/to/emsdk_env.sh
+"""
+
+INSTALL_WASI_SDK = """
+wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from
+https://github.com/WebAssembly/wasi-sdk/releases and install it to
+"/opt/wasi-sdk". Alternatively you can install the SDK in a different location
+and point the environment variable WASI_SDK_PATH to the root directory
+of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW.
+"""
+
+INSTALL_WASMTIME = """
+wasm32-wasi tests require wasmtime on PATH. Please follow instructions at
+https://wasmtime.dev/ to install wasmtime.
+"""
+
+
+def get_emscripten_root(emconfig: pathlib.Path = EM_CONFIG) -> pathlib.PurePath:
+    """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT
+
+    The ".emscripten" config file is a Python snippet that uses "EM_CONFIG"
+    environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten"
+    subdirectory with tools like "emconfigure".
+    """
+    if not emconfig.exists():
+        return _MISSING
+    with open(emconfig, encoding="utf-8") as f:
+        code = f.read()
+    # EM_CONFIG file is a Python snippet
+    local: Dict[str, Any] = {}
+    exec(code, globals(), local)
+    return pathlib.Path(local["EMSCRIPTEN_ROOT"])
+
+
+EMSCRIPTEN_ROOT = get_emscripten_root()
+
+
+class ConditionError(ValueError):
+    def __init__(self, info: str, text: str):
+        self.info = info
+        self.text = text
+
+    def __str__(self):
+        return f"{type(self).__name__}: '{self.info}'\n{self.text}"
+
+
+class MissingDependency(ConditionError):
+    pass
+
+
+class DirtySourceDirectory(ConditionError):
+    pass
+
+
+@dataclasses.dataclass
+class Platform:
+    """Platform-specific settings
+
+    - CONFIG_SITE override
+    - configure wrapper (e.g. emconfigure)
+    - make wrapper (e.g. emmake)
+    - additional environment variables
+    - check function to verify SDK
+    """
+
+    name: str
+    pythonexe: str
+    config_site: Optional[pathlib.PurePath]
+    configure_wrapper: Optional[pathlib.PurePath]
+    make_wrapper: Optional[pathlib.PurePath]
+    environ: dict
+    check: Callable[[], None]
+
+    def getenv(self, profile: "BuildProfile") -> dict:
+        return self.environ.copy()
+
+
+def _check_clean_src():
+    candidates = [
+        SRCDIR / "Programs" / "python.o",
+        SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h",
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR)
+
+
+NATIVE = Platform(
+    "native",
+    # macOS has python.exe
+    pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python",
+    config_site=None,
+    configure_wrapper=None,
+    make_wrapper=None,
+    environ={},
+    check=_check_clean_src,
+)
+
+
+def _check_emscripten():
+    if EMSCRIPTEN_ROOT is _MISSING:
+        raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK)
+    # sanity check
+    emconfigure = EMSCRIPTEN.configure_wrapper
+    if not emconfigure.exists():
+        raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK)
+    # version check
+    version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt"
+    if not version_txt.exists():
+        raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK)
+    with open(version_txt) as f:
+        version = f.read().strip().strip('"')
+    version_tuple = tuple(int(v) for v in version.split("."))
+    if version_tuple < EMSDK_MIN_VERSION:
+        raise MissingDependency(
+            os.fspath(version_txt),
+            f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than "
+            "minimum required version "
+            f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.",
+        )
+    _check_clean_src()
+
+
+EMSCRIPTEN = Platform(
+    "emscripten",
+    pythonexe="python.js",
+    config_site=WASMTOOLS / "config.site-wasm32-emscripten",
+    configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure",
+    make_wrapper=EMSCRIPTEN_ROOT / "emmake",
+    environ={"EM_COMPILER_WRAPPER": "ccache"} if HAS_CCACHE else {},
+    check=_check_emscripten,
+)
+
+
+def _check_wasi():
+    wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld"
+    if not wasm_ld.exists():
+        raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK)
+    wasmtime = shutil.which("wasmtime")
+    if wasmtime is None:
+        raise MissingDependency("wasmtime", INSTALL_WASMTIME)
+    _check_clean_src()
+
+
+WASI = Platform(
+    "wasi",
+    pythonexe="python.wasm",
+    config_site=WASMTOOLS / "config.site-wasm32-wasi",
+    configure_wrapper=WASMTOOLS / "wasi-env",
+    make_wrapper=None,
+    environ={
+        "WASI_SDK_PATH": WASI_SDK_PATH,
+        # workaround for https://github.com/python/cpython/issues/95952
+        "HOSTRUNNER": (
+            "wasmtime run "
+            "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-$(VERSION):/Lib "
+            "--mapdir /::{srcdir} --"
+        ),
+    },
+    check=_check_wasi,
+)
+
+
+class Host(enum.Enum):
+    """Target host triplet"""
+
+    wasm32_emscripten = "wasm32-unknown-emscripten"
+    wasm64_emscripten = "wasm64-unknown-emscripten"
+    wasm32_wasi = "wasm32-unknown-wasi"
+    wasm64_wasi = "wasm64-unknown-wasi"
+    # current platform
+    build = sysconfig.get_config_var("BUILD_GNU_TYPE")
+
+    @property
+    def platform(self) -> Platform:
+        if self.is_emscripten:
+            return EMSCRIPTEN
+        elif self.is_wasi:
+            return WASI
+        else:
+            return NATIVE
+
+    @property
+    def is_emscripten(self) -> bool:
+        cls = type(self)
+        return self in {cls.wasm32_emscripten, cls.wasm64_emscripten}
+
+    @property
+    def is_wasi(self) -> bool:
+        cls = type(self)
+        return self in {cls.wasm32_wasi, cls.wasm64_wasi}
+
+
+class EmscriptenTarget(enum.Enum):
+    """Emscripten-specific targets (--with-emscripten-target)"""
+
+    browser = "browser"
+    browser_debug = "browser-debug"
+    node = "node"
+    node_debug = "node-debug"
+
+    @property
+    def can_execute(self) -> bool:
+        cls = type(self)
+        return self not in {cls.browser, cls.browser_debug}
+
+
+@dataclasses.dataclass
+class BuildProfile:
+    name: str
+    host: Host
+    target: Union[EmscriptenTarget, None] = None
+    dynamic_linking: Union[bool, None] = None
+    pthreads: Union[bool, None] = None
+    testopts: str = "-j2"
+
+    @property
+    def can_execute(self) -> bool:
+        """Can target run pythoninfo and tests?
+
+        Disabled for browser, enabled for all other targets
+        """
+        return self.target is None or self.target.can_execute
+
+    @property
+    def builddir(self) -> pathlib.Path:
+        """Path to build directory"""
+        return BUILDDIR / self.name
+
+    @property
+    def python_cmd(self) -> pathlib.Path:
+        """Path to python executable"""
+        return self.builddir / self.host.platform.pythonexe
+
+    @property
+    def makefile(self) -> pathlib.Path:
+        """Path to Makefile"""
+        return self.builddir / "Makefile"
+
+    @property
+    def configure_cmd(self) -> List[str]:
+        """Generate configure command"""
+        # use relative path, so WASI tests can find lib prefix.
+        # pathlib.Path.relative_to() does not work here.
+        configure = os.path.relpath(CONFIGURE, self.builddir)
+        cmd = [configure, "-C"]
+        platform = self.host.platform
+        if platform.configure_wrapper:
+            cmd.insert(0, os.fspath(platform.configure_wrapper))
+
+        cmd.append(f"--host={self.host.value}")
+        cmd.append(f"--build={Host.build.value}")
+
+        if self.target is not None:
+            assert self.host.is_emscripten
+            cmd.append(f"--with-emscripten-target={self.target.value}")
+
+        if self.dynamic_linking is not None:
+            assert self.host.is_emscripten
+            opt = "enable" if self.dynamic_linking else "disable"
+            cmd.append(f"--{opt}-wasm-dynamic-linking")
+
+        if self.pthreads is not None:
+            assert self.host.is_emscripten
+            opt = "enable" if self.pthreads else "disable"
+            cmd.append(f"--{opt}-wasm-pthreads")
+
+        if self.host != Host.build:
+            cmd.append(f"--with-build-python={BUILD.python_cmd}")
+
+        if platform.config_site is not None:
+            cmd.append(f"CONFIG_SITE={platform.config_site}")
+
+        return cmd
+
+    @property
+    def make_cmd(self) -> List[str]:
+        """Generate make command"""
+        cmd = ["make"]
+        platform = self.host.platform
+        if platform.make_wrapper:
+            cmd.insert(0, os.fspath(platform.make_wrapper))
+        return cmd
+
+    def getenv(self) -> dict:
+        """Generate environ dict for platform"""
+        env = os.environ.copy()
+        env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}")
+        platenv = self.host.platform.getenv(self)
+        for key, value in platenv.items():
+            if isinstance(value, str):
+                value = value.format(
+                    relbuilddir=self.builddir.relative_to(SRCDIR),
+                    srcdir=SRCDIR,
+                )
+            env[key] = value
+        return env
+
+    def _run_cmd(self, cmd: Iterable[str], args: Iterable[str]):
+        cmd = list(cmd)
+        cmd.extend(args)
+        return subprocess.check_call(
+            cmd,
+            cwd=os.fspath(self.builddir),
+            env=self.getenv(),
+        )
+
+    def _check_execute(self):
+        if not self.can_execute:
+            raise ValueError(f"Cannot execute on {self.target}")
+
+    def run_build(self, force_configure: bool = False):
+        """Run configure (if necessary) and make"""
+        if force_configure or not self.makefile.exists():
+            self.run_configure()
+        self.run_make()
+
+    def run_configure(self, *args):
+        """Run configure script to generate Makefile"""
+        os.makedirs(self.builddir, exist_ok=True)
+        return self._run_cmd(self.configure_cmd, args)
+
+    def run_make(self, *args):
+        """Run make (defaults to build all)"""
+        return self._run_cmd(self.make_cmd, args)
+
+    def run_pythoninfo(self):
+        """Run 'make pythoninfo'"""
+        self._check_execute()
+        return self.run_make("pythoninfo")
+
+    def run_test(self):
+        """Run buildbottests"""
+        self._check_execute()
+        return self.run_make("buildbottest", f"TESTOPTS={self.testopts}")
+
+    def run_py(self, *args):
+        """Run Python with hostrunner"""
+        self._check_execute()
+        self.run_make(
+            "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run"
+        )
+
+    def clean(self, all: bool = False):
+        """Clean build directory"""
+        if all:
+            if self.builddir.exists():
+                shutil.rmtree(self.builddir)
+        elif self.makefile.exists():
+            self.run_make("clean")
+
+
+# native build (build Python)
+BUILD = BuildProfile(
+    "build",
+    host=Host.build,
+)
+
+_profiles = [
+    BUILD,
+    # wasm32-emscripten
+    BuildProfile(
+        "emscripten-browser",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.browser,
+        dynamic_linking=True,
+    ),
+    BuildProfile(
+        "emscripten-browser-debug",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.browser_debug,
+        dynamic_linking=True,
+    ),
+    BuildProfile(
+        "emscripten-node-dl",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.node,
+        dynamic_linking=True,
+    ),
+    BuildProfile(
+        "emscripten-node-dl-debug",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.node_debug,
+        dynamic_linking=True,
+    ),
+    BuildProfile(
+        "emscripten-node-pthreads",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.node,
+        pthreads=True,
+    ),
+    BuildProfile(
+        "emscripten-node-pthreads-debug",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.node_debug,
+        pthreads=True,
+    ),
+    # wasm64-emscripten (currently not working)
+    BuildProfile(
+        "wasm64-emscripten-node-debug",
+        host=Host.wasm64_emscripten,
+        target=EmscriptenTarget.node_debug,
+        # MEMORY64 is not compatible with dynamic linking
+        dynamic_linking=False,
+        pthreads=False,
+    ),
+    # wasm32-wasi
+    BuildProfile(
+        "wasi",
+        host=Host.wasm32_wasi,
+        # skip sysconfig test_srcdir
+        testopts="-i '*.test_srcdir' -j2",
+    ),
+    # no SDK available yet
+    # BuildProfile(
+    #    "wasm64-wasi",
+    #    host=Host.wasm64_wasi,
+    # ),
+]
+
+PROFILES = {p.name: p for p in _profiles}
+
+parser = argparse.ArgumentParser(
+    "wasm_build.py",
+    description=__doc__,
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+)
+parser.add_argument(
+    "--clean", "-c", help="Clean build directories first", action="store_true"
+)
+
+platforms = list(PROFILES) + ["cleanall"]
+parser.add_argument(
+    "platform",
+    metavar="PLATFORM",
+    help=f"Build platform: {', '.join(platforms)}",
+    choices=platforms,
+)
+
+ops = ["compile", "pythoninfo", "test", "repl", "clean", "cleanall"]
+parser.add_argument(
+    "op",
+    metavar="OP",
+    help=f"operation: {', '.join(ops)}",
+    choices=ops,
+    default="compile",
+    nargs="?",
+)
+
+
+def main():
+    args = parser.parse_args()
+    if args.platform == "cleanall":
+        for builder in PROFILES.values():
+            builder.clean(all=True)
+        parser.exit(0)
+
+    builder = PROFILES[args.platform]
+    try:
+        builder.host.platform.check()
+    except ConditionError as e:
+        parser.error(str(e))
+
+    # hack for WASI
+    if builder.host.is_wasi and not SETUP_LOCAL.exists():
+        SETUP_LOCAL.touch()
+
+    if args.op in {"compile", "pythoninfo", "repl", "test"}:
+        # all targets need a build Python
+        if builder is not BUILD:
+            if args.clean:
+                BUILD.clean(all=False)
+                BUILD.run_build()
+            elif not BUILD.python_cmd.exists():
+                BUILD.run_build()
+
+        if args.clean:
+            builder.clean(all=False)
+
+        if args.op == "compile":
+            builder.run_build(force_configure=True)
+        else:
+            if not builder.makefile.exists():
+                builder.run_configure()
+            if args.op == "pythoninfo":
+                builder.run_pythoninfo()
+            elif args.op == "repl":
+                builder.run_py()
+            elif args.op == "test":
+                builder.run_test()
+    elif args.op == "clean":
+        builder.clean(all=False)
+    elif args.op == "cleanall":
+        builder.clean(all=True)
+    else:
+        raise ValueError(args.op)
+
+    print(builder.builddir)
+    parser.exit(0)
+
+
+if __name__ == "__main__":
+    main()