From bfb0788bfcaab7474c1be0605552744e15082ee9 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Tue, 3 Dec 2024 00:30:24 +0100 Subject: gh-127111: Emscripten Make web example work again (#127113) Moves the Emscripten web example into a standalone folder, and updates Makefile targets to build the web example. Instructions for usage have also been added. --- Makefile.pre.in | 51 +-- .../2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst | 2 + Tools/wasm/README.md | 122 +++++-- Tools/wasm/emscripten/web_example/python.html | 373 +++++++++++++++++++++ .../wasm/emscripten/web_example/python.worker.mjs | 104 ++++++ Tools/wasm/emscripten/web_example/server.py | 40 +++ Tools/wasm/emscripten/web_example/wasm_assets.py | 245 ++++++++++++++ Tools/wasm/python.html | 373 --------------------- Tools/wasm/python.worker.js | 87 ----- Tools/wasm/wasm_assets.py | 246 -------------- Tools/wasm/wasm_webserver.py | 46 --- configure | 14 +- configure.ac | 16 +- 13 files changed, 901 insertions(+), 818 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst create mode 100644 Tools/wasm/emscripten/web_example/python.html create mode 100644 Tools/wasm/emscripten/web_example/python.worker.mjs create mode 100755 Tools/wasm/emscripten/web_example/server.py create mode 100755 Tools/wasm/emscripten/web_example/wasm_assets.py delete mode 100644 Tools/wasm/python.html delete mode 100644 Tools/wasm/python.worker.js delete mode 100755 Tools/wasm/wasm_assets.py delete mode 100755 Tools/wasm/wasm_webserver.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 7243547..dd8a3ab 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -269,10 +269,6 @@ SRCDIRS= @SRCDIRS@ # Other subdirectories SUBDIRSTOO= Include Lib Misc -# assets for Emscripten browser builds -WASM_ASSETS_DIR=.$(prefix) -WASM_STDLIB=$(WASM_ASSETS_DIR)/lib/python$(VERSION)/os.py - # Files and directories to be distributed CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in DISTFILES= README.rst ChangeLog $(CONFIGFILES) @@ -737,6 +733,9 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ python-config checksharedmods +.PHONY: build_emscripten +build_emscripten: build_wasm web_example + # Check that the source is clean when building out of source. .PHONY: check-clean-src check-clean-src: @@ -1016,23 +1015,38 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) else true; \ fi -# wasm32-emscripten browser build -# wasm assets directory is relative to current build dir, e.g. "./usr/local". -# --preload-file turns a relative asset path into an absolute path. +# wasm32-emscripten browser web example + +WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/ +web_example/python.html: $(WEBEX_DIR)/python.html + @mkdir -p web_example + @cp $< $@ + +web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs + @mkdir -p web_example + @cp $< $@ -.PHONY: wasm_stdlib -wasm_stdlib: $(WASM_STDLIB) -$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ - $(srcdir)/Tools/wasm/wasm_assets.py \ +web_example/server.py: $(WEBEX_DIR)/server.py + @mkdir -p web_example + @cp $< $@ + +WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip +$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ + $(WEBEX_DIR)/wasm_assets.py \ Makefile pybuilddir.txt Modules/Setup.local - $(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \ - --buildroot . --prefix $(prefix) + $(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \ + --buildroot . --prefix $(prefix) -o $@ -python.html: $(srcdir)/Tools/wasm/python.html python.worker.js - @cp $(srcdir)/Tools/wasm/python.html $@ +web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) + @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ + echo "Can only build web_example when target is Emscripten" ;\ + exit 1 ;\ + fi + cp python.mjs web_example/python.mjs + cp python.wasm web_example/python.wasm -python.worker.js: $(srcdir)/Tools/wasm/python.worker.js - @cp $(srcdir)/Tools/wasm/python.worker.js $@ +.PHONY: web_example +web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB) ############################################################################ # Header files @@ -3053,8 +3067,7 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f _bootstrap_python - -rm -f python.html python*.js python.data python*.symbols python*.map - -rm -f $(WASM_STDLIB) + -rm -rf web_example python.mjs python.wasm python*.symbols python*.map -rm -f Programs/_testembed Programs/_freeze_module -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h diff --git a/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst new file mode 100644 index 0000000..d90067c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst @@ -0,0 +1,2 @@ +Updated the Emscripten web example to use ES6 modules and be built into a +distinct ``web_example`` subfolder. diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 3f4211f..4802d96 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -23,9 +23,9 @@ https://github.com/psf/webassembly for more information. To cross compile to the ``wasm32-emscripten`` platform you need [the Emscripten compiler toolchain](https://emscripten.org/), -a Python interpreter, and an installation of Node version 18 or newer. Emscripten -version 3.1.42 or newer is recommended. All commands below are relative to a checkout -of the Python repository. +a Python interpreter, and an installation of Node version 18 or newer. +Emscripten version 3.1.73 or newer is recommended. All commands below are +relative to a checkout of the Python repository. #### Install [the Emscripten compiler toolchain](https://emscripten.org/docs/getting_started/downloads.html) @@ -50,7 +50,7 @@ sourced. Otherwise the source script removes the environment variable. export EM_COMPILER_WRAPPER=ccache ``` -### Compile and build Python interpreter +#### Compile and build Python interpreter You can use `python Tools/wasm/emscripten` to compile and build targetting Emscripten. You can do everything at once with: @@ -70,6 +70,88 @@ instance, to do a debug build, you can use: python Tools/wasm/emscripten build --with-py-debug ``` +### Running from node + +If you want to run the normal Python CLI, you can use `python.sh`. It takes the +same options as the normal Python CLI entrypoint, though the REPL does not +function and will crash. + +`python.sh` invokes `node_entry.mjs` which imports the Emscripten module for the +Python process and starts it up with the appropriate settings. If you wish to +make a node application that "embeds" the interpreter instead of acting like the +CLI you will need to write your own alternative to `node_entry.mjs`. + + +### The Web Example + +When building for Emscripten, the web example will be built automatically. It is +in the ``web_example`` directory. To run the web example, ``cd`` into the +``web_example`` directory, then run ``python server.py``. This will start a web +server; you can then visit ``http://localhost:8000/python.html`` in a browser to +see a simple REPL example. + +The web example relies on a bug fix in Emscripten version 3.1.73 so if you build +with earlier versions of Emscripten it may not work. The web example uses +``SharedArrayBuffer``. For security reasons browsers only provide +``SharedArrayBuffer`` in secure environments with cross-origin isolation. The +webserver must send cross-origin headers and correct MIME types for the +JavaScript and WebAssembly files. Otherwise the terminal will fail to load with +an error message like ``ReferenceError: SharedArrayBuffer is not defined``. See +more information here: +https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements + +Note that ``SharedArrayBuffer`` is _not required_ to use Python itself, only the +web example. If cross-origin isolation is not appropriate for your use case you +may make your own application embedding `python.mjs` which does not use +``SharedArrayBuffer`` and serve it without the cross-origin isolation headers. + +### Embedding Python in a custom JavaScript application + +You can look at `python.worker.mjs` and `node_entry.mjs` for inspiration. At a +minimum you must import ``createEmscriptenModule`` and you need to call +``createEmscriptenModule`` with an appropriate settings object. This settings +object will need a prerun hook that installs the Python standard library into +the Emscripten file system. + +#### NodeJs + +In Node, you can use the NodeFS to mount the standard library in your native +file system into the Emscripten file system: +```js +import createEmscriptenModule from "./python.mjs"; + +await createEmscriptenModule({ + preRun(Module) { + Module.FS.mount( + Module.FS.filesystems.NODEFS, + { root: "/path/to/python/stdlib" }, + "/lib/", + ); + }, +}); +``` + +#### Browser + +In the browser, the simplest approach is to put the standard library in a zip +file it and install it. With Python 3.14 this could look like: +```js +import createEmscriptenModule from "./python.mjs"; + +await createEmscriptenModule({ + async preRun(Module) { + Module.FS.mkdirTree("/lib/python3.14/lib-dynload/"); + Module.addRunDependency("install-stdlib"); + const resp = await fetch("python3.14.zip"); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { + canOwn: true, + }); + Module.removeRunDependency("install-stdlib"); + }, +}); +``` + ### Limitations and issues #### Network stack @@ -151,38 +233,6 @@ python Tools/wasm/emscripten build --with-py-debug - Test modules are disabled by default. Use ``--enable-test-modules`` build test modules like ``_testcapi``. -### wasm32-emscripten in node - -Node builds use ``NODERAWFS``. - -- Node RawFS allows direct access to the host file system without need to - perform ``FS.mount()`` call. - -### Hosting Python WASM builds - -The simple REPL terminal uses SharedArrayBuffer. For security reasons -browsers only provide the feature in secure environments with cross-origin -isolation. The webserver must send cross-origin headers and correct MIME types -for the JavaScript and WASM files. Otherwise the terminal will fail to load -with an error message like ``Browsers disable shared array buffer``. - -#### Apache HTTP .htaccess - -Place a ``.htaccess`` file in the same directory as ``python.wasm``. - -``` -# .htaccess -Header set Cross-Origin-Opener-Policy same-origin -Header set Cross-Origin-Embedder-Policy require-corp - -AddType application/javascript js -AddType application/wasm wasm - - - AddOutputFilterByType DEFLATE text/html application/javascript application/wasm - -``` - ## WASI (wasm32-wasi) See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi). diff --git a/Tools/wasm/emscripten/web_example/python.html b/Tools/wasm/emscripten/web_example/python.html new file mode 100644 index 0000000..fae1e9a --- /dev/null +++ b/Tools/wasm/emscripten/web_example/python.html @@ -0,0 +1,373 @@ + + + + + + + + + wasm-python terminal + + + + + + +

Simple REPL for Python WASM

+ +
+ + + + +
+
+
+ The simple REPL provides a limited Python experience in the browser. + + Tools/wasm/README.md contains a list of known limitations and + issues. Networking, subprocesses, and threading are not available. +
+ + diff --git a/Tools/wasm/emscripten/web_example/python.worker.mjs b/Tools/wasm/emscripten/web_example/python.worker.mjs new file mode 100644 index 0000000..42c2e1e --- /dev/null +++ b/Tools/wasm/emscripten/web_example/python.worker.mjs @@ -0,0 +1,104 @@ +import createEmscriptenModule from "./python.mjs"; + +class StdinBuffer { + constructor() { + this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT) + this.buffer = new Int32Array(this.sab) + this.readIndex = 1; + this.numberOfCharacters = 0; + this.sentNull = true + } + + prompt() { + this.readIndex = 1 + Atomics.store(this.buffer, 0, -1) + postMessage({ + type: 'stdin', + buffer: this.sab + }) + Atomics.wait(this.buffer, 0, -1) + this.numberOfCharacters = this.buffer[0] + } + + stdin = () => { + while (this.numberOfCharacters + 1 === this.readIndex) { + if (!this.sentNull) { + // Must return null once to indicate we're done for now. + this.sentNull = true + return null + } + this.sentNull = false + // Prompt will reset this.readIndex to 1 + this.prompt() + } + const char = this.buffer[this.readIndex] + this.readIndex += 1 + return char + } +} + +const stdout = (charCode) => { + if (charCode) { + postMessage({ + type: 'stdout', + stdout: charCode, + }) + } else { + console.log(typeof charCode, charCode) + } +} + +const stderr = (charCode) => { + if (charCode) { + postMessage({ + type: 'stderr', + stderr: charCode, + }) + } else { + console.log(typeof charCode, charCode) + } +} + +const stdinBuffer = new StdinBuffer() + +const emscriptenSettings = { + noInitialRun: true, + stdin: stdinBuffer.stdin, + stdout: stdout, + stderr: stderr, + onRuntimeInitialized: () => { + postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab}) + }, + async preRun(Module) { + const versionHex = Module.HEAPU32[Module._Py_Version/4].toString(16); + const versionTuple = versionHex.padStart(8, "0").match(/.{1,2}/g).map((x) => parseInt(x, 16)); + const [major, minor, ..._] = versionTuple; + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); + Module.addRunDependency("install-stdlib"); + const resp = await fetch(`python${major}.${minor}.zip`); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile(`/lib/python${major}${minor}.zip`, new Uint8Array(stdlibBuffer), { canOwn: true }); + Module.removeRunDependency("install-stdlib"); + } +} + +const modulePromise = createEmscriptenModule(emscriptenSettings); + + +onmessage = async (event) => { + if (event.data.type === 'run') { + const Module = await modulePromise; + if (event.data.files) { + for (const [filename, contents] of Object.entries(event.data.files)) { + Module.FS.writeFile(filename, contents) + } + } + const ret = Module.callMain(event.data.args); + postMessage({ + type: 'finished', + returnCode: ret + }) + } +} + diff --git a/Tools/wasm/emscripten/web_example/server.py b/Tools/wasm/emscripten/web_example/server.py new file mode 100755 index 0000000..768e6f8 --- /dev/null +++ b/Tools/wasm/emscripten/web_example/server.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +import argparse +from http import server + +parser = argparse.ArgumentParser( + description="Start a local webserver with a Python terminal." +) +parser.add_argument( + "--port", type=int, default=8000, help="port for the http server to listen on" +) +parser.add_argument( + "--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)" +) + + +class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler): + def end_headers(self) -> None: + self.send_my_headers() + super().end_headers() + + def send_my_headers(self) -> None: + self.send_header("Cross-Origin-Opener-Policy", "same-origin") + self.send_header("Cross-Origin-Embedder-Policy", "require-corp") + + +def main() -> None: + args = parser.parse_args() + if not args.bind: + args.bind = None + + server.test( # type: ignore[attr-defined] + HandlerClass=MyHTTPRequestHandler, + protocol="HTTP/1.1", + port=args.port, + bind=args.bind, + ) + + +if __name__ == "__main__": + main() diff --git a/Tools/wasm/emscripten/web_example/wasm_assets.py b/Tools/wasm/emscripten/web_example/wasm_assets.py new file mode 100755 index 0000000..7f0fa7a --- /dev/null +++ b/Tools/wasm/emscripten/web_example/wasm_assets.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python +"""Create a WASM asset bundle directory structure. + +The WASM asset bundles are pre-loaded by the final WASM build. The bundle +contains: + +- a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip +- os.py as marker module {PREFIX}/lib/python3.11/os.py +- empty lib-dynload directory, to make sure it is copied into the bundle: + {PREFIX}/lib/python3.11/lib-dynload/.empty +""" + +import argparse +import pathlib +import shutil +import sys +import sysconfig +import zipfile +from typing import Dict + +# source directory +SRCDIR = pathlib.Path(__file__).parents[4].absolute() +SRCDIR_LIB = SRCDIR / "Lib" + + +# Library directory relative to $(prefix). +WASM_LIB = pathlib.PurePath("lib") +WASM_STDLIB_ZIP = ( + WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" +) +WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" +WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" + + +# Don't ship large files / packages that are not particularly useful at +# the moment. +OMIT_FILES = ( + # regression tests + "test/", + # package management + "ensurepip/", + "venv/", + # other platforms + "_aix_support.py", + "_osx_support.py", + # webbrowser + "antigravity.py", + "webbrowser.py", + # Pure Python implementations of C extensions + "_pydecimal.py", + "_pyio.py", + # concurrent threading + "concurrent/futures/thread.py", + # Misc unused or large files + "pydoc_data/", +) + +# Synchronous network I/O and protocols are not supported; for example, +# socket.create_connection() raises an exception: +# "BlockingIOError: [Errno 26] Operation in progress". +OMIT_NETWORKING_FILES = ( + "email/", + "ftplib.py", + "http/", + "imaplib.py", + "mailbox.py", + "poplib.py", + "smtplib.py", + "socketserver.py", + # keep urllib.parse for pydoc + "urllib/error.py", + "urllib/request.py", + "urllib/response.py", + "urllib/robotparser.py", + "wsgiref/", +) + +OMIT_MODULE_FILES = { + "_asyncio": ["asyncio/"], + "_curses": ["curses/"], + "_ctypes": ["ctypes/"], + "_decimal": ["decimal.py"], + "_dbm": ["dbm/ndbm.py"], + "_gdbm": ["dbm/gnu.py"], + "_json": ["json/"], + "_multiprocessing": ["concurrent/futures/process.py", "multiprocessing/"], + "pyexpat": ["xml/", "xmlrpc/"], + "readline": ["rlcompleter.py"], + "_sqlite3": ["sqlite3/"], + "_ssl": ["ssl.py"], + "_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"], + "_zoneinfo": ["zoneinfo/"], +} + +SYSCONFIG_NAMES = ( + "_sysconfigdata__emscripten_wasm32-emscripten", + "_sysconfigdata__emscripten_wasm32-emscripten", + "_sysconfigdata__wasi_wasm32-wasi", + "_sysconfigdata__wasi_wasm64-wasi", +) + + +def get_builddir(args: argparse.Namespace) -> pathlib.Path: + """Get builddir path from pybuilddir.txt""" + with open("pybuilddir.txt", encoding="utf-8") as f: + builddir = f.read() + return pathlib.Path(builddir) + + +def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path: + """Get path to sysconfigdata relative to build root""" + assert isinstance(args.builddir, pathlib.Path) + data_name: str = sysconfig._get_sysconfigdata_name() # type: ignore[attr-defined] + if not data_name.startswith(SYSCONFIG_NAMES): + raise ValueError(f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES) + filename = data_name + ".py" + return args.builddir / filename + + +def create_stdlib_zip( + args: argparse.Namespace, + *, + optimize: int = 0, +) -> None: + def filterfunc(filename: str) -> bool: + pathname = pathlib.Path(filename).resolve() + return pathname not in args.omit_files_absolute + + with zipfile.PyZipFile( + args.output, + mode="w", + compression=args.compression, + optimize=optimize, + ) as pzf: + if args.compresslevel is not None: + pzf.compresslevel = args.compresslevel + pzf.writepy(args.sysconfig_data) + for entry in sorted(args.srcdir_lib.iterdir()): + entry = entry.resolve() + if entry.name == "__pycache__": + continue + if entry.name.endswith(".py") or entry.is_dir(): + # writepy() writes .pyc files (bytecode). + pzf.writepy(entry, filterfunc=filterfunc) + + +def detect_extension_modules(args: argparse.Namespace) -> Dict[str, bool]: + modules = {} + + # disabled by Modules/Setup.local ? + with open(args.buildroot / "Makefile") as f: + for line in f: + if line.startswith("MODDISABLED_NAMES="): + disabled = line.split("=", 1)[1].strip().split() + for modname in disabled: + modules[modname] = False + break + + # disabled by configure? + with open(args.sysconfig_data) as f: + data = f.read() + loc: Dict[str, Dict[str, str]] = {} + exec(data, globals(), loc) + + for key, value in loc["build_time_vars"].items(): + if not key.startswith("MODULE_") or not key.endswith("_STATE"): + continue + if value not in {"yes", "disabled", "missing", "n/a"}: + raise ValueError(f"Unsupported value '{value}' for {key}") + + modname = key[7:-6].lower() + if modname not in modules: + modules[modname] = value == "yes" + return modules + + +def path(val: str) -> pathlib.Path: + return pathlib.Path(val).absolute() + + +parser = argparse.ArgumentParser() +parser.add_argument( + "--buildroot", + help="absolute path to build root", + default=pathlib.Path(".").absolute(), + type=path, +) +parser.add_argument( + "--prefix", + help="install prefix", + default=pathlib.Path("/usr/local"), + type=path, +) +parser.add_argument( + "-o", + "--output", + help="output file", + type=path, +) + + +def main() -> None: + args = parser.parse_args() + + relative_prefix = args.prefix.relative_to(pathlib.Path("/")) + args.srcdir = SRCDIR + args.srcdir_lib = SRCDIR_LIB + args.wasm_root = args.buildroot / relative_prefix + args.wasm_stdlib = args.wasm_root / WASM_STDLIB + args.wasm_dynload = args.wasm_root / WASM_DYNLOAD + + # bpo-17004: zipimport supports only zlib compression. + # Emscripten ZIP_STORED + -sLZ4=1 linker flags results in larger file. + args.compression = zipfile.ZIP_DEFLATED + args.compresslevel = 9 + + args.builddir = get_builddir(args) + args.sysconfig_data = get_sysconfigdata(args) + if not args.sysconfig_data.is_file(): + raise ValueError(f"sysconfigdata file {args.sysconfig_data} missing.") + + extmods = detect_extension_modules(args) + omit_files = list(OMIT_FILES) + if sysconfig.get_platform().startswith("emscripten"): + omit_files.extend(OMIT_NETWORKING_FILES) + for modname, modfiles in OMIT_MODULE_FILES.items(): + if not extmods.get(modname): + omit_files.extend(modfiles) + + args.omit_files_absolute = { + (args.srcdir_lib / name).resolve() for name in omit_files + } + + # Empty, unused directory for dynamic libs, but required for site initialization. + args.wasm_dynload.mkdir(parents=True, exist_ok=True) + marker = args.wasm_dynload / ".empty" + marker.touch() + # The rest of stdlib that's useful in a WASM context. + create_stdlib_zip(args) + size = round(args.output.stat().st_size / 1024**2, 2) + parser.exit(0, f"Created {args.output} ({size} MiB)\n") + + +if __name__ == "__main__": + main() diff --git a/Tools/wasm/python.html b/Tools/wasm/python.html deleted file mode 100644 index 81a035a..0000000 --- a/Tools/wasm/python.html +++ /dev/null @@ -1,373 +0,0 @@ - - - - - - - - - wasm-python terminal - - - - - - -

Simple REPL for Python WASM

- -
- - - - -
-
-
- The simple REPL provides a limited Python experience in the browser. - - Tools/wasm/README.md contains a list of known limitations and - issues. Networking, subprocesses, and threading are not available. -
- - diff --git a/Tools/wasm/python.worker.js b/Tools/wasm/python.worker.js deleted file mode 100644 index 4ce4e16..0000000 --- a/Tools/wasm/python.worker.js +++ /dev/null @@ -1,87 +0,0 @@ -class StdinBuffer { - constructor() { - this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT) - this.buffer = new Int32Array(this.sab) - this.readIndex = 1; - this.numberOfCharacters = 0; - this.sentNull = true - } - - prompt() { - this.readIndex = 1 - Atomics.store(this.buffer, 0, -1) - postMessage({ - type: 'stdin', - buffer: this.sab - }) - Atomics.wait(this.buffer, 0, -1) - this.numberOfCharacters = this.buffer[0] - } - - stdin = () => { - while (this.numberOfCharacters + 1 === this.readIndex) { - if (!this.sentNull) { - // Must return null once to indicate we're done for now. - this.sentNull = true - return null - } - this.sentNull = false - // Prompt will reset this.readIndex to 1 - this.prompt() - } - const char = this.buffer[this.readIndex] - this.readIndex += 1 - return char - } -} - -const stdout = (charCode) => { - if (charCode) { - postMessage({ - type: 'stdout', - stdout: charCode, - }) - } else { - console.log(typeof charCode, charCode) - } -} - -const stderr = (charCode) => { - if (charCode) { - postMessage({ - type: 'stderr', - stderr: charCode, - }) - } else { - console.log(typeof charCode, charCode) - } -} - -const stdinBuffer = new StdinBuffer() - -var Module = { - noInitialRun: true, - stdin: stdinBuffer.stdin, - stdout: stdout, - stderr: stderr, - onRuntimeInitialized: () => { - postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab}) - } -} - -onmessage = (event) => { - if (event.data.type === 'run') { - if (event.data.files) { - for (const [filename, contents] of Object.entries(event.data.files)) { - Module.FS.writeFile(filename, contents) - } - } - const ret = callMain(event.data.args) - postMessage({ - type: 'finished', - returnCode: ret - }) - } -} - -importScripts('python.js') diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/wasm_assets.py deleted file mode 100755 index ffa5e30..0000000 --- a/Tools/wasm/wasm_assets.py +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env python -"""Create a WASM asset bundle directory structure. - -The WASM asset bundles are pre-loaded by the final WASM build. The bundle -contains: - -- a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip -- os.py as marker module {PREFIX}/lib/python3.11/os.py -- empty lib-dynload directory, to make sure it is copied into the bundle: - {PREFIX}/lib/python3.11/lib-dynload/.empty -""" - -import argparse -import pathlib -import shutil -import sys -import sysconfig -import zipfile -from typing import Dict - -# source directory -SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() -SRCDIR_LIB = SRCDIR / "Lib" - - -# Library directory relative to $(prefix). -WASM_LIB = pathlib.PurePath("lib") -WASM_STDLIB_ZIP = ( - WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" -) -WASM_STDLIB = ( - WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" -) -WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" - - -# Don't ship large files / packages that are not particularly useful at -# the moment. -OMIT_FILES = ( - # regression tests - "test/", - # package management - "ensurepip/", - "venv/", - # other platforms - "_aix_support.py", - "_osx_support.py", - # webbrowser - "antigravity.py", - "webbrowser.py", - # Pure Python implementations of C extensions - "_pydecimal.py", - "_pyio.py", - # concurrent threading - "concurrent/futures/thread.py", - # Misc unused or large files - "pydoc_data/", -) - -# Synchronous network I/O and protocols are not supported; for example, -# socket.create_connection() raises an exception: -# "BlockingIOError: [Errno 26] Operation in progress". -OMIT_NETWORKING_FILES = ( - "email/", - "ftplib.py", - "http/", - "imaplib.py", - "mailbox.py", - "poplib.py", - "smtplib.py", - "socketserver.py", - # keep urllib.parse for pydoc - "urllib/error.py", - "urllib/request.py", - "urllib/response.py", - "urllib/robotparser.py", - "wsgiref/", -) - -OMIT_MODULE_FILES = { - "_asyncio": ["asyncio/"], - "_curses": ["curses/"], - "_ctypes": ["ctypes/"], - "_decimal": ["decimal.py"], - "_dbm": ["dbm/ndbm.py"], - "_gdbm": ["dbm/gnu.py"], - "_json": ["json/"], - "_multiprocessing": ["concurrent/futures/process.py", "multiprocessing/"], - "pyexpat": ["xml/", "xmlrpc/"], - "readline": ["rlcompleter.py"], - "_sqlite3": ["sqlite3/"], - "_ssl": ["ssl.py"], - "_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"], - "_zoneinfo": ["zoneinfo/"], -} - -SYSCONFIG_NAMES = ( - "_sysconfigdata__emscripten_wasm32-emscripten", - "_sysconfigdata__emscripten_wasm32-emscripten", - "_sysconfigdata__wasi_wasm32-wasi", - "_sysconfigdata__wasi_wasm64-wasi", -) - - -def get_builddir(args: argparse.Namespace) -> pathlib.Path: - """Get builddir path from pybuilddir.txt""" - with open("pybuilddir.txt", encoding="utf-8") as f: - builddir = f.read() - return pathlib.Path(builddir) - - -def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path: - """Get path to sysconfigdata relative to build root""" - assert isinstance(args.builddir, pathlib.Path) - data_name: str = sysconfig._get_sysconfigdata_name() # type: ignore[attr-defined] - if not data_name.startswith(SYSCONFIG_NAMES): - raise ValueError( - f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES - ) - filename = data_name + ".py" - return args.builddir / filename - - -def create_stdlib_zip( - args: argparse.Namespace, - *, - optimize: int = 0, -) -> None: - def filterfunc(filename: str) -> bool: - pathname = pathlib.Path(filename).resolve() - return pathname not in args.omit_files_absolute - - with zipfile.PyZipFile( - args.wasm_stdlib_zip, - mode="w", - compression=args.compression, - optimize=optimize, - ) as pzf: - if args.compresslevel is not None: - pzf.compresslevel = args.compresslevel - pzf.writepy(args.sysconfig_data) - for entry in sorted(args.srcdir_lib.iterdir()): - entry = entry.resolve() - if entry.name == "__pycache__": - continue - if entry.name.endswith(".py") or entry.is_dir(): - # writepy() writes .pyc files (bytecode). - pzf.writepy(entry, filterfunc=filterfunc) - - -def detect_extension_modules(args: argparse.Namespace) -> Dict[str, bool]: - modules = {} - - # disabled by Modules/Setup.local ? - with open(args.buildroot / "Makefile") as f: - for line in f: - if line.startswith("MODDISABLED_NAMES="): - disabled = line.split("=", 1)[1].strip().split() - for modname in disabled: - modules[modname] = False - break - - # disabled by configure? - with open(args.sysconfig_data) as f: - data = f.read() - loc: Dict[str, Dict[str, str]] = {} - exec(data, globals(), loc) - - for key, value in loc["build_time_vars"].items(): - if not key.startswith("MODULE_") or not key.endswith("_STATE"): - continue - if value not in {"yes", "disabled", "missing", "n/a"}: - raise ValueError(f"Unsupported value '{value}' for {key}") - - modname = key[7:-6].lower() - if modname not in modules: - modules[modname] = value == "yes" - return modules - - -def path(val: str) -> pathlib.Path: - return pathlib.Path(val).absolute() - - -parser = argparse.ArgumentParser() -parser.add_argument( - "--buildroot", - help="absolute path to build root", - default=pathlib.Path(".").absolute(), - type=path, -) -parser.add_argument( - "--prefix", - help="install prefix", - default=pathlib.Path("/usr/local"), - type=path, -) - - -def main() -> None: - args = parser.parse_args() - - relative_prefix = args.prefix.relative_to(pathlib.Path("/")) - args.srcdir = SRCDIR - args.srcdir_lib = SRCDIR_LIB - args.wasm_root = args.buildroot / relative_prefix - args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP - args.wasm_stdlib = args.wasm_root / WASM_STDLIB - args.wasm_dynload = args.wasm_root / WASM_DYNLOAD - - # bpo-17004: zipimport supports only zlib compression. - # Emscripten ZIP_STORED + -sLZ4=1 linker flags results in larger file. - args.compression = zipfile.ZIP_DEFLATED - args.compresslevel = 9 - - args.builddir = get_builddir(args) - args.sysconfig_data = get_sysconfigdata(args) - if not args.sysconfig_data.is_file(): - raise ValueError(f"sysconfigdata file {args.sysconfig_data} missing.") - - extmods = detect_extension_modules(args) - omit_files = list(OMIT_FILES) - if sysconfig.get_platform().startswith("emscripten"): - omit_files.extend(OMIT_NETWORKING_FILES) - for modname, modfiles in OMIT_MODULE_FILES.items(): - if not extmods.get(modname): - omit_files.extend(modfiles) - - args.omit_files_absolute = { - (args.srcdir_lib / name).resolve() for name in omit_files - } - - # Empty, unused directory for dynamic libs, but required for site initialization. - args.wasm_dynload.mkdir(parents=True, exist_ok=True) - marker = args.wasm_dynload / ".empty" - marker.touch() - # os.py is a marker for finding the correct lib directory. - shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib) - # The rest of stdlib that's useful in a WASM context. - create_stdlib_zip(args) - size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2) - parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n") - - -if __name__ == "__main__": - main() diff --git a/Tools/wasm/wasm_webserver.py b/Tools/wasm/wasm_webserver.py deleted file mode 100755 index 3d1d5d4..0000000 --- a/Tools/wasm/wasm_webserver.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python -import argparse -from http import server - -parser = argparse.ArgumentParser( - description="Start a local webserver with a Python terminal." -) -parser.add_argument( - "--port", type=int, default=8000, help="port for the http server to listen on" -) -parser.add_argument( - "--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)" -) - - -class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler): - extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy() - extensions_map.update( - { - ".wasm": "application/wasm", - } - ) - - def end_headers(self) -> None: - self.send_my_headers() - super().end_headers() - - def send_my_headers(self) -> None: - self.send_header("Cross-Origin-Opener-Policy", "same-origin") - self.send_header("Cross-Origin-Embedder-Policy", "require-corp") - - -def main() -> None: - args = parser.parse_args() - if not args.bind: - args.bind = None - - server.test( # type: ignore[attr-defined] - HandlerClass=MyHTTPRequestHandler, - protocol="HTTP/1.1", - port=args.port, - bind=args.bind, - ) - -if __name__ == "__main__": - main() diff --git a/configure b/configure index 4e40432..7efda04 100755 --- a/configure +++ b/configure @@ -8333,8 +8333,12 @@ fi fi -elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then - DEF_MAKE_ALL_RULE="build_wasm" +elif test "$ac_sys_system" = "Emscripten"; then + DEF_MAKE_ALL_RULE="build_emscripten" + REQUIRE_PGO="no" + DEF_MAKE_RULE="all" +elif test "$ac_sys_system" = "WASI"; then + DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" DEF_MAKE_RULE="all" else @@ -9425,12 +9429,13 @@ else $as_nop wasm_debug=no fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520" + as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520" as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS" + as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS,callMain" + as_fn_append LDFLAGS_NODIST " -sEXPORTED_FUNCTIONS=_main,_Py_Version" if test "x$enable_wasm_dynamic_linking" = xyes then : @@ -9447,7 +9452,6 @@ then : as_fn_append LINKFORSHARED " -sPROXY_TO_PTHREAD" fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH" as_fn_append LDFLAGS_NODIST " -sEXIT_RUNTIME" WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" diff --git a/configure.ac b/configure.ac index 4cfced1..15f7d07 100644 --- a/configure.ac +++ b/configure.ac @@ -1854,9 +1854,13 @@ if test "$Py_OPT" = 'true' ; then LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition" ], [], [-Werror]) ]) -elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then - dnl Emscripten does not support shared extensions yet. Build - dnl "python.[js,wasm]", "pybuilddir.txt", and "platform" files. +elif test "$ac_sys_system" = "Emscripten"; then + dnl Build "python.[js,wasm]", "pybuilddir.txt", and "platform" files. + DEF_MAKE_ALL_RULE="build_emscripten" + REQUIRE_PGO="no" + DEF_MAKE_RULE="all" +elif test "$ac_sys_system" = "WASI"; then + dnl Build "python.wasm", "pybuilddir.txt", and "platform" files. DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" DEF_MAKE_RULE="all" @@ -2321,14 +2325,15 @@ AS_CASE([$ac_sys_system], AS_VAR_IF([Py_DEBUG], [yes], [wasm_debug=yes], [wasm_debug=no]) dnl Start with 20 MB and allow to grow - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"]) dnl map int64_t and uint64_t to JS bigint AS_VAR_APPEND([LDFLAGS_NODIST], [" -sWASM_BIGINT"]) dnl Include file system support AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) @@ -2339,7 +2344,6 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([LDFLAGS_NODIST], [" -sUSE_PTHREADS"]) AS_VAR_APPEND([LINKFORSHARED], [" -sPROXY_TO_PTHREAD"]) ]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH"]) dnl not completely sure whether or not we want -sEXIT_RUNTIME, keeping it for now. AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXIT_RUNTIME"]) WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" -- cgit v0.12