From 96e09837fb8031aebe8d823dd19ef664a34bcfad Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 5 Apr 2022 12:21:11 +0300 Subject: bpo-40280: Add limited Emscripten REPL (GH-32284) Co-authored-by: Katie Bell --- Makefile.pre.in | 11 +- .../2022-04-03-11-47-45.bpo-40280.Q_IJik.rst | 2 + Tools/wasm/README.md | 59 +++-- Tools/wasm/python.html | 245 +++++++++++++++++++++ Tools/wasm/python.worker.js | 87 ++++++++ Tools/wasm/wasm_webserver.py | 39 ++++ configure | 2 +- configure.ac | 2 +- 8 files changed, 428 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2022-04-03-11-47-45.bpo-40280.Q_IJik.rst create mode 100644 Tools/wasm/python.html create mode 100644 Tools/wasm/python.worker.js create mode 100755 Tools/wasm/wasm_webserver.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 9e0dae0..d9b96f5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -807,15 +807,22 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) else true; \ fi -# wasm32-emscripten build +# wasm32-emscripten browser build # wasm assets directory is relative to current build dir, e.g. "./usr/local". # --preload-file turns a relative asset path into an absolute path. $(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ - pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py + pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py \ + python.html python.worker.js $(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \ --builddir . --prefix $(prefix) +python.html: $(srcdir)/Tools/wasm/python.html python.worker.js + @cp $(srcdir)/Tools/wasm/python.html $@ + +python.worker.js: $(srcdir)/Tools/wasm/python.worker.js + @cp $(srcdir)/Tools/wasm/python.worker.js $@ + ########################################################################## # Build static libmpdec.a LIBMPDEC_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBMPDEC_CFLAGS@ diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-04-03-11-47-45.bpo-40280.Q_IJik.rst b/Misc/NEWS.d/next/Tools-Demos/2022-04-03-11-47-45.bpo-40280.Q_IJik.rst new file mode 100644 index 0000000..07a9686 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-04-03-11-47-45.bpo-40280.Q_IJik.rst @@ -0,0 +1,2 @@ +Replace Emscripten's limited shell with Katie Bell's browser-ui REPL from +python-wasm project. diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 6b1e7b0..40b82e8 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -55,9 +55,13 @@ emrun builddir/emscripten-browser/python.html or ```shell -python3 -m http.server +./Tools/wasm/wasm_webserver.py ``` +and open http://localhost:8000/builddir/emscripten-browser/python.html . This +directory structure enables the *C/C++ DevTools Support (DWARF)* to load C +and header files with debug builds. + ### Cross compile to wasm32-emscripten for node ``` @@ -79,17 +83,17 @@ popd node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscripten-node/python.js ``` -## wasm32-emscripten limitations and issues +# wasm32-emscripten limitations and issues -- Heap and stack are limited. -- Most stdlib modules with a dependency on external libraries are missing: - ``ctypes``, ``readline``, ``sqlite3``, ``ssl``, and more. -- Shared extension modules are not implemented yet. All extension modules - are statically linked into the main binary. - The experimental configure option ``--enable-wasm-dynamic-linking`` enables - dynamic extensions. -- Processes are not supported. System calls like fork, popen, and subprocess - fail with ``ENOSYS`` or ``ENOSUP``. +Emscripten before 3.1.8 has known bugs that can cause memory corruption and +resource leaks. 3.1.8 contains several fixes for bugs in date and time +functions. + +## Network stack + +- Python's socket module does not work with Emscripten's emulated POSIX + sockets yet. Network modules like ``asyncio``, ``urllib``, ``selectors``, + etc. are not available. - Only ``AF_INET`` and ``AF_INET6`` with ``SOCK_STREAM`` (TCP) or ``SOCK_DGRAM`` (UDP) are available. ``AF_UNIX`` is not supported. - ``socketpair`` does not work. @@ -98,8 +102,21 @@ node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscri does not resolve to a real IP address. IPv6 is not available. - The ``select`` module is limited. ``select.select()`` crashes the runtime due to lack of exectfd support. + +## processes, threads, signals + +- Processes are not supported. System calls like fork, popen, and subprocess + fail with ``ENOSYS`` or ``ENOSUP``. - Signal support is limited. ``signal.alarm``, ``itimer``, ``sigaction`` are not available or do not work correctly. ``SIGTERM`` exits the runtime. +- Keyboard interrupt (CTRL+C) handling is not implemented yet. +- Browser builds cannot start new threads. Node's web workers consume + extra file descriptors. +- Resource-related functions like ``os.nice`` and most functions of the + ``resource`` module are not available. + +## file system + - Most user, group, and permission related function and modules are not supported or don't work as expected, e.g.``pwd`` module, ``grp`` module, ``os.setgroups``, ``os.chown``, and so on. ``lchown`` and `lchmod`` are @@ -113,23 +130,35 @@ node --experimental-wasm-threads --experimental-wasm-bulk-memory builddir/emscri and are disabled. - Large file support crashes the runtime and is disabled. - ``mmap`` module is unstable. flush (``msync``) can crash the runtime. -- Resource-related functions like ``os.nice`` and most functions of the - ``resource`` module are not available. + +## Misc + +- Heap memory and stack size are limited. Recursion or extensive memory + consumption can crash Python. +- Most stdlib modules with a dependency on external libraries are missing, + e.g. ``ctypes``, ``readline``, ``sqlite3``, ``ssl``, and more. +- Shared extension modules are not implemented yet. All extension modules + are statically linked into the main binary. + The experimental configure option ``--enable-wasm-dynamic-linking`` enables + dynamic extensions. - glibc extensions for date and time formatting are not available. - ``locales`` module is affected by musl libc issues, [bpo-46390](https://bugs.python.org/issue46390). - Python's object allocator ``obmalloc`` is disabled by default. - ``ensurepip`` is not available. -### wasm32-emscripten in browsers +## wasm32-emscripten in browsers +- The interactive shell does not handle copy 'n paste and unicode support + well. - The bundled stdlib is limited. Network-related modules, distutils, multiprocessing, dbm, tests and similar modules are not shipped. All other modules are bundled as pre-compiled ``pyc`` files. - Threading is not supported. +- In-memory file system (MEMFS) is not persistent and limited. -### wasm32-emscripten in node +## wasm32-emscripten in node Node builds use ``NODERAWFS``, ``USE_PTHREADS`` and ``PROXY_TO_PTHREAD`` linker options. diff --git a/Tools/wasm/python.html b/Tools/wasm/python.html new file mode 100644 index 0000000..c8d1748 --- /dev/null +++ b/Tools/wasm/python.html @@ -0,0 +1,245 @@ + + + + + + + + + wasm-python terminal + + + + + + +

Simple REPL for Python WASM

+
+
+ + +
+
+ The simple REPL provides a limited Python experience in the browser. + + Tools/wasm/README.md contains a list of known limitations and + issues. Networking, subprocesses, and threading are not available. +
+ + diff --git a/Tools/wasm/python.worker.js b/Tools/wasm/python.worker.js new file mode 100644 index 0000000..c3a8bdf --- /dev/null +++ b/Tools/wasm/python.worker.js @@ -0,0 +1,87 @@ +class StdinBuffer { + constructor() { + this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT) + this.buffer = new Int32Array(this.sab) + this.readIndex = 1; + this.numberOfCharacters = 0; + this.sentNull = true + } + + prompt() { + this.readIndex = 1 + Atomics.store(this.buffer, 0, -1) + postMessage({ + type: 'stdin', + buffer: this.sab + }) + Atomics.wait(this.buffer, 0, -1) + this.numberOfCharacters = this.buffer[0] + } + + stdin = () => { + if (this.numberOfCharacters + 1 === this.readIndex) { + if (!this.sentNull) { + // Must return null once to indicate we're done for now. + this.sentNull = true + return null + } + this.sentNull = false + this.prompt() + } + const char = this.buffer[this.readIndex] + this.readIndex += 1 + // How do I send an EOF?? + return char + } +} + +const stdoutBufSize = 128; +const stdoutBuf = new Int32Array() +let index = 0; + +const stdout = (charCode) => { + if (charCode) { + postMessage({ + type: 'stdout', + stdout: String.fromCharCode(charCode), + }) + } else { + console.log(typeof charCode, charCode) + } +} + +const stderr = (charCode) => { + if (charCode) { + postMessage({ + type: 'stderr', + stderr: String.fromCharCode(charCode), + }) + } else { + console.log(typeof charCode, charCode) + } +} + +const stdinBuffer = new StdinBuffer() + +var Module = { + noInitialRun: true, + stdin: stdinBuffer.stdin, + stdout: stdout, + stderr: stderr, + onRuntimeInitialized: () => { + postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab}) + } +} + +onmessage = (event) => { + if (event.data.type === 'run') { + // TODO: Set up files from event.data.files + const ret = callMain(event.data.args) + postMessage({ + type: 'finished', + returnCode: ret + }) + } +} + +importScripts('python.js') diff --git a/Tools/wasm/wasm_webserver.py b/Tools/wasm/wasm_webserver.py new file mode 100755 index 0000000..ef642bf --- /dev/null +++ b/Tools/wasm/wasm_webserver.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +import argparse +from http import server + +parser = argparse.ArgumentParser( + description="Start a local webserver with a Python terminal." +) +parser.add_argument( + "--port", type=int, default=8000, help="port for the http server to listen on" +) +parser.add_argument( + "--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)" +) + + +class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler): + def end_headers(self): + self.send_my_headers() + super().end_headers() + + def send_my_headers(self): + self.send_header("Cross-Origin-Opener-Policy", "same-origin") + self.send_header("Cross-Origin-Embedder-Policy", "require-corp") + + +def main(): + args = parser.parse_args() + if not args.bind: + args.bind = None + + server.test( + HandlerClass=MyHTTPRequestHandler, + protocol="HTTP/1.1", + port=args.port, + bind=args.bind, + ) + +if __name__ == "__main__": + main() diff --git a/configure b/configure index 72d8880..d6bc717 100755 --- a/configure +++ b/configure @@ -6338,7 +6338,7 @@ else case $ac_sys_system/$ac_sys_emscripten_target in #( Emscripten/browser*) : - EXEEXT=.html ;; #( + EXEEXT=.js ;; #( Emscripten/node*) : EXEEXT=.js ;; #( WASI/*) : diff --git a/configure.ac b/configure.ac index fda2312..53bbc3e 100644 --- a/configure.ac +++ b/configure.ac @@ -1137,7 +1137,7 @@ AC_ARG_WITH([suffix], ) ], [ AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], - [Emscripten/browser*], [EXEEXT=.html], + [Emscripten/browser*], [EXEEXT=.js], [Emscripten/node*], [EXEEXT=.js], [WASI/*], [EXEEXT=.wasm], [EXEEXT=] -- cgit v0.12