From 3b18af964da9814474a5db9e502962c7e0593e8d Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Tue, 10 Dec 2024 03:32:58 +0100
Subject: gh-127629: Add ctypes to the Emscripten build (#127683)
Adds tooling to build libffi and add ctypes to the stdlib for Emscripten.
---
.../2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst | 1 +
Tools/wasm/README.md | 1 +
Tools/wasm/emscripten/__main__.py | 64 ++++++++++++++++++----
Tools/wasm/emscripten/make_libffi.sh | 21 +++++++
4 files changed, 76 insertions(+), 11 deletions(-)
create mode 100644 Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst
create mode 100755 Tools/wasm/emscripten/make_libffi.sh
diff --git a/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst b/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst
new file mode 100644
index 0000000..52ee84f
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst
@@ -0,0 +1 @@
+Emscripten builds now include ctypes support.
diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md
index 4802d96..2e0fa2f 100644
--- a/Tools/wasm/README.md
+++ b/Tools/wasm/README.md
@@ -61,6 +61,7 @@ or you can break it out into four separate steps:
```shell
python Tools/wasm/emscripten configure-build-python
python Tools/wasm/emscripten make-build-python
+python Tools/wasm/emscripten make-libffi
python Tools/wasm/emscripten configure-host
python Tools/wasm/emscripten make-host
```
diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py
index 6843b6f..4a53e0b 100644
--- a/Tools/wasm/emscripten/__main__.py
+++ b/Tools/wasm/emscripten/__main__.py
@@ -9,6 +9,7 @@ import subprocess
import sys
import sysconfig
import tempfile
+from urllib.request import urlopen
from pathlib import Path
from textwrap import dedent
@@ -22,9 +23,13 @@ EMSCRIPTEN_DIR = Path(__file__).parent
CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent
CROSS_BUILD_DIR = CHECKOUT / "cross-build"
-BUILD_DIR = CROSS_BUILD_DIR / "build"
+NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build"
HOST_TRIPLE = "wasm32-emscripten"
-HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE
+
+DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build"
+HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build"
+HOST_DIR = HOST_BUILD_DIR / "python"
+PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix"
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8")
@@ -118,16 +123,16 @@ def build_platform():
def build_python_path():
"""The path to the build Python binary."""
- binary = BUILD_DIR / "python"
+ binary = NATIVE_BUILD_DIR / "python"
if not binary.is_file():
binary = binary.with_suffix(".exe")
if not binary.is_file():
- raise FileNotFoundError("Unable to find `python(.exe)` in " f"{BUILD_DIR}")
+ raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}")
return binary
-@subdir(BUILD_DIR, clean_ok=True)
+@subdir(NATIVE_BUILD_DIR, clean_ok=True)
def configure_build_python(context, working_dir):
"""Configure the build/host Python."""
if LOCAL_SETUP.exists():
@@ -143,7 +148,7 @@ def configure_build_python(context, working_dir):
call(configure, quiet=context.quiet)
-@subdir(BUILD_DIR)
+@subdir(NATIVE_BUILD_DIR)
def make_build_python(context, working_dir):
"""Make/build the build Python."""
call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet)
@@ -159,6 +164,23 @@ def make_build_python(context, working_dir):
print(f"🎉 {binary} {version}")
+@subdir(HOST_BUILD_DIR, clean_ok=True)
+def make_emscripten_libffi(context, working_dir):
+ shutil.rmtree(working_dir / "libffi-3.4.6", ignore_errors=True)
+ with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tmp_file:
+ with urlopen(
+ "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz"
+ ) as response:
+ shutil.copyfileobj(response, tmp_file)
+ shutil.unpack_archive(tmp_file.name, working_dir)
+ call(
+ [EMSCRIPTEN_DIR / "make_libffi.sh"],
+ env=updated_env({"PREFIX": PREFIX_DIR}),
+ cwd=working_dir / "libffi-3.4.6",
+ quiet=context.quiet,
+ )
+
+
@subdir(HOST_DIR, clean_ok=True)
def configure_emscripten_python(context, working_dir):
"""Configure the emscripten/host build."""
@@ -168,7 +190,7 @@ def configure_emscripten_python(context, working_dir):
emscripten_build_dir = working_dir.relative_to(CHECKOUT)
- python_build_dir = BUILD_DIR / "build"
+ python_build_dir = NATIVE_BUILD_DIR / "build"
lib_dirs = list(python_build_dir.glob("lib.*"))
assert (
len(lib_dirs) == 1
@@ -183,12 +205,18 @@ def configure_emscripten_python(context, working_dir):
sysconfig_data += "-pydebug"
host_runner = context.host_runner
- env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
+ pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve()
+ env_additions = {
+ "CONFIG_SITE": config_site,
+ "HOSTRUNNER": host_runner,
+ "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir),
+ }
build_python = os.fsdecode(build_python_path())
configure = [
"emconfigure",
os.path.relpath(CHECKOUT / "configure", working_dir),
"CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2",
+ "PKG_CONFIG=pkg-config",
f"--host={HOST_TRIPLE}",
f"--build={build_platform()}",
f"--with-build-python={build_python}",
@@ -197,7 +225,7 @@ def configure_emscripten_python(context, working_dir):
"--disable-ipv6",
"--enable-big-digits=30",
"--enable-wasm-dynamic-linking",
- f"--prefix={HOST_DIR}",
+ f"--prefix={PREFIX_DIR}",
]
if pydebug:
configure.append("--with-pydebug")
@@ -264,6 +292,7 @@ def build_all(context):
steps = [
configure_build_python,
make_build_python,
+ make_emscripten_libffi,
configure_emscripten_python,
make_emscripten_python,
]
@@ -292,6 +321,9 @@ def main():
configure_build = subcommands.add_parser(
"configure-build-python", help="Run `configure` for the " "build Python"
)
+ make_libffi_cmd = subcommands.add_parser(
+ "make-libffi", help="Clone libffi repo, configure and build it for emscripten"
+ )
make_build = subcommands.add_parser(
"make-build-python", help="Run `make` for the build Python"
)
@@ -299,11 +331,20 @@ def main():
"configure-host",
help="Run `configure` for the host/emscripten (pydebug builds are inferred from the build Python)",
)
- make_host = subcommands.add_parser("make-host", help="Run `make` for the host/emscripten")
+ make_host = subcommands.add_parser(
+ "make-host", help="Run `make` for the host/emscripten"
+ )
clean = subcommands.add_parser(
"clean", help="Delete files and directories created by this script"
)
- for subcommand in build, configure_build, make_build, configure_host, make_host:
+ for subcommand in (
+ build,
+ configure_build,
+ make_libffi_cmd,
+ make_build,
+ configure_host,
+ make_host,
+ ):
subcommand.add_argument(
"--quiet",
action="store_true",
@@ -336,6 +377,7 @@ def main():
context = parser.parse_args()
dispatch = {
+ "make-libffi": make_emscripten_libffi,
"configure-build-python": configure_build_python,
"make-build-python": make_build_python,
"configure-host": configure_emscripten_python,
diff --git a/Tools/wasm/emscripten/make_libffi.sh b/Tools/wasm/emscripten/make_libffi.sh
new file mode 100755
index 0000000..3c75c4d
--- /dev/null
+++ b/Tools/wasm/emscripten/make_libffi.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set +e
+
+export CFLAGS="-O2 -fPIC -DWASM_BIGINT"
+export CXXFLAGS="$CFLAGS"
+
+# Build paths
+export CPATH="$PREFIX/include"
+export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig"
+export EM_PKG_CONFIG_PATH="$PKG_CONFIG_PATH"
+
+# Specific variables for cross-compilation
+export CHOST="wasm32-unknown-linux" # wasm32-unknown-emscripten
+
+emconfigure ./configure --host=$CHOST --prefix="$PREFIX" --enable-static --disable-shared --disable-dependency-tracking \
+ --disable-builddir --disable-multi-os-directory --disable-raw-api --disable-docs
+
+make install
+# Some forgotten headers?
+cp fficonfig.h $PREFIX/include/
+cp include/ffi_common.h $PREFIX/include/
--
cgit v0.12