gh-127629: Add ctypes to the Emscripten build (#127683)

Adds tooling to build libffi and add ctypes to the stdlib for Emscripten.

Author: hoodmane

Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst

@@ -0,0 +1 @@
+Emscripten builds now include ctypes support.

Tools/wasm/README.md

@@ -61,6 +61,7 @@ or you can break it out into four separate steps:
 ```shell
 python Tools/wasm/emscripten configure-build-python
 python Tools/wasm/emscripten make-build-python
+python Tools/wasm/emscripten make-libffi
 python Tools/wasm/emscripten configure-host
 python Tools/wasm/emscripten make-host
 ```

Tools/wasm/emscripten/__main__.py

@@ -9,6 +9,7 @@
 import sys
 import sysconfig
 import tempfile
+from urllib.request import urlopen
 from pathlib import Path
 from textwrap import dedent
 
@@ -22,9 +23,13 @@
 CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent
 
 CROSS_BUILD_DIR = CHECKOUT / "cross-build"
-BUILD_DIR = CROSS_BUILD_DIR / "build"
+NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build"
 HOST_TRIPLE = "wasm32-emscripten"
-HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE
+
+DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build"
+HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build"
+HOST_DIR = HOST_BUILD_DIR / "python"
+PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix"
 
 LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
 LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8")
@@ -118,16 +123,16 @@ def build_platform():
 
 def build_python_path():
     """The path to the build Python binary."""
-    binary = BUILD_DIR / "python"
+    binary = NATIVE_BUILD_DIR / "python"
     if not binary.is_file():
         binary = binary.with_suffix(".exe")
         if not binary.is_file():
-            raise FileNotFoundError("Unable to find `python(.exe)` in " f"{BUILD_DIR}")
+            raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}")
 
     return binary
 
 
-@subdir(BUILD_DIR, clean_ok=True)
+@subdir(NATIVE_BUILD_DIR, clean_ok=True)
 def configure_build_python(context, working_dir):
     """Configure the build/host Python."""
     if LOCAL_SETUP.exists():
@@ -143,7 +148,7 @@ def configure_build_python(context, working_dir):
     call(configure, quiet=context.quiet)
 
 
-@subdir(BUILD_DIR)
+@subdir(NATIVE_BUILD_DIR)
 def make_build_python(context, working_dir):
     """Make/build the build Python."""
     call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet)
@@ -159,6 +164,23 @@ def make_build_python(context, working_dir):
     print(f"🎉 {binary} {version}")
 
 
+@subdir(HOST_BUILD_DIR, clean_ok=True)
+def make_emscripten_libffi(context, working_dir):
+    shutil.rmtree(working_dir / "libffi-3.4.6", ignore_errors=True)
+    with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tmp_file:
+        with urlopen(
+            "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz"
+        ) as response:
+            shutil.copyfileobj(response, tmp_file)
+        shutil.unpack_archive(tmp_file.name, working_dir)
+    call(
+        [EMSCRIPTEN_DIR / "make_libffi.sh"],
+        env=updated_env({"PREFIX": PREFIX_DIR}),
+        cwd=working_dir / "libffi-3.4.6",
+        quiet=context.quiet,
+    )
+
+
 @subdir(HOST_DIR, clean_ok=True)
 def configure_emscripten_python(context, working_dir):
     """Configure the emscripten/host build."""
@@ -168,7 +190,7 @@ def configure_emscripten_python(context, working_dir):
 
     emscripten_build_dir = working_dir.relative_to(CHECKOUT)
 
-    python_build_dir = BUILD_DIR / "build"
+    python_build_dir = NATIVE_BUILD_DIR / "build"
     lib_dirs = list(python_build_dir.glob("lib.*"))
     assert (
         len(lib_dirs) == 1
@@ -183,12 +205,18 @@ def configure_emscripten_python(context, working_dir):
         sysconfig_data += "-pydebug"
 
     host_runner = context.host_runner
-    env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
+    pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve()
+    env_additions = {
+        "CONFIG_SITE": config_site,
+        "HOSTRUNNER": host_runner,
+        "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir),
+    }
     build_python = os.fsdecode(build_python_path())
     configure = [
         "emconfigure",
         os.path.relpath(CHECKOUT / "configure", working_dir),
         "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2",
+        "PKG_CONFIG=pkg-config",
         f"--host={HOST_TRIPLE}",
         f"--build={build_platform()}",
         f"--with-build-python={build_python}",
@@ -197,7 +225,7 @@ def configure_emscripten_python(context, working_dir):
         "--disable-ipv6",
         "--enable-big-digits=30",
         "--enable-wasm-dynamic-linking",
-        f"--prefix={HOST_DIR}",
+        f"--prefix={PREFIX_DIR}",
     ]
     if pydebug:
         configure.append("--with-pydebug")
@@ -264,6 +292,7 @@ def build_all(context):
     steps = [
         configure_build_python,
         make_build_python,
+        make_emscripten_libffi,
         configure_emscripten_python,
         make_emscripten_python,
     ]
@@ -292,18 +321,30 @@ def main():
     configure_build = subcommands.add_parser(
         "configure-build-python", help="Run `configure` for the " "build Python"
     )
+    make_libffi_cmd = subcommands.add_parser(
+        "make-libffi", help="Clone libffi repo, configure and build it for emscripten"
+    )
     make_build = subcommands.add_parser(
         "make-build-python", help="Run `make` for the build Python"
     )
     configure_host = subcommands.add_parser(
         "configure-host",
         help="Run `configure` for the host/emscripten (pydebug builds are inferred from the build Python)",
     )
-    make_host = subcommands.add_parser("make-host", help="Run `make` for the host/emscripten")
+    make_host = subcommands.add_parser(
+        "make-host", help="Run `make` for the host/emscripten"
+    )
     clean = subcommands.add_parser(
         "clean", help="Delete files and directories created by this script"
     )
-    for subcommand in build, configure_build, make_build, configure_host, make_host:
+    for subcommand in (
+        build,
+        configure_build,
+        make_libffi_cmd,
+        make_build,
+        configure_host,
+        make_host,
+    ):
         subcommand.add_argument(
             "--quiet",
             action="store_true",
@@ -336,6 +377,7 @@ def main():
     context = parser.parse_args()
 
     dispatch = {
+        "make-libffi": make_emscripten_libffi,
         "configure-build-python": configure_build_python,
         "make-build-python": make_build_python,
         "configure-host": configure_emscripten_python,

Tools/wasm/emscripten/make_libffi.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set +e
+
+export CFLAGS="-O2 -fPIC -DWASM_BIGINT"
+export CXXFLAGS="$CFLAGS"
+
+# Build paths
+export CPATH="$PREFIX/include"
+export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig"
+export EM_PKG_CONFIG_PATH="$PKG_CONFIG_PATH"
+
+# Specific variables for cross-compilation
+export CHOST="wasm32-unknown-linux" # wasm32-unknown-emscripten
+
+emconfigure ./configure --host=$CHOST --prefix="$PREFIX" --enable-static --disable-shared --disable-dependency-tracking \
+  --disable-builddir --disable-multi-os-directory --disable-raw-api --disable-docs
+
+make install
+# Some forgotten headers?
+cp fficonfig.h $PREFIX/include/
+cp include/ffi_common.h $PREFIX/include/