Merge pull request #244 from hoodmane/emscripten-ci2

Add CI tests for wasm32-emscripten wheels in Pyodide

Author: messense

.github/workflows/ci.yml

@@ -402,3 +402,52 @@ jobs:
         shell: msys2 {0}
         run: |
           PATH="$PATH:/c/Users/runneradmin/.cargo/bin" nox -s test-mingw
+
+  test-emscripten:
+    name: Test Emscripten
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - run: |
+          PYODIDE_VERSION=0.21.0-alpha.3
+
+          cd emscripten
+          npm i [email protected] prettier
+          cd node_modules/pyodide/
+          node ../prettier/bin-prettier.js -w pyodide.asm.js
+          EMSCRIPTEN_VERSION=$(node -p "require('./repodata.json').info.platform.split('_').slice(1).join('.')")
+          PYTHON_VERSION=3.10.2
+
+          echo "PYODIDE_VERSION=$PYODIDE_VERSION" >> $GITHUB_ENV
+          echo "EMSCRIPTEN_VERSION=$EMSCRIPTEN_VERSION" >> $GITHUB_ENV
+          echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV
+          echo "ORIG_PATH=$PATH" >> $GITHUB_ENV
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          components: rust-src
+          target: wasm32-unknown-emscripten
+          override: true
+      - uses: mymindstorm/setup-emsdk@v11
+        with:
+          version: ${{env.EMSCRIPTEN_VERSION}}
+          actions-cache-folder: emsdk-cache
+      - uses: actions/setup-python@v2
+        id: setup-python
+        with:
+          python-version: ${{env.PYTHON_VERSION}}
+      - run: pip install nox
+      - uses: actions/cache@v3
+        with:
+          path: |
+            tests/pyodide
+          key: ${{ hashFiles('tests/*.js') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }}
+      - uses: Swatinem/rust-cache@v1
+      - name: Test
+        run: |
+          export PATH=$ORIG_PATH:$PATH
+          nox -s test-examples-emscripten

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 
 ## Unreleased
+### Added
+- Add support for extension modules built for wasm32-unknown-emscripten with Pyodide.
+
 ### Changed
 - Locate cdylib artifacts by handling messages from cargo instead of searching target dir (fixes build on MSYS2). [#267](https://github.com/PyO3/setuptools-rust/pull/267)
 - No longer guess cross-compile environment using `HOST_GNU_TYPE` / `BUILD_GNU_TYPE` sysconfig variables. [#269](https://github.com/PyO3/setuptools-rust/pull/269)

emscripten/.gitignore

@@ -0,0 +1,6 @@
+builddir
+main.*
+!main.c
+pybuilddir.txt
+pyodide
+node_modules

emscripten/_sysconfigdata__emscripten_wasm32-emscripten.py

@@ -0,0 +1,16 @@
+# system configuration generated and used by the sysconfig module
+build_time_vars = {
+    "ABIFLAGS": "",
+    "AR": "/src/emsdk/emsdk/upstream/emscripten/emar",
+    "ARFLAGS": "rcs",
+    "BLDSHARED": "emcc -sSIDE_MODULE=1 -L/src/emscripten/python-lib/",
+    "CC": "emcc -I/src/emscripten/python-include/",
+    "CCSHARED": "",
+    "CFLAGS": "-Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g "
+    "-fwrapv -O3 -Wall -O2 -g0 -fPIC",
+    "EXT_SUFFIX": ".cpython-310-wasm32-emscripten.so",
+    "HOST_GNU_TYPE": "wasm32-unknown-emscripten",
+    "LDSHARED": "emcc -sSIDE_MODULE=1",
+    "Py_DEBUG": "0",
+    "py_version_nodot": "310",
+}

emscripten/emcc_wrapper.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+
+
+def update_args(args):
+    # remove -lc. Not sure if it makes a difference but -lc doesn't belong here.
+    # https://github.com/emscripten-core/emscripten/issues/17191
+    for i in reversed(range(len(args))):
+        if args[i] == "c" and args[i - 1] == "-l":
+            del args[i - 1 : i + 1]
+
+    return args
+
+
+def main(args):
+    args = update_args(args)
+    return subprocess.call(["emcc"] + args)
+
+
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    sys.exit(main(args))

emscripten/pyo3_config.ini

@@ -0,0 +1,7 @@
+implementation=CPython
+version=3.10
+shared=true
+abi3=false
+lib_name=python3.10
+pointer_width=32
+suppress_build_script_link_lines=false

emscripten/runner.js

@@ -0,0 +1,38 @@
+const { opendir } = require("node:fs/promises");
+const { loadPyodide } = require("pyodide");
+
+async function findWheel(distDir) {
+  const dir = await opendir(distDir);
+  for await (const dirent of dir) {
+    if (dirent.name.endsWith("whl")) {
+      return dirent.name;
+    }
+  }
+}
+
+const pkgDir = process.argv[2];
+const distDir = pkgDir + "/dist";
+const testDir = pkgDir + "/tests";
+
+async function main() {
+  const wheelName = await findWheel(distDir);
+  const wheelURL = `file:${distDir}/${wheelName}`;
+
+  try {
+    pyodide = await loadPyodide();
+    const FS = pyodide.FS;
+    const NODEFS = FS.filesystems.NODEFS;
+    FS.mkdir("/test_dir");
+    FS.mount(NODEFS, { root: testDir }, "/test_dir");
+    await pyodide.loadPackage(["micropip", "pytest", "tomli"]);
+    const micropip = pyodide.pyimport("micropip");
+    await micropip.install(wheelURL);
+    const pytest = pyodide.pyimport("pytest");
+    errcode = pytest.main(pyodide.toPy(["/test_dir", "-vv"]));
+  } catch (e) {
+    console.error(e);
+    process.exit(1);
+  }
+}
+
+main();

examples/html-py-ever/tests/conftest.py

@@ -0,0 +1,11 @@
+import sys
+import pytest
+
+if sys.platform == "emscripten":
+
+    @pytest.fixture
+    def benchmark():
+        def result(func, *args, **kwargs):
+            return func(*args, **kwargs)
+
+        return result

noxfile.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import tarfile
 from glob import glob
 from pathlib import Path
@@ -73,3 +74,42 @@ def chdir(path: Path):
         session.install("pytest", "cffi")
         session.install("--no-build-isolation", str(examples / "html-py-ever"))
         session.run("pytest", str(examples / "html-py-ever"))
+
+
+@nox.session(name="test-examples-emscripten")
+def test_examples_emscripten(session: nox.Session):
+    session.install(".")
+    emscripten_dir = Path("./emscripten").resolve()
+
+    session.run(
+        "rustup",
+        "component",
+        "add",
+        "rust-src",
+        "--toolchain",
+        "nightly",
+        external=True,
+    )
+    examples_dir = Path("examples").absolute()
+    test_crates = [
+        examples_dir / "html-py-ever",
+        examples_dir / "namespace_package",
+    ]
+    for example in test_crates:
+        env = os.environ.copy()
+        env.update(
+            RUSTUP_TOOLCHAIN="nightly",
+            PYTHONPATH=str(emscripten_dir),
+            _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata__emscripten_wasm32-emscripten",
+            _PYTHON_HOST_PLATFORM="emscripten_3_1_14_wasm32",
+            CARGO_BUILD_TARGET="wasm32-unknown-emscripten",
+            CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER=str(
+                emscripten_dir / "emcc_wrapper.py"
+            ),
+            PYO3_CONFIG_FILE=str(emscripten_dir / "pyo3_config.ini"),
+        )
+        with session.chdir(example):
+            session.run("python", "setup.py", "bdist_wheel", env=env, external=True)
+
+        with session.chdir(emscripten_dir):
+            session.run("node", "runner.js", str(example), external=True)

setuptools_rust/build.py

@@ -168,7 +168,7 @@ def build_extension(
             ]
 
             # OSX requires special linker arguments
-            if sys.platform == "darwin":
+            if rustc_cfgs.get("target_os") == "macos":
                 ext_basename = os.path.basename(self.get_dylib_ext_path(ext, ext.name))
                 rustc_args.extend(
                     [
@@ -184,6 +184,12 @@ def build_extension(
                 # the cdylib, see https://github.com/rust-lang/cargo/issues/10143
                 rustflags.append("-Ctarget-feature=-crt-static")
 
+            elif (rustc_cfgs.get("target_arch"), rustc_cfgs.get("target_os")) == (
+                "wasm32",
+                "emscripten",
+            ):
+                rustc_args.extend(["-C", f"link-args=-sSIDE_MODULE=2 -sWASM_BIGINT"])
+
             command = [
                 self.cargo,
                 "rustc",