build wheels for emscripten (#185)

Co-authored-by: Samuel Colvin <[email protected]>

Author: davidhewitt

.github/actions/build-pgo-wheel/action.yml

@@ -40,9 +40,9 @@ runs:
 
     - name: generate pgo data
       run: |
-        pip install -U pip
-        pip install -r tests/requirements.txt
-        pip install jiter --no-index --no-deps --find-links pgo-wheel --force-reinstall
+        python -m pip install -U pip
+        python -m pip install -r tests/requirements.txt
+        python -m pip install jiter --no-index --no-deps --find-links pgo-wheel --force-reinstall
         python bench.py jiter jiter-cache
         RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2)
         rustup run ${{ inputs.rust-toolchain }} bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/$RUST_HOST/bin/llvm-profdata >> "$GITHUB_ENV"'

.github/workflows/ci.yml

@@ -395,6 +395,64 @@ jobs:
           name: pypi_files_${{ matrix.os }}_${{ matrix.interpreter }}
           path: crates/jiter-python/dist
 
+  build-wasm-emscripten:
+    # only run on push to main and on release
+    if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - id: setup-python
+        name: set up python
+        uses: quansight-labs/setup-python@v5
+        with:
+          python-version: 3.12
+          allow-prereleases: true
+
+      - name: install rust nightly
+        uses: dtolnay/rust-toolchain@nightly
+        with:
+          components: rust-src
+          targets: wasm32-unknown-emscripten
+
+      - name: cache rust
+        uses: Swatinem/rust-cache@v2
+
+      - uses: mymindstorm/setup-emsdk@v14
+        with:
+          # NOTE!: as per https://github.com/pydantic/pydantic-core/pull/149 this version needs to match the version
+          # in node_modules/pyodide/repodata.json, to get the version, run:
+          # `cat node_modules/pyodide/repodata.json | python -m json.tool | rg platform`
+          version: "3.1.58"
+          actions-cache-folder: emsdk-cache
+
+      - name: install deps
+        run: pip install -U pip maturin
+
+      - name: build wheels
+        run: maturin build --release --target wasm32-unknown-emscripten --out dist -i 3.12
+        working-directory: crates/jiter-python
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "18"
+
+      - run: npm install
+        working-directory: crates/jiter-python
+
+      - run: npm run test
+        working-directory: crates/jiter-python
+
+      - run: |
+          ls -lh dist/
+          ls -l dist/
+        working-directory: crates/jiter-python
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wasm_wheels
+          path: crates/jiter-python/dist
+
   inspect-pypi-assets:
     needs: [build, build-sdist, build-pgo]
     runs-on: ubuntu-latest
@@ -524,7 +582,7 @@ jobs:
 
   release:
     needs: [check]
-    if: "success() && startsWith(github.ref, 'refs/tags/')"
+    if: success() && startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     environment: release
 
@@ -541,8 +599,13 @@ jobs:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
 
   release-python:
-    needs: [test-builds-arch, test-builds-os, build-sdist, check]
-    if: "success() && startsWith(github.ref, 'refs/tags/')"
+    needs:
+      - check
+      - test-builds-arch
+      - test-builds-os
+      - build-sdist
+      - build-wasm-emscripten
+    if: success() && startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     environment: release-python
     permissions:
@@ -573,3 +636,16 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages-dir: dist/
+
+      - name: get wasm dist artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: wasm_wheels
+          path: wasm
+
+      - name: upload to github release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            wasm/*.whl
+          prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') }}

crates/jiter-python/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "jiter",
+  "version": "1.0.0",
+  "description": "for running wasm tests.",
+  "author": "Samuel Colvin",
+  "license": "MIT",
+  "homepage": "https://github.com/pydantic/jiter#readme",
+  "main": "tests/emscripten_runner.js",
+  "dependencies": {
+    "prettier": "^2.7.1",
+    "pyodide": "^0.26.3"
+  },
+  "scripts": {
+    "test": "node tests/emscripten_runner.js",
+    "format": "prettier --write 'tests/emscripten_runner.js' 'wasm-preview/*.{html,js}'",
+    "lint": "prettier --check 'tests/emscripten_runner.js' 'wasm-preview/*.{html,js}'"
+  },
+  "prettier": {
+    "singleQuote": true,
+    "trailingComma": "all",
+    "tabWidth": 2,
+    "printWidth": 119,
+    "bracketSpacing": false,
+    "arrowParens": "avoid"
+  }
+}

crates/jiter-python/tests/emscripten_runner.js

@@ -0,0 +1,71 @@
+const {opendir} = require('node:fs/promises');
+const {loadPyodide} = require('pyodide');
+const path = require('path');
+
+async function find_wheel(dist_dir) {
+  const dir = await opendir(dist_dir);
+  for await (const dirent of dir) {
+    if (dirent.name.endsWith('.whl')) {
+      return path.join(dist_dir, dirent.name);
+    }
+  }
+}
+
+async function main() {
+  const root_dir = path.resolve(__dirname, '..');
+  const wheel_path = await find_wheel(path.join(root_dir, 'dist'));
+  const stdout = []
+  const stderr = []
+  let errcode = 1;
+  try {
+    const pyodide = await loadPyodide({
+      stdout: (msg) => {
+        stdout.push(msg)
+      },
+      stderr: (msg) => {
+        stderr.push(msg)
+      }
+    });
+    const FS = pyodide.FS;
+    FS.mkdir('/test_dir');
+    FS.mount(FS.filesystems.NODEFS, {root: path.join(root_dir, 'tests')}, '/test_dir');
+    FS.chdir('/test_dir');
+
+    // mount jiter crate source for benchmark data
+    FS.mkdir('/jiter');
+    FS.mount(FS.filesystems.NODEFS, {root: path.resolve(root_dir, "..", "jiter")}, '/jiter');
+
+    await pyodide.loadPackage(['micropip', 'pytest']);
+    // language=python
+    errcode = await pyodide.runPythonAsync(`
+import micropip
+import importlib
+
+# ugly hack to get tests to work on arm64 (my m1 mac)
+# see https://github.com/pyodide/pyodide/issues/2840
+# import sys; sys.setrecursionlimit(200)
+
+await micropip.install([
+  'dirty_equals',
+  'file:${wheel_path}'
+])
+importlib.invalidate_caches()
+
+print('installed packages:', micropip.list())
+
+import pytest
+pytest.main()
+`);
+  } catch (e) {
+    console.error(e);
+    process.exit(1);
+  }
+  let out = stdout.join('\n')
+  let err = stderr.join('\n')
+  console.log('stdout:\n', out)
+  console.log('stderr:\n', err)
+
+  process.exit(errcode);
+}
+
+main();

crates/jiter-python/tests/test_jiter.py

@@ -2,6 +2,7 @@
 import json
 from decimal import Decimal
 from pathlib import Path
+import sys
 from typing import Any
 
 import jiter
@@ -352,6 +353,9 @@ def test_against_json():
         assert jiter.from_json(data) == json.loads(data)
 
 
+@pytest.mark.skipif(
+    sys.platform == 'emscripten', reason='threads not supported on pyodide'
+)
 def test_multithreaded_parsing():
     """Basic sanity check that running a parse in multiple threads is fine."""
     expected_datas = [json.loads(data) for data in JITER_BENCH_DATAS]