feat: Build and test against Pyodide 0.29 (new pyodide_2025_0 ABI)

.github/workflows/release-python.yml

@@ -360,6 +360,81 @@ jobs:
           name: wheel-${{ matrix.package }}-${{ matrix.job_config }}
           path: dist/*.whl
 
+  build-wheel-pyodide:
+    name: build-wheels (polars, pyodide, wasm32)
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.sha }}
+
+      # Avoid potential out-of-memory errors
+      - name: Set swap space for Linux
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Disable incompatible features
+        env:
+          # Note: If you change this line, you must also copy the change to `test-pyodide.yml`.
+          FEATURES: csv|ipc|ipc_streaming|parquet|sql|async|json|extract_jsonpath|catalog|cloud|polars_cloud|tokio|clipboard|decompress|new_streaming
+        run: |
+          sed -i 's/serde_json = { workspace = true, optional = true }/serde_json = { workspace = true }/' crates/polars-python/Cargo.toml
+          sed -i 's/"serde_json", //' crates/polars-python/Cargo.toml
+          sed -E -i "/^  \"(${FEATURES})\",$/d" crates/polars-python/Cargo.toml py-polars/Cargo.toml
+
+      - name: Set CFLAGS and RUSTFLAGS for wasm32
+        run: |
+            echo "CFLAGS=-fPIC -fwasm-exceptions" >> $GITHUB_ENV
+            echo "RUSTFLAGS=-C link-self-contained=no -C codegen-units=1" >> $GITHUB_ENV
+
+      - name: Check out Emscripten v4.0.9 patches for Pyodide
+        uses: actions/checkout@v6
+        with:
+          ref: 0.29.2
+          path: pyodide-patches
+          repository: pyodide/pyodide
+          sparse-checkout: |
+            emsdk/patches/
+
+      - name: Increase swapfile
+        run: |
+          sudo swapoff -a
+          sudo fallocate -l 15G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          sudo swapon --show
+
+      - name: Build Pyodide wheel
+        uses: pypa/cibuildwheel@9a15d0851eda6787ba362104ce414974f89201bf # brings v0.29.2, todo update with release
+        with:
+          package-dir: py-polars
+          output-dir: wasm-dist
+          only: cp313-pyodide_wasm32
+        env:
+          CIBW_BEFORE_BUILD: |
+            # Apply Emscripten patches for Pyodide
+            for patch in pyodide-patches/emsdk/patches/*.patch; do
+              patch -p1 --verbose -d /home/runner/.cache/cibuildwheel/emsdk-4.0.9/emsdk-4.0.9/upstream/emscripten/ < "$patch"
+            done
+
+            # Add Rust target
+            rustup target add wasm32-unknown-emscripten
+          CIBW_CONFIG_SETTINGS_PYODIDE: "build-args=--profile=dist-release build-args=--manifest-path=Cargo.toml"
+
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-polars-emscripten-wasm32
+          path: wasm-dist/*.whl
+
   publish-to-pypi:
     needs: [base-package, create-sdist, build-wheels]
     environment:
@@ -387,13 +462,19 @@ jobs:
           verbose: true
 
   publish-to-github:
-    needs: [publish-to-pypi]
+    needs: [publish-to-pypi, build-wheel-pyodide]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
         with:
           ref: ${{ inputs.sha }}
 
+      - name: Download Pyodide wheel
+        uses: actions/download-artifact@v4
+        with:
+          name: wheel-polars-emscripten-wasm32
+          path: wasm-dist
+
       - name: Get version from Cargo.toml
         id: version
         run: |

.github/workflows/test-pyodide.yml

@@ -10,8 +10,9 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  RUSTFLAGS: -C link-self-contained=no -C debuginfo=0
-  CFLAGS: -fPIC
+  CARGO_BUILD_JOBS: 1
+  RUSTFLAGS: -C link-self-contained=no -C debuginfo=0 -C codegen-units=1
+  CFLAGS: -fPIC -fwasm-exceptions
 
 defaults:
   run:
@@ -27,33 +28,62 @@ jobs:
       - name: Disable incompatible features
         env:
           # Note: If you change this line, you must also copy the change to `release-python.yml`.
-          FEATURES: csv|ipc|ipc_streaming|parquet|async|scan_lines|json|extract_jsonpath|catalog|cloud|polars_cloud|tokio|clipboard|decompress|new_streaming
+          FEATURES: csv|ipc|ipc_streaming|parquet|sql|async|scan_lines|json|extract_jsonpath|catalog|cloud|polars_cloud|tokio|clipboard|decompress|new_streaming
         run: |
           sed -i 's/serde_json = { workspace = true, optional = true }/serde_json = { workspace = true }/' crates/polars-python/Cargo.toml
           sed -i 's/"serde_json",//' crates/polars-python/Cargo.toml
           sed -E -i "/^  \"(${FEATURES})\",$/d" crates/polars-python/Cargo.toml py-polars/runtime/polars-runtime-32/Cargo.toml
 
-      - name: Setup emsdk
-        uses: mymindstorm/setup-emsdk@v14
+      - name: Check out Emscripten v4.0.9 patches for Pyodide
+        uses: actions/checkout@v6
         with:
-          # This should match the exact version of Emscripten used by Pyodide
-          version: 3.1.58
+          ref: 0.29.2
+          path: pyodide-patches
+          repository: pyodide/pyodide
+          sparse-checkout: |
+            emsdk/patches/
 
-      - name: Install LLVM
-        # This should match the major version of LLVM expected by Emscripten
+      - name: Increase swapfile
         run: |
-          wget https://apt.llvm.org/llvm.sh
-          chmod +x llvm.sh
-          sudo ./llvm.sh 19
-          echo "EM_LLVM_ROOT=/usr/lib/llvm-19/bin" >> $GITHUB_ENV
+          sudo swapoff -a
+          sudo fallocate -l 15G /swapfile
+          sudo chmod 600 /swapfile
+          sudo mkswap /swapfile
+          sudo swapon /swapfile
+          sudo swapon --show
 
-      - name: Build wheel
-        uses: PyO3/maturin-action@v1
+      - name: Build Pyodide wheel
+        uses: pypa/cibuildwheel@9a15d0851eda6787ba362104ce414974f89201bf # brings v0.29.2, todo update with release
         with:
-          command: build
-          target: wasm32-unknown-emscripten
-          args: >
-            --profile dev
-            --manifest-path py-polars/runtime/polars-runtime-32/Cargo.toml
-            --interpreter python3.10
-          maturin-version: 1.7.4
+          package-dir: py-polars/runtime/polars-runtime-32
+          output-dir: wasm-dist
+          only: cp313-pyodide_wasm32
+        env:
+          CIBW_BEFORE_BUILD: |
+            # Apply Emscripten patches for Pyodide
+            for patch in pyodide-patches/emsdk/patches/*.patch; do
+              patch -p1 --verbose -d /home/runner/.cache/cibuildwheel/emsdk-4.0.9/emsdk-4.0.9/upstream/emscripten/ < "$patch"
+            done
+
+            # Add Rust target
+            rustup target add wasm32-unknown-emscripten
+
+            # Build py-polars wheel for later testing
+            pipx run build py-polars --wheel --outdir wasm-dist
+          CIBW_CONFIG_SETTINGS_PYODIDE: "build-args=--profile=dist-release build-args=--manifest-path=Cargo.toml"
+          # This is a subset of the testing + optional dependencies for polars for which
+          # the test suite runs reliably. We exclude some dependencies based on missing
+          # features in the wasm build and other known issues. Note that pytest-xdist
+          # is included to allow recognising xdist group markers in tests, but it will
+          # serve no effect as we run with -n 0 as it is not really supported on WASM.
+          CIBW_TEST_REQUIRES: pytest hypothesis numpy pandas matplotlib pytest-xdist altair
+          CIBW_TEST_COMMAND: |
+            pip install {project}/wasm-dist/polars-*.whl
+            pytest -n 0 {project}/py-polars/tests
+
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: pyodide-wasm-wheel
+          path: wasm-dist/*.whl

py-polars/requirements-dev.txt

@@ -67,7 +67,7 @@ orjson
 pytest==8.3.2
 pytest-codspeed==3.2.0
 pytest-cov==6.0.0
-pytest-xdist==3.6.1
+pytest-xdist==3.6.1; sys_platform != 'emscripten'
 
 # Need moto.server to mock s3fs - see: https://github.com/aio-libs/aiobotocore/issues/755
 moto[s3]==5.0.9

py-polars/tests/unit/cloud/conftest.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+import pytest
+
+from tests.unit.conftest import IS_WASM
+
+if IS_WASM:
+    pytest.skip(
+        "cloud features are not enabled on Emscripten/Pyodide builds",
+        allow_module_level=True,
+    )

py-polars/tests/unit/cloud/test_prepare_cloud_plan.py

@@ -6,10 +6,15 @@
 import polars as pl
 from polars._utils.cloud import prepare_cloud_plan
 from polars.exceptions import InvalidOperationError
+from tests.unit.conftest import IS_WASM
 
 CLOUD_SOURCE = "s3://my-nonexistent-bucket/dataset"
 DST = "s3://my-nonexistent-bucket/output"
 
+pytestmark = pytest.mark.skipif(
+    IS_WASM, reason="Parquet lazy scanning is not available on emscripten"
+)
+
 
 @pytest.mark.parametrize(
     "lf",

py-polars/tests/unit/conftest.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
 import gc
+import importlib.util
 import os
+import platform
 import random
 import string
 import sys
@@ -25,6 +27,27 @@
     profile=os.environ.get("POLARS_HYPOTHESIS_PROFILE", "fast"),  # type: ignore[arg-type]
 )
 
+IS_WASM = sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"]
+
+requires_json = pytest.mark.skipif(IS_WASM, reason="json feature not available in WASM")
+requires_csv = pytest.mark.skipif(IS_WASM, reason="csv feature not available in WASM")
+requires_parquet = pytest.mark.skipif(
+    IS_WASM, reason="parquet feature not available in WASM"
+)
+requires_ipc = pytest.mark.skipif(IS_WASM, reason="ipc feature not available in WASM")
+requires_new_streaming = pytest.mark.skipif(
+    IS_WASM, reason="new_streaming feature not available in WASM"
+)
+requires_multiprocessing = pytest.mark.skipif(
+    IS_WASM, reason="multiprocessing not available in WASM"
+)
+requires_datetime_range = pytest.mark.skipif(
+    IS_WASM, reason="datetime range has usize conversion issues in WASM"
+)
+skip_wasm_differences = pytest.mark.skipif(
+    IS_WASM, reason="test has different results in WASM (hash/random differences)"
+)
+
 # Data type groups
 SIGNED_INTEGER_DTYPES = [pl.Int8(), pl.Int16(), pl.Int32(), pl.Int64(), pl.Int128()]
 UNSIGNED_INTEGER_DTYPES = [

py-polars/tests/unit/constructors/test_constructors.py

@@ -9,18 +9,20 @@
 
 import numpy as np
 import pandas as pd
-import pyarrow as pa
 import pytest
+
+pa = pytest.importorskip("pyarrow")
+
 from packaging.version import parse as parse_version
 from pydantic import BaseModel, Field, TypeAdapter
 
 import polars as pl
 import polars.selectors as cs
 from polars._dependencies import dataclasses, pydantic
 from polars._utils.construction.utils import try_get_type_hints
 from polars.datatypes import numpy_char_code_to_dtype
 from polars.exceptions import DuplicateError, ShapeError
 from polars.testing import assert_frame_equal, assert_series_equal
 from tests.unit.utils.pycapsule_utils import PyCapsuleArrayHolder, PyCapsuleStreamHolder
 
 if TYPE_CHECKING:

py-polars/tests/unit/dataframe/test_df.py

@@ -13,9 +13,10 @@
 from zoneinfo import ZoneInfo
 
 import numpy as np
-import pyarrow as pa
 import pytest
 
+pa = pytest.importorskip("pyarrow")
+
 import polars as pl
 import polars.selectors as cs
 from polars._plr import PySeries

py-polars/tests/unit/dataframe/test_glimpse.py

@@ -7,6 +7,14 @@
 import pytest
 
 import polars as pl
+from tests.unit.conftest import IS_WASM
+
+# TODO: why?
+if IS_WASM:
+    pytest.skip(
+        "Datetime range conversion is not available on emscripten.",
+        allow_module_level=True,
+    )
 
 TEST_DF = pl.DataFrame(
     {

py-polars/tests/unit/dataframe/test_serde.py

@@ -14,13 +14,21 @@
 from polars.exceptions import ComputeError
 from polars.testing import assert_frame_equal
 from polars.testing.parametric import dataframes
+from tests.unit.conftest import IS_WASM
 
 if TYPE_CHECKING:
     from pathlib import Path
 
     from polars._typing import SerializationFormat
 
 
+if IS_WASM:
+    pytest.skip(
+        "JSON serialisation feature is not enabled on Emscripten/Pyodide builds, and threading/multiprocessing is not supported in Pyodide",
+        allow_module_level=True,
+    )
+
+
 def test_df_serde_roundtrip_binary(df: pl.DataFrame) -> None:
     serialized = df.serialize()
     result = pl.DataFrame.deserialize(serialized, format="binary")

py-polars/tests/unit/dataframe/test_upsample.py

@@ -9,6 +9,7 @@
 import polars as pl
 from polars.exceptions import InvalidOperationError
 from polars.testing import assert_frame_equal
+from tests.unit.conftest import requires_datetime_range
 
 if TYPE_CHECKING:
     from datetime import timezone
@@ -77,6 +78,7 @@ def test_upsample(time_zone: str | None, tzinfo: ZoneInfo | timezone | None) ->
 
 
 @pytest.mark.parametrize("time_zone", [None, "America/Chicago"])
+@requires_datetime_range
 def test_upsample_crossing_dst(time_zone: str | None) -> None:
     df = pl.DataFrame(
         {

py-polars/tests/unit/datatypes/test_bool.py

@@ -5,6 +5,7 @@
 
 import polars as pl
 from polars.testing import assert_frame_equal
+from tests.unit.conftest import requires_new_streaming
 
 
 @pytest.mark.slow
@@ -64,6 +65,7 @@ def val(expr: pl.Expr) -> dict[str, list[bool]]:
     assert val(True ^ pl.col("x")) == {"literal": [True, False]}
 
 
+@requires_new_streaming
 def test_bool_min_max_streaming_23343() -> None:
     n_rows = pl.thread_pool_size() + 1
     df = pl.DataFrame({"k": list(range(n_rows)), "a": [True] * n_rows})

py-polars/tests/unit/datatypes/test_categorical.py

@@ -5,8 +5,6 @@
 import pickle
 from typing import TYPE_CHECKING
 
-import pyarrow as pa
-import pyarrow.parquet as pq
 import pytest
 
 import polars as pl
@@ -15,6 +13,9 @@
 if TYPE_CHECKING:
     from collections.abc import Callable
 
+pa = pytest.importorskip("pyarrow")
+pq = pytest.importorskip("pyarrow.parquet")
+
 
 def test_categorical_full_outer_join() -> None:
     df1 = pl.DataFrame(

py-polars/tests/unit/datatypes/test_categories.py

@@ -11,6 +11,12 @@
 from polars.exceptions import ComputeError, SchemaError
 from polars.testing import assert_frame_equal, assert_series_equal
 
+from tests.unit.conftest import IS_WASM
+
+if IS_WASM:
+    pytest.skip("Sink operations not available in WASM", allow_module_level=True)
+
+
 CATS = [
     pl.Categories(),
     pl.Categories("foo"),