ci: Run PySpark selectively (#2366)

* ci: Run PySpark selectively

* pragma no cover for --all-cpu-constructors flag

* run py3.10 instead of 3.12 for pyspark

Author: FBruzzesi

.github/workflows/pytest-pyspark.yml

@@ -0,0 +1,37 @@
+name: PyTest PySpark constructor
+
+on:
+  pull_request:
+    types: [opened, labeled, synchronize]  # Triggers on PR creation, updates, and label changes
+
+env:
+  PY_COLORS: 1
+
+jobs:
+
+  pytest-pyspark-constructor:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'pyspark') || contains(github.event.pull_request.labels.*.name, 'release') }}
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11"]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: "true"
+          cache-suffix: ${{ matrix.python-version }}
+          cache-dependency-glob: "pyproject.toml"
+      - name: install-reqs
+        run: uv pip install -e . --group core-tests --group extra --system
+      - name: install pyspark
+        run: echo "setuptools<78" | uv pip install -b - -e ".[pyspark]" --system
+      - name: show-deps
+        run: uv pip freeze
+      - name: Run pytest
+        run: pytest tests --cov=narwhals/_spark_like --cov-fail-under=95 --runslow --constructors pyspark

.github/workflows/pytest.yml

@@ -86,10 +86,6 @@ jobs:
         run: uv pip install -e ".[modin, dask]" --group core-tests --group extra --system
       - name: install duckdb nightly
         run: uv pip install -U --pre duckdb --system
-      - name: install pyspark
-        run: echo "setuptools<78" | uv pip install -b - -e ".[pyspark]" --system
-        # PySpark is not yet available on Python3.12+
-        if: matrix.python-version != '3.13'
       - name: install ibis
         run: uv pip install -e ".[ibis]" --system
         # Ibis puts upper bounds on dependencies, and requires Python3.10+,
@@ -98,7 +94,7 @@ jobs:
       - name: show-deps
         run: uv pip freeze
       - name: Run pytest
-        run: pytest tests --cov=narwhals --cov=tests --cov-fail-under=100 --runslow --all-cpu-constructors
+        run: pytest tests --cov=narwhals --cov=tests --cov-fail-under=100 --runslow --constructors=pandas,pandas[nullable],pandas[pyarrow],pyarrow,modin[pyarrow],polars[eager],polars[lazy],dask,duckdb,sqlframe
       - name: Run doctests
         # reprs differ between versions, so we only run doctests on the latest Python
         if: matrix.python-version == '3.13'

tests/conftest.py

@@ -229,7 +229,7 @@ def sqlframe_pyspark_lazy_constructor(obj: Data) -> SQLFrameDataFrame:  # pragma
 
 
 def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
-    if metafunc.config.getoption("all_cpu_constructors"):
+    if metafunc.config.getoption("all_cpu_constructors"):  # pragma: no cover
         selected_constructors: list[str] = [
             *iter(EAGER_CONSTRUCTORS.keys()),
             *iter(LAZY_CONSTRUCTORS.keys()),
@@ -250,13 +250,9 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
 
     for constructor in selected_constructors:
         if (
-            (
-                constructor in {"pandas[nullable]", "pandas[pyarrow]"}
-                and MIN_PANDAS_NULLABLE_VERSION > PANDAS_VERSION
-            )
-            or (constructor == "sqlframe" and sys.version_info < (3, 9))
-            or (constructor == "pyspark" and sys.version_info >= (3, 12))
-        ):
+            constructor in {"pandas[nullable]", "pandas[pyarrow]"}
+            and MIN_PANDAS_NULLABLE_VERSION > PANDAS_VERSION
+        ) or (constructor == "sqlframe" and sys.version_info < (3, 9)):
             continue  # pragma: no cover
 
         if constructor in EAGER_CONSTRUCTORS: