Ci test selection (#502)

* Remove -x from mpl pytest runs

* ci: add test impact selection

Author: cvanelteren

.github/workflows/build-ultraplot.yml

@@ -8,6 +8,14 @@ on:
       matplotlib-version:
         required: true
         type: string
+      test-mode:
+        required: false
+        type: string
+        default: full
+      test-nodeids:
+        required: false
+        type: string
+        default: ""
 
 env:
   LC_ALL: en_US.UTF-8
@@ -21,6 +29,9 @@ jobs:
     defaults:
       run:
         shell: bash -el {0}
+    env:
+      TEST_MODE: ${{ inputs.test-mode }}
+      TEST_NODEIDS: ${{ inputs.test-nodeids }}
     steps:
       - uses: actions/checkout@v6
         with:
@@ -43,7 +54,11 @@ jobs:
 
       - name: Test Ultraplot
         run: |
-          pytest -n auto --cov=ultraplot --cov-branch --cov-report term-missing --cov-report=xml ultraplot
+          if [ "${TEST_MODE}" = "selected" ] && [ -n "${TEST_NODEIDS}" ]; then
+            pytest -n auto --cov=ultraplot --cov-branch --cov-report term-missing --cov-report=xml ${TEST_NODEIDS}
+          else
+            pytest -n auto --cov=ultraplot --cov-branch --cov-report term-missing --cov-report=xml ultraplot
+          fi
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5
@@ -56,6 +71,8 @@ jobs:
     runs-on: ubuntu-latest
     env:
       IS_PR: ${{ github.event_name == 'pull_request' }}
+      TEST_MODE: ${{ inputs.test-mode }}
+      TEST_NODEIDS: ${{ inputs.test-nodeids }}
     defaults:
       run:
         shell: bash -el {0}
@@ -102,10 +119,17 @@ jobs:
 
           # Generate the baseline images and hash library
           python -c "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')"
-          pytest -W ignore \
-            --mpl-generate-path=./ultraplot/tests/baseline/ \
-            --mpl-default-style="./ultraplot.yml"\
-            ultraplot/tests
+          if [ "${TEST_MODE}" = "selected" ] && [ -n "${TEST_NODEIDS}" ]; then
+            pytest -W ignore \
+              --mpl-generate-path=./ultraplot/tests/baseline/ \
+              --mpl-default-style="./ultraplot.yml" \
+              ${TEST_NODEIDS}
+          else
+            pytest -W ignore \
+              --mpl-generate-path=./ultraplot/tests/baseline/ \
+              --mpl-default-style="./ultraplot.yml" \
+              ultraplot/tests
+          fi
 
           # Return to the PR branch for the rest of the job
           if [ -n "${{ github.event.pull_request.base.sha }}" ]; then
@@ -120,13 +144,23 @@ jobs:
 
           mkdir -p results
           python -c "import ultraplot as plt; plt.config.Configurator()._save_yaml('ultraplot.yml')"
-          pytest -W ignore \
-            --mpl \
-            --mpl-baseline-path=./ultraplot/tests/baseline \
-            --mpl-results-path=./results/ \
-            --mpl-generate-summary=html \
-            --mpl-default-style="./ultraplot.yml" \
-            ultraplot/tests
+          if [ "${TEST_MODE}" = "selected" ] && [ -n "${TEST_NODEIDS}" ]; then
+            pytest -W ignore \
+              --mpl \
+              --mpl-baseline-path=./ultraplot/tests/baseline \
+              --mpl-results-path=./results/ \
+              --mpl-generate-summary=html \
+              --mpl-default-style="./ultraplot.yml" \
+              ${TEST_NODEIDS}
+          else
+            pytest -W ignore \
+              --mpl \
+              --mpl-baseline-path=./ultraplot/tests/baseline \
+              --mpl-results-path=./results/ \
+              --mpl-generate-summary=html \
+              --mpl-default-style="./ultraplot.yml" \
+              ultraplot/tests
+          fi
 
       # Return the html output of the comparison even if failed
       - name: Upload comparison failures

.github/workflows/main.yml

@@ -19,6 +19,60 @@ jobs:
             python:
               - 'ultraplot/**'
 
+  select-tests:
+    runs-on: ubuntu-latest
+    needs:
+      - run-if-changes
+    if: always() && needs.run-if-changes.outputs.run == 'true'
+    outputs:
+      mode: ${{ steps.select.outputs.mode }}
+      tests: ${{ steps.select.outputs.tests }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Prepare workspace
+        run: mkdir -p .ci
+
+      - name: Restore test map cache
+        id: restore-map
+        uses: actions/cache/restore@v4
+        with:
+          path: .ci/test-map.json
+          key: test-map-${{ github.event.pull_request.base.sha }}
+          restore-keys: |
+            test-map-
+
+      - name: Select impacted tests
+        id: select
+        run: |
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
+            echo "mode=full" >> $GITHUB_OUTPUT
+            echo "tests=" >> $GITHUB_OUTPUT
+            exit 0
+          fi
+
+          git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} > .ci/changed.txt
+
+          python tools/ci/select_tests.py \
+            --map .ci/test-map.json \
+            --changed-files .ci/changed.txt \
+            --output .ci/selection.json \
+            --always-full 'pyproject.toml' \
+            --always-full 'environment.yml' \
+            --always-full 'ultraplot/__init__.py' \
+            --ignore 'docs/**' \
+            --ignore 'README.rst'
+
+          python - <<'PY' > .ci/selection.out
+          import json
+          data = json.load(open(".ci/selection.json", "r", encoding="utf-8"))
+          print(f"mode={data['mode']}")
+          print("tests=" + " ".join(data.get("tests", [])))
+          PY
+          cat .ci/selection.out >> $GITHUB_OUTPUT
+
   get-versions:
     runs-on: ubuntu-latest
     needs:
@@ -121,7 +175,8 @@ jobs:
     needs:
       - get-versions
       - run-if-changes
-    if: always() && needs.run-if-changes.outputs.run == 'true' && needs.get-versions.result == 'success'
+      - select-tests
+    if: always() && needs.run-if-changes.outputs.run == 'true' && needs.get-versions.result == 'success' && needs.select-tests.result == 'success'
     strategy:
       matrix:
         python-version: ${{ fromJson(needs.get-versions.outputs.python-versions) }}
@@ -134,6 +189,8 @@ jobs:
     with:
       python-version: ${{ matrix.python-version }}
       matplotlib-version: ${{ matrix.matplotlib-version }}
+      test-mode: ${{ needs.select-tests.outputs.mode }}
+      test-nodeids: ${{ needs.select-tests.outputs.tests }}
 
   build-success:
     needs:

.github/workflows/test-map.yml

@@ -0,0 +1,46 @@
+name: Build Test Map
+on:
+  push:
+    branches: [main]
+  schedule:
+    - cron: "0 3 * * *"
+  workflow_dispatch:
+
+jobs:
+  build-map:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - uses: mamba-org/setup-micromamba@v2.0.7
+        with:
+          environment-file: ./environment.yml
+          init-shell: bash
+          create-args: >-
+            --verbose
+            python=3.11
+            matplotlib=3.9
+          cache-environment: true
+          cache-downloads: false
+
+      - name: Build Ultraplot
+        run: |
+          pip install --no-build-isolation --no-deps .
+
+      - name: Generate test coverage map
+        run: |
+          mkdir -p .ci
+          pytest -n auto --cov=ultraplot --cov-branch --cov-context=test --cov-report= ultraplot
+          python tools/ci/build_test_map.py --coverage-file .coverage --output .ci/test-map.json --root .
+
+      - name: Cache test map
+        uses: actions/cache@v4
+        with:
+          path: .ci/test-map.json
+          key: test-map-${{ github.sha }}

tools/ci/build_test_map.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def build_map(coverage_file: str, repo_root: str) -> dict[str, list[str]]:
+    try:
+        from coverage import Coverage
+    except Exception as exc:  # pragma: no cover - diagnostic path
+        raise SystemExit(
+            f"coverage.py is required to build the test map: {exc}"
+        ) from exc
+
+    cov = Coverage(data_file=coverage_file)
+    cov.load()
+    data = cov.get_data()
+
+    files_map: dict[str, set[str]] = {}
+    for filename in data.measured_files():
+        if not filename:
+            continue
+        rel = os.path.relpath(filename, repo_root)
+        if rel.startswith(".."):
+            continue
+        try:
+            contexts_by_line = data.contexts_by_lineno(filename)
+        except Exception:
+            continue
+
+        contexts = set()
+        for ctxs in contexts_by_line.values():
+            if ctxs:
+                contexts.update(ctxs)
+        if contexts:
+            files_map[rel] = contexts
+
+    return {path: sorted(contexts) for path, contexts in files_map.items()}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Build a test impact map from coverage contexts."
+    )
+    parser.add_argument("--coverage-file", default=".coverage")
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--root", default=".")
+    args = parser.parse_args()
+
+    repo_root = os.path.abspath(args.root)
+    mapping = {
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "files": build_map(args.coverage_file, repo_root),
+    }
+
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with output_path.open("w", encoding="utf-8") as f:
+        json.dump(mapping, f, indent=2, sort_keys=True)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/ci/select_tests.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from fnmatch import fnmatch
+from pathlib import Path
+
+
+def load_map(path: str) -> dict[str, list[str]] | None:
+    map_path = Path(path)
+    if not map_path.is_file():
+        return None
+    with map_path.open("r", encoding="utf-8") as f:
+        data = json.load(f)
+    return data.get("files", {})
+
+
+def read_changed_files(path: str) -> list[str]:
+    changed_path = Path(path)
+    if not changed_path.is_file():
+        return []
+    return [
+        line.strip()
+        for line in changed_path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+
+
+def matches_any(path: str, patterns: list[str]) -> bool:
+    return any(fnmatch(path, pattern) for pattern in patterns)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Select impacted pytest nodeids from a test map."
+    )
+    parser.add_argument("--map", dest="map_path", required=True)
+    parser.add_argument("--changed-files", required=True)
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--always-full", action="append", default=[])
+    parser.add_argument("--ignore", action="append", default=[])
+    parser.add_argument("--source-prefix", default="ultraplot/")
+    parser.add_argument("--tests-prefix", default="ultraplot/tests/")
+    args = parser.parse_args()
+
+    files_map = load_map(args.map_path)
+    changed_files = read_changed_files(args.changed_files)
+
+    result = {"mode": "full", "tests": []}
+    if not files_map or not changed_files:
+        Path(args.output).write_text(json.dumps(result, indent=2), encoding="utf-8")
+        return 0
+
+    tests = set()
+    for path in changed_files:
+        path = path.replace("\\", "/")
+        if matches_any(path, args.ignore):
+            continue
+        if matches_any(path, args.always_full):
+            tests.clear()
+            result["mode"] = "full"
+            break
+        if path.startswith(args.tests_prefix):
+            tests.add(path)
+            continue
+        if path in files_map:
+            tests.update(files_map[path])
+            continue
+        if path.startswith(args.source_prefix):
+            tests.clear()
+            result["mode"] = "full"
+            break
+
+    if tests:
+        result["mode"] = "selected"
+        result["tests"] = sorted(tests)
+
+    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+    Path(args.output).write_text(json.dumps(result, indent=2), encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())