ci: split test suite (#2436)

Co-authored-by: Douwe Maan <[email protected]>

Author: Kludex

.github/workflows/ci.yml

@@ -107,6 +107,7 @@ jobs:
         with:
           model: qwen2:0.5b
 
+      - run: uv sync --only-dev
       - run: >
           uv run
           --package pydantic-ai-slim
@@ -130,13 +131,14 @@ jobs:
           CO_API_KEY: ${{ secrets.COHERE_API_KEY }}
 
   test:
-    name: test on ${{ matrix.python-version }}
+    name: test on ${{ matrix.python-version }} (${{ matrix.install }})
     runs-on: ubuntu-latest
     timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        install: ["slim", "standard", "all-extras"]
     env:
       UV_PYTHON: ${{ matrix.python-version }}
       CI: true
@@ -148,46 +150,98 @@ jobs:
         with:
           enable-cache: true
 
-      - uses: denoland/setup-deno@v2
-        with:
-          deno-version: v2.x
+      - run: |
+          if [ "${{ matrix.install }}" = "slim" ]; then
+            echo "--package pydantic-ai-slim"
+          elif [ "${{ matrix.install }}" = "standard" ]; then
+            echo ""
+          elif [ "${{ matrix.install }}" = "all-extras" ]; then
+            echo "--all-extras"
+          fi
+        id: install-command
 
       - run: mkdir .coverage
 
-      # run tests with just `pydantic-ai-slim` dependencies
-      - run: uv run --package pydantic-ai-slim coverage run -m pytest -n auto --dist=loadgroup
+      - run: uv sync --group dev
+      - run: uv run ${{ steps.install-command.outputs.install-command }} coverage run -m pytest -n auto --dist=loadgroup
         env:
-          COVERAGE_FILE: .coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-slim
+          COVERAGE_FILE: .coverage/.coverage.${{ matrix.python-version }}-${{ matrix.install }}
 
-      - run: uv run coverage run -m pytest -n auto --dist=loadgroup
-        env:
-          COVERAGE_FILE: .coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-standard
+      - name: store coverage files
+        uses: actions/upload-artifact@v4
+        if: matrix.python-version != '3.9'
+        with:
+          name: coverage-${{ matrix.python-version }}-${{ matrix.install }}
+          path: .coverage
+          include-hidden-files: true
 
-      - run: uv run --all-extras coverage run -m pytest -n auto --dist=loadgroup
-        env:
-          COVERAGE_FILE: .coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-all-extras
+  test-lowest-versions:
+    name: test on ${{ matrix.python-version }} (lowest-versions)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    env:
+      UV_PYTHON: ${{ matrix.python-version }}
+      CI: true
+      COVERAGE_PROCESS_START: ./pyproject.toml
+    steps:
+      - uses: actions/checkout@v4
 
-      - run: uv run --all-extras python tests/import_examples.py
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
 
-      # this must run last as it modifies the environment!
-      - name: test lowest versions
-        if: matrix.python-version != '3.9'
-        run: |
-          unset UV_FROZEN
-          uv run --all-extras --resolution lowest-direct coverage run -m pytest -n auto --dist=loadgroup
+      - uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+
+      - run: mkdir .coverage
+
+      - run: uv sync --group dev
+
+      - run: unset UV_FROZEN
+
+      - run: uv run --all-extras --resolution lowest-direct coverage run -m pytest -n auto --dist=loadgroup
         env:
-          COVERAGE_FILE: .coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-lowest-versions
+          COVERAGE_FILE: .coverage/.coverage.${{matrix.python-version}}-lowest-versions
 
       - name: store coverage files
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-${{ matrix.python-version }}
+          name: coverage-${{ matrix.python-version }}-lowest-versions
           path: .coverage
           include-hidden-files: true
 
+  test-examples:
+    name: test examples on ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    env:
+      UV_PYTHON: ${{ matrix.python-version }}
+      CI: true
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+
+      - run: uv run --all-extras python tests/import_examples.py
+
   coverage:
     runs-on: ubuntu-latest
-    needs: [test]
+    needs: [test, test-lowest-versions]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -204,8 +258,7 @@ jobs:
         with:
           enable-cache: true
 
-      - run: uv sync --package pydantic-ai-slim --only-dev
-      - run: rm .coverage/.coverage.*-py3.9-* # Exclude 3.9 coverage as it gets the wrong line numbers, causing invalid failures.
+      - run: uv sync --group dev
       - run: uv run coverage combine
       - run: uv run coverage report
 
@@ -239,7 +292,7 @@ jobs:
   # https://github.com/marketplace/actions/alls-green#why used for branch protection checks
   check:
     if: always()
-    needs: [lint, mypy, docs, test-live, test, coverage, test-mcp-run-python]
+    needs: [lint, mypy, docs, test-live, test, test-lowest-versions, test-examples, coverage, test-mcp-run-python]
     runs-on: ubuntu-latest
 
     steps:

Makefile

@@ -60,7 +60,7 @@ typecheck-both: typecheck-pyright typecheck-mypy
 
 .PHONY: test
 test: ## Run tests and collect coverage data
-	COVERAGE_PROCESS_START=./pyproject.toml uv run coverage run -m pytest -n auto --dist=loadgroup
+	uv run coverage run -m pytest -n auto --dist=loadgroup
 	@uv run coverage combine
 	@uv run coverage report
 

examples/pyproject.toml

@@ -52,7 +52,7 @@ dependencies = [
     "pydantic-evals=={{ version }}",
     "asyncpg>=0.30.0",
     "fastapi>=0.115.4",
-    "logfire[asyncpg,fastapi,sqlite3,httpx]>=2.6",
+    "logfire[asyncpg,fastapi,sqlite3,httpx]>=3.14.1",
     "python-multipart>=0.0.17",
     "rich>=13.9.2",
     "uvicorn>=0.32.0",

pydantic_ai_slim/pyproject.toml

@@ -60,7 +60,7 @@ dependencies = [
 
 [tool.hatch.metadata.hooks.uv-dynamic-versioning.optional-dependencies]
 # WARNING if you add optional groups, please update docs/install.md
-logfire = ["logfire>=3.11.0"]
+logfire = ["logfire>=3.14.1"]
 # Models
 openai = ["openai>=1.92.0"]
 cohere = ["cohere>=5.16.0; platform_system != 'Emscripten'"]
@@ -87,27 +87,6 @@ ag-ui = ["ag-ui-protocol>=0.1.8", "starlette>=0.45.3"]
 # Retries
 retries = ["tenacity>=8.2.3"]
 
-[dependency-groups]
-dev = [
-    "anyio>=4.5.0",
-    "asgi-lifespan>=2.1.0",
-    "devtools>=0.12.2",
-    "coverage[toml]>=7.10.2",
-    "dirty-equals>=0.9.0",
-    "duckduckgo-search>=7.0.0",
-    "inline-snapshot>=0.19.3",
-    "pytest>=8.3.3",
-    "pytest-examples>=0.0.14",
-    "pytest-mock>=3.14.0",
-    "pytest-pretty>=1.3.0",
-    "pytest-recording>=0.13.2",
-    "diff-cover>=9.2.0",
-    "boto3-stubs[bedrock-runtime]",
-    "strict-no-cover @ git+https://github.com/pydantic/strict-no-cover.git@7fc59da2c4dff919db2095a0f0e47101b657131d",
-    "pytest-xdist>=3.6.1",
-    "coverage-enable-subprocess>=0.1.0",
-]
-
 [tool.hatch.metadata]
 allow-direct-references = true
 

pydantic_evals/pyproject.toml

@@ -49,7 +49,7 @@ requires-python = ">=3.9"
 [tool.hatch.metadata.hooks.uv-dynamic-versioning]
 dependencies = [
     "rich>=13.9.4",
-    "logfire-api>=1.2.0",
+    "logfire-api>=3.14.1",
     "pydantic>=2.10",
     "pydantic-ai-slim=={{ version }}",
     "anyio>=0",
@@ -58,7 +58,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-logfire = ["logfire>=2.3"]
+logfire = ["logfire>=3.14.1"]
 
 [project.urls]
 Homepage = "https://ai.pydantic.dev/evals"

pydantic_graph/pyproject.toml

@@ -47,7 +47,7 @@ classifiers = [
 requires-python = ">=3.9"
 dependencies = [
     "httpx>=0.27",
-    "logfire-api>=1.2.0",
+    "logfire-api>=3.14.1",
     "pydantic>=2.10",
     "typing-inspection>=0.4.0",
 ]

pyproject.toml

@@ -52,7 +52,7 @@ dependencies = [
 
 [tool.hatch.metadata.hooks.uv-dynamic-versioning.optional-dependencies]
 examples = ["pydantic-ai-examples=={{ version }}"]
-logfire = ["logfire>=3.11.0"]
+logfire = ["logfire>=3.14.1"]
 a2a = ["fasta2a>=0.4.1"]
 
 [project.urls]
@@ -81,8 +81,28 @@ members = [
     "examples",
 ]
 
+[tool.uv]
+default-groups = ["dev", "lint", "docs"]
+
 [dependency-groups]
-# dev dependencies are defined in `pydantic-ai-slim/pyproject.toml` to allow for minimal testing
+dev = [
+    "anyio>=4.5.0",
+    "asgi-lifespan>=2.1.0",
+    "devtools>=0.12.2",
+    "coverage[toml]>=7.10.3",
+    "dirty-equals>=0.9.0",
+    "duckduckgo-search>=7.0.0",
+    "inline-snapshot>=0.19.3",
+    "pytest>=8.3.3",
+    "pytest-examples>=0.0.18",
+    "pytest-mock>=3.14.0",
+    "pytest-pretty>=1.3.0",
+    "pytest-recording>=0.13.2",
+    "diff-cover>=9.2.0",
+    "boto3-stubs[bedrock-runtime]",
+    "strict-no-cover @ git+https://github.com/pydantic/strict-no-cover.git@7fc59da2c4dff919db2095a0f0e47101b657131d",
+    "pytest-xdist>=3.6.1",
+]
 lint = ["mypy>=1.11.2", "pyright>=1.1.390", "ruff>=0.6.9"]
 docs = [
     "pydantic-ai[a2a]",
@@ -216,7 +236,7 @@ filterwarnings = [
 
 # https://coverage.readthedocs.io/en/latest/config.html#run
 [tool.coverage.run]
-parallel = true
+patch = ["subprocess"]
 concurrency = ["multiprocessing", "thread"]
 # We use a subdirectory for coverage data to avoid noisy coverage data files.
 data_file = ".coverage/.coverage"

tests/test_logfire.py

@@ -558,10 +558,7 @@ def get_model():
 @pytest.mark.skipif(not logfire_installed, reason='logfire not installed')
 @pytest.mark.anyio
 async def test_feedback(capfire: CaptureLogfire) -> None:
-    try:
-        from logfire.experimental.annotations import record_feedback
-    except ImportError:  # pragma: lax no cover
-        pytest.skip('Requires recent version of logfire')
+    from logfire.experimental.annotations import record_feedback
 
     my_agent = Agent(model=TestModel(), instrument=True)