tests: speed up the test suite (#2414)

Kludex · web-flow · commit 544ff88690a0 · 2025-08-05T13:49:56.000+02:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -140,6 +140,7 @@ jobs:
     env:
       UV_PYTHON: ${{ matrix.python-version }}
       CI: true
+      COVERAGE_PROCESS_START: ./pyproject.toml
     steps:
       - uses: actions/checkout@v4
 
@@ -151,20 +152,20 @@ jobs:
         with:
           deno-version: v2.x
 
-      - run: mkdir coverage
+      - run: mkdir .coverage
 
       # run tests with just `pydantic-ai-slim` dependencies
-      - run: uv run --package pydantic-ai-slim coverage run -m pytest
+      - run: uv run --package pydantic-ai-slim coverage run -m pytest -n auto --dist=loadgroup
         env:
-          COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-slim
+          COVERAGE_FILE: .coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-slim
 
-      - run: uv run coverage run -m pytest
+      - run: uv run coverage run -m pytest -n auto --dist=loadgroup
         env:
-          COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-standard
+          COVERAGE_FILE: .coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-standard
 
-      - run: uv run --all-extras coverage run -m pytest
+      - run: uv run --all-extras coverage run -m pytest -n auto --dist=loadgroup
         env:
-          COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-all-extras
+          COVERAGE_FILE: .coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-all-extras
 
       - run: uv run --all-extras python tests/import_examples.py
 
@@ -173,15 +174,15 @@ jobs:
         if: matrix.python-version != '3.9'
         run: |
           unset UV_FROZEN
-          uv run --all-extras --resolution lowest-direct coverage run -m pytest
+          uv run --all-extras --resolution lowest-direct coverage run -m pytest -n auto --dist=loadgroup
         env:
-          COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-lowest-versions
+          COVERAGE_FILE: .coverage/.coverage.${{ runner.os }}-py${{ matrix.python-version }}-lowest-versions
 
       - name: store coverage files
         uses: actions/upload-artifact@v4
         with:
           name: coverage-${{ matrix.python-version }}
-          path: coverage
+          path: .coverage
           include-hidden-files: true
 
   coverage:
@@ -197,15 +198,15 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           merge-multiple: true
-          path: coverage
+          path: .coverage
 
       - uses: astral-sh/setup-uv@v5
         with:
           enable-cache: true
 
       - run: uv sync --package pydantic-ai-slim --only-dev
-      - run: rm coverage/.coverage.*-py3.9-* # Exclude 3.9 coverage as it gets the wrong line numbers, causing invalid failures.
-      - run: uv run coverage combine coverage
+      - run: rm .coverage/.coverage.*-py3.9-* # Exclude 3.9 coverage as it gets the wrong line numbers, causing invalid failures.
+      - run: uv run coverage combine
 
       - run: uv run coverage html --show-contexts --title "Pydantic AI coverage for ${{ github.sha }}"
 
@@ -228,7 +229,10 @@ jobs:
 
       - run: uv run coverage report --fail-under 100
       - run: uv run diff-cover coverage.xml --fail-under 100
+
       - run: uv run strict-no-cover
+        env:
+          COVERAGE_FILE: .coverage/.coverage
 
   test-mcp-run-python:
     runs-on: ubuntu-latest
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -10,7 +10,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 - **Format code**: `make format`
 - **Lint code**: `make lint`
 - **Type checking**: `make typecheck` (uses pyright) or `make typecheck-both` (pyright + mypy)
-- **Run tests**: `make test` (with coverage) or `make test-fast` (parallel, no coverage)
+- **Run tests**: `make test` (with coverage)
 - **Build docs**: `make docs` or `make docs-serve` (local development)
 
 ### Single Test Commands
diff --git a/Makefile b/Makefile
@@ -60,13 +60,10 @@ typecheck-both: typecheck-pyright typecheck-mypy
 
 .PHONY: test
 test: ## Run tests and collect coverage data
-	uv run coverage run -m pytest
+	COVERAGE_PROCESS_START=./pyproject.toml uv run coverage run -m pytest -n auto --dist=loadgroup
+	@uv run coverage combine
 	@uv run coverage report
 
-.PHONY: test-fast
-test-fast: ## Same as test except no coverage and 4x faster depending on hardware
-	uv run pytest -n auto --dist=loadgroup
-
 .PHONY: test-all-python
 test-all-python: ## Run tests on Python 3.9 to 3.13
 	UV_PROJECT_ENVIRONMENT=.venv39 uv run --python 3.9 --all-extras --all-packages coverage run -p -m pytest
diff --git a/pydantic_ai_slim/pydantic_ai/_agent_graph.py b/pydantic_ai_slim/pydantic_ai/_agent_graph.py
@@ -620,7 +620,7 @@ async def process_function_tools(  # noqa: C901
                 result_data = await tool_manager.handle_call(call)
             except exceptions.UnexpectedModelBehavior as e:
                 ctx.state.increment_retries(ctx.deps.max_result_retries, e)
-                raise e  # pragma: no cover
+                raise e  # pragma: lax no cover
             except ToolRetryError as e:
                 ctx.state.increment_retries(ctx.deps.max_result_retries, e)
                 yield _messages.FunctionToolCallEvent(call)
diff --git a/pydantic_ai_slim/pydantic_ai/exceptions.py b/pydantic_ai_slim/pydantic_ai/exceptions.py
@@ -5,9 +5,9 @@
 from typing import TYPE_CHECKING
 
 if sys.version_info < (3, 11):
-    from exceptiongroup import ExceptionGroup
+    from exceptiongroup import ExceptionGroup as ExceptionGroup  # pragma: lax no cover
 else:
-    ExceptionGroup = ExceptionGroup
+    ExceptionGroup = ExceptionGroup  # pragma: lax no cover
 
 if TYPE_CHECKING:
     from .messages import RetryPromptPart
diff --git a/pydantic_ai_slim/pydantic_ai/models/anthropic.py b/pydantic_ai_slim/pydantic_ai/models/anthropic.py
@@ -256,7 +256,7 @@ async def _messages_create(
         except APIStatusError as e:
             if (status_code := e.status_code) >= 400:
                 raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e
-            raise  # pragma: no cover
+            raise  # pragma: lax no cover
 
     def _process_response(self, response: BetaMessage) -> ModelResponse:
         """Process a non-streamed response, and prepare a message to return."""
diff --git a/pydantic_ai_slim/pydantic_ai/models/bedrock.py b/pydantic_ai_slim/pydantic_ai/models/bedrock.py
@@ -665,4 +665,4 @@ async def __anext__(self) -> T:
             if type(e.__cause__) is StopIteration:
                 raise StopAsyncIteration
             else:
-                raise e  # pragma: no cover
+                raise e  # pragma: lax no cover
diff --git a/pydantic_ai_slim/pydantic_ai/models/cohere.py b/pydantic_ai_slim/pydantic_ai/models/cohere.py
@@ -183,7 +183,7 @@ async def _chat(
         except ApiError as e:
             if (status_code := e.status_code) and status_code >= 400:
                 raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e
-            raise  # pragma: no cover
+            raise  # pragma: lax no cover
 
     def _process_response(self, response: V2ChatResponse) -> ModelResponse:
         """Process a non-streamed response, and prepare a message to return."""
diff --git a/pydantic_ai_slim/pydantic_ai/models/gemini.py b/pydantic_ai_slim/pydantic_ai/models/gemini.py
@@ -236,7 +236,7 @@ async def _make_request(
 
         if gemini_labels := model_settings.get('gemini_labels'):
             if self._system == 'google-vertex':
-                request_data['labels'] = gemini_labels
+                request_data['labels'] = gemini_labels  # pragma: lax no cover
 
         headers = {'Content-Type': 'application/json', 'User-Agent': get_user_agent()}
         url = f'/{self._model_name}:{"streamGenerateContent" if streamed else "generateContent"}'
@@ -366,11 +366,11 @@ async def _map_user_prompt(self, part: UserPromptPart) -> list[_GeminiPartUnion]
                             inline_data={'data': downloaded_item['data'], 'mime_type': downloaded_item['data_type']}
                         )
                         content.append(inline_data)
-                    else:
+                    else:  # pragma: lax no cover
                         file_data = _GeminiFileDataPart(file_data={'file_uri': item.url, 'mime_type': item.media_type})
                         content.append(file_data)
                 else:
-                    assert_never(item)
+                    assert_never(item)  # pragma: lax no cover
         return content
 
     def _map_response_schema(self, o: OutputObjectDefinition) -> dict[str, Any]:
diff --git a/pydantic_ai_slim/pydantic_ai/models/google.py b/pydantic_ai_slim/pydantic_ai/models/google.py
@@ -407,7 +407,7 @@ async def _map_user_prompt(self, part: UserPromptPart) -> list[PartDict]:
                     content.append(inline_data_dict)  # type: ignore
                 elif isinstance(item, VideoUrl) and item.is_youtube:
                     file_data_dict = {'file_data': {'file_uri': item.url, 'mime_type': item.media_type}}
-                    if item.vendor_metadata:
+                    if item.vendor_metadata:  # pragma: no branch
                         file_data_dict['video_metadata'] = item.vendor_metadata
                     content.append(file_data_dict)  # type: ignore
                 elif isinstance(item, FileUrl):
@@ -421,7 +421,9 @@ async def _map_user_prompt(self, part: UserPromptPart) -> list[PartDict]:
                         inline_data = {'data': downloaded_item['data'], 'mime_type': downloaded_item['data_type']}
                         content.append({'inline_data': inline_data})  # type: ignore
                     else:
-                        content.append({'file_data': {'file_uri': item.url, 'mime_type': item.media_type}})
+                        content.append(
+                            {'file_data': {'file_uri': item.url, 'mime_type': item.media_type}}
+                        )  # pragma: lax no cover
                 else:
                     assert_never(item)
         return content
diff --git a/pydantic_ai_slim/pydantic_ai/models/groq.py b/pydantic_ai_slim/pydantic_ai/models/groq.py
@@ -249,7 +249,7 @@ async def _completions_create(
         except APIStatusError as e:
             if (status_code := e.status_code) >= 400:
                 raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e
-            raise  # pragma: no cover
+            raise  # pragma: lax no cover
 
     def _process_response(self, response: chat.ChatCompletion) -> ModelResponse:
         """Process a non-streamed response, and prepare a message to return."""
diff --git a/pydantic_ai_slim/pydantic_ai/models/instrumented.py b/pydantic_ai_slim/pydantic_ai/models/instrumented.py
@@ -18,11 +18,7 @@
 from opentelemetry.util.types import AttributeValue
 from pydantic import TypeAdapter
 
-from ..messages import (
-    ModelMessage,
-    ModelRequest,
-    ModelResponse,
-)
+from ..messages import ModelMessage, ModelRequest, ModelResponse
 from ..settings import ModelSettings
 from . import KnownModelName, Model, ModelRequestParameters, StreamedResponse
 from .wrapper import WrapperModel
@@ -138,7 +134,7 @@ def __init__(
                 **tokens_histogram_kwargs,
                 explicit_bucket_boundaries_advisory=TOKEN_HISTOGRAM_BOUNDARIES,
             )
-        except TypeError:
+        except TypeError:  # pragma: lax no cover
             # Older OTel/logfire versions don't support explicit_bucket_boundaries_advisory
             self.tokens_histogram = self.meter.create_histogram(
                 **tokens_histogram_kwargs,  # pyright: ignore
diff --git a/pydantic_ai_slim/pydantic_ai/models/mistral.py b/pydantic_ai_slim/pydantic_ai/models/mistral.py
@@ -218,7 +218,7 @@ async def _completions_create(
         except SDKError as e:
             if (status_code := e.status_code) >= 400:
                 raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e
-            raise  # pragma: no cover
+            raise  # pragma: lax no cover
 
         assert response, 'A unexpected empty response from Mistral.'
         return response
diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py
@@ -359,7 +359,7 @@ async def _completions_create(
         except APIStatusError as e:
             if (status_code := e.status_code) >= 400:
                 raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e
-            raise  # pragma: no cover
+            raise  # pragma: lax no cover
 
     def _process_response(self, response: chat.ChatCompletion | str) -> ModelResponse:
         """Process a non-streamed response, and prepare a message to return."""
@@ -814,7 +814,7 @@ async def _responses_create(
         except APIStatusError as e:
             if (status_code := e.status_code) >= 400:
                 raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e
-            raise  # pragma: no cover
+            raise  # pragma: lax no cover
 
     def _get_reasoning(self, model_settings: OpenAIResponsesModelSettings) -> Reasoning | NotGiven:
         reasoning_effort = model_settings.get('openai_reasoning_effort', None)
diff --git a/pydantic_ai_slim/pyproject.toml b/pydantic_ai_slim/pyproject.toml
@@ -92,7 +92,7 @@ dev = [
     "anyio>=4.5.0",
     "asgi-lifespan>=2.1.0",
     "devtools>=0.12.2",
-    "coverage[toml]>=7.6.2",
+    "coverage[toml]>=7.10.2",
     "dirty-equals>=0.9.0",
     "duckduckgo-search>=7.0.0",
     "inline-snapshot>=0.19.3",
@@ -103,8 +103,9 @@ dev = [
     "pytest-recording>=0.13.2",
     "diff-cover>=9.2.0",
     "boto3-stubs[bedrock-runtime]",
-    "strict-no-cover>=0.1.1",
+    "strict-no-cover @ git+https://github.com/pydantic/strict-no-cover.git@7fc59da2c4dff919db2095a0f0e47101b657131d",
     "pytest-xdist>=3.6.1",
+    "coverage-enable-subprocess>=0.1.0",
 ]
 
 [tool.hatch.metadata]
diff --git a/pydantic_evals/pydantic_evals/dataset.py b/pydantic_evals/pydantic_evals/dataset.py
@@ -1038,7 +1038,7 @@ def _get_span_duration(span: logfire_api.LogfireSpan, fallback: float) -> float:
     """
     try:
         return (span.end_time - span.start_time) / 1_000_000_000  # type: ignore
-    except (AttributeError, TypeError):  # pragma: no cover
+    except (AttributeError, TypeError):  # pragma: lax no cover
         return fallback
 
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -215,6 +215,10 @@ filterwarnings = [
 
 # https://coverage.readthedocs.io/en/latest/config.html#run
 [tool.coverage.run]
+parallel = true
+concurrency = ["multiprocessing", "thread"]
+# We use a subdirectory for coverage data to avoid noisy coverage data files.
+data_file = ".coverage/.coverage"
 # required to avoid warnings about files created by create_module fixture
 include = [
     "pydantic_ai_slim/**/*.py",
@@ -245,6 +249,7 @@ source = [
 
 # https://coverage.readthedocs.io/en/latest/config.html#report
 [tool.coverage.report]
+fail_under = 100
 skip_covered = true
 show_missing = true
 ignore_errors = true
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -251,7 +251,7 @@ def method_matcher(r1: vcr_request.Request, r2: vcr_request.Request) -> None:
 def mock_vcr_aiohttp_content(mocker: MockerFixture):
     try:
         from vcr.stubs import aiohttp_stubs
-    except ImportError:
+    except ImportError:  # pragma: lax no cover
         return
 
     # google-genai calls `self.response_stream.content.readline()` where `self.response_stream` is a `MockClientResponse`,
@@ -416,9 +416,9 @@ def expired(self) -> bool:
 
 
 @pytest.fixture()
-async def vertex_provider():
+async def vertex_provider():  # pragma: lax no cover
     # NOTE: You need to comment out this line to rewrite the cassettes locally.
-    if not os.getenv('CI', False):  # pragma: lax no cover
+    if not os.getenv('CI', False):
         pytest.skip('Requires properly configured local google vertex config to pass')
 
     try:
diff --git a/tests/evals/test_dataset.py b/tests/evals/test_dataset.py
diff --git a/tests/models/test_google.py b/tests/models/test_google.py
diff --git a/uv.lock b/uv.lock
