Merge branch 'main' into fix/cli-refactor

Author: assadyousuf

docs/changelog.md

@@ -12,6 +12,12 @@ Pydantic AI is still pre-version 1, so breaking changes will occur, however:
 !!! note
     Here's a filtered list of the breaking changes for each version to help you upgrade Pydantic AI.
 
+### v0.5.0 (2025-08-04)
+
+See [#2388](https://github.com/pydantic/pydantic-ai/pull/2388) - The `source` field of an `EvaluationResult` is now of type `EvaluatorSpec` rather than the actual source `Evaluator` instance, to help with serialization/deserialization.
+
+See [#2163](https://github.com/pydantic/pydantic-ai/pull/2163) - The `EvaluationReport.print` and `EvaluationReport.console_table` methods now require most arguments be passed by keyword.
+
 ### v0.4.0 (2025-07-08)
 
 See [#1799](https://github.com/pydantic/pydantic-ai/pull/1799) - Pydantic Evals `EvaluationReport` and `ReportCase` are now generic dataclasses instead of Pydantic models. If you were serializing them using `model_dump()`, you will now need to use the `EvaluationReportAdapter` and `ReportCaseAdapter` type adapters instead.

docs/tools.md

@@ -12,7 +12,7 @@ There are a number of ways to register tools with an agent:
 - via the [`@agent.tool_plain`][pydantic_ai.Agent.tool_plain] decorator — for tools that do not need access to the agent [context][pydantic_ai.tools.RunContext]
 - via the [`tools`][pydantic_ai.Agent.__init__] keyword argument to `Agent` which can take either plain functions, or instances of [`Tool`][pydantic_ai.tools.Tool]
 
-For more advanced use cases, the [toolsets](toolsets.md) feature lets you manage collections of tools (built by you or provided by an [MCP server](mcp/client.md) or other [third party](#third-party-tools)) and register them with an agent in one go via the [`toolsets`][pydantic_ai.Agent.__init__] keyword argument to `Agent`.
+For more advanced use cases, the [toolsets](toolsets.md) feature lets you manage collections of tools (built by you or provided by an [MCP server](mcp/client.md) or other [third party](#third-party-tools)) and register them with an agent in one go via the [`toolsets`][pydantic_ai.Agent.__init__] keyword argument to `Agent`. Internally, all `tools` and `toolsets` are gathered into a single [combined toolset](toolsets.md#combining-toolsets) that's made available to the model.
 
 !!! info "Function tools vs. RAG"
     Function tools are basically the "R" of RAG (Retrieval-Augmented Generation) — they augment what the model can do by letting it request extra information.
@@ -724,6 +724,12 @@ def my_flaky_tool(query: str) -> str:
 
 Raising `ModelRetry` also generates a `RetryPromptPart` containing the exception message, which is sent back to the LLM to guide its next attempt. Both `ValidationError` and `ModelRetry` respect the `retries` setting configured on the `Tool` or `Agent`.
 
+### Parallel tool calls & concurrency
+
+When a model returns multiple tool calls in one response, Pydantic AI schedules them concurrently using `asyncio.create_task`.
+
+Async functions are run on the event loop, while sync functions are offloaded to threads. To get the best performance, _always_ use an async function _unless_ you're doing blocking I/O (and there's no way to use a non-blocking library instead) or CPU-bound work (like `numpy` or `scikit-learn` operations), so that simple functions are not offloaded to threads unnecessarily.
+
 ## Third-Party Tools
 
 ### MCP Tools {#mcp-tools}

pydantic_ai_slim/pydantic_ai/_function_schema.py

@@ -285,7 +285,6 @@ def _build_schema(
     td_schema = core_schema.typed_dict_schema(
         fields,
         config=core_config,
-        total=var_kwargs_schema is None,
         extras_schema=gen_schema.generate_schema(var_kwargs_schema) if var_kwargs_schema else None,
     )
     return td_schema, None

pydantic_ai_slim/pydantic_ai/messages.py

@@ -106,7 +106,7 @@ class FileUrl(ABC):
     - `GoogleModel`: `VideoUrl.vendor_metadata` is used as `video_metadata`: https://ai.google.dev/gemini-api/docs/video-understanding#customize-video-processing
     """
 
-    _media_type: str | None = field(init=False, repr=False)
+    _media_type: str | None = field(init=False, repr=False, compare=False)
 
     def __init__(
         self,
@@ -120,19 +120,21 @@ def __init__(
         self.force_download = force_download
         self._media_type = media_type
 
-    @abstractmethod
-    def _infer_media_type(self) -> str:
-        """Return the media type of the file, based on the url."""
-
     @property
     def media_type(self) -> str:
-        """Return the media type of the file, based on the url or the provided `_media_type`."""
+        """Return the media type of the file, based on the URL or the provided `media_type`."""
         return self._media_type or self._infer_media_type()
 
+    @abstractmethod
+    def _infer_media_type(self) -> str:
+        """Infer the media type of the file based on the URL."""
+        raise NotImplementedError
+
     @property
     @abstractmethod
     def format(self) -> str:
         """The file format."""
+        raise NotImplementedError
 
     __repr__ = _utils.dataclasses_no_defaults_repr
 
@@ -182,7 +184,9 @@ def _infer_media_type(self) -> VideoMediaType:
         elif self.is_youtube:
             return 'video/mp4'
         else:
-            raise ValueError(f'Unknown video file extension: {self.url}')
+            raise ValueError(
+                f'Could not infer media type from video URL: {self.url}. Explicitly provide a `media_type` instead.'
+            )
 
     @property
     def is_youtube(self) -> bool:
@@ -238,7 +242,9 @@ def _infer_media_type(self) -> AudioMediaType:
         if self.url.endswith('.aac'):
             return 'audio/aac'
 
-        raise ValueError(f'Unknown audio file extension: {self.url}')
+        raise ValueError(
+            f'Could not infer media type from audio URL: {self.url}. Explicitly provide a `media_type` instead.'
+        )
 
     @property
     def format(self) -> AudioFormat:
@@ -278,7 +284,9 @@ def _infer_media_type(self) -> ImageMediaType:
         elif self.url.endswith('.webp'):
             return 'image/webp'
         else:
-            raise ValueError(f'Unknown image file extension: {self.url}')
+            raise ValueError(
+                f'Could not infer media type from image URL: {self.url}. Explicitly provide a `media_type` instead.'
+            )
 
     @property
     def format(self) -> ImageFormat:
@@ -324,10 +332,16 @@ def _infer_media_type(self) -> str:
             return 'application/pdf'
         elif self.url.endswith('.rtf'):
             return 'application/rtf'
+        elif self.url.endswith('.docx'):
+            return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+        elif self.url.endswith('.xlsx'):
+            return 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
 
         type_, _ = guess_type(self.url)
         if type_ is None:
-            raise ValueError(f'Unknown document file extension: {self.url}')
+            raise ValueError(
+                f'Could not infer media type from document URL: {self.url}. Explicitly provide a `media_type` instead.'
+            )
         return type_
 
     @property

pydantic_ai_slim/pydantic_ai/profiles/openai.py

@@ -166,11 +166,13 @@ def transform(self, schema: JsonSchema) -> JsonSchema:  # noqa C901
                 schema['required'] = list(schema['properties'].keys())
 
             elif self.strict is None:
-                if (
-                    schema.get('additionalProperties') is not False
-                    or 'properties' not in schema
-                    or 'required' not in schema
-                ):
+                if schema.get('additionalProperties', None) not in (None, False):
+                    self.is_strict_compatible = False
+                else:
+                    # additional properties are disallowed by default
+                    schema['additionalProperties'] = False
+
+                if 'properties' not in schema or 'required' not in schema:
                     self.is_strict_compatible = False
                 else:
                     required = schema['required']

pydantic_ai_slim/pydantic_ai/tools.py

@@ -133,11 +133,19 @@ async def turn_on_strict_if_openai(
 
 class GenerateToolJsonSchema(GenerateJsonSchema):
     def typed_dict_schema(self, schema: core_schema.TypedDictSchema) -> JsonSchemaValue:
-        s = super().typed_dict_schema(schema)
-        total = schema.get('total')
-        if 'additionalProperties' not in s and (total is True or total is None):
-            s['additionalProperties'] = False
-        return s
+        json_schema = super().typed_dict_schema(schema)
+        # Workaround for https://github.com/pydantic/pydantic/issues/12123
+        if 'additionalProperties' not in json_schema:  # pragma: no branch
+            extra = schema.get('extra_behavior') or schema.get('config', {}).get('extra_fields_behavior')
+            if extra == 'allow':
+                extras_schema = schema.get('extras_schema', None)
+                if extras_schema is not None:
+                    json_schema['additionalProperties'] = self.generate_inner(extras_schema) or True
+                else:
+                    json_schema['additionalProperties'] = True  # pragma: no cover
+            elif extra == 'forbid':
+                json_schema['additionalProperties'] = False
+        return json_schema
 
     def _named_required_fields_schema(self, named_required_fields: Sequence[tuple[str, bool, Any]]) -> JsonSchemaValue:
         # Remove largely-useless property titles

pydantic_evals/pydantic_evals/dataset.py

@@ -38,9 +38,9 @@
 from ._utils import get_unwrapped_function_name, task_group_gather
 from .evaluators import EvaluationResult, Evaluator
 from .evaluators._run_evaluator import run_evaluator
-from .evaluators._spec import EvaluatorSpec
 from .evaluators.common import DEFAULT_EVALUATORS
 from .evaluators.context import EvaluatorContext
+from .evaluators.spec import EvaluatorSpec
 from .otel import SpanTree
 from .otel._context_subtree import context_subtree
 from .reporting import EvaluationReport, ReportCase, ReportCaseAggregate

pydantic_evals/pydantic_evals/evaluators/__init__.py

@@ -10,7 +10,7 @@
     Python,
 )
 from .context import EvaluatorContext
-from .evaluator import EvaluationReason, EvaluationResult, Evaluator, EvaluatorOutput
+from .evaluator import EvaluationReason, EvaluationResult, Evaluator, EvaluatorOutput, EvaluatorSpec
 
 __all__ = (
     # common
@@ -27,7 +27,8 @@
     'EvaluatorContext',
     # evaluator
     'Evaluator',
-    'EvaluationReason',
     'EvaluatorOutput',
+    'EvaluatorSpec',
+    'EvaluationReason',
     'EvaluationResult',
 )

pydantic_evals/pydantic_evals/evaluators/_run_evaluator.py

@@ -48,7 +48,9 @@ async def run_evaluator(
     for name, result in results.items():
         if not isinstance(result, EvaluationReason):
             result = EvaluationReason(value=result)
-        details.append(EvaluationResult(name=name, value=result.value, reason=result.reason, source=evaluator))
+        details.append(
+            EvaluationResult(name=name, value=result.value, reason=result.reason, source=evaluator.as_spec())
+        )
 
     return details
 

pydantic_evals/pydantic_evals/evaluators/evaluator.py

@@ -17,15 +17,16 @@
 from pydantic_ai import _utils
 
 from .._utils import get_event_loop
-from ._spec import EvaluatorSpec
 from .context import EvaluatorContext
+from .spec import EvaluatorSpec
 
 __all__ = (
     'EvaluationReason',
     'EvaluationResult',
     'EvaluationScalar',
     'Evaluator',
     'EvaluatorOutput',
+    'EvaluatorSpec',
 )
 
 EvaluationScalar = Union[bool, int, float, str]
@@ -71,13 +72,13 @@ class EvaluationResult(Generic[EvaluationScalarT]):
         name: The name of the evaluation.
         value: The scalar result of the evaluation.
         reason: An optional explanation of the evaluation result.
-        source: The evaluator that produced this result.
+        source: The spec of the evaluator that produced this result.
     """
 
     name: str
     value: EvaluationScalarT
     reason: str | None
-    source: Evaluator
+    source: EvaluatorSpec
 
     def downcast(self, *value_types: type[T]) -> EvaluationResult[T] | None:
         """Attempt to downcast this result to a more specific type.
@@ -246,6 +247,13 @@ def serialize(self, info: SerializationInfo) -> Any:
         Returns:
             A JSON-serializable representation of this evaluator as an EvaluatorSpec.
         """
+        return to_jsonable_python(
+            self.as_spec(),
+            context=info.context,
+            serialize_unknown=True,
+        )
+
+    def as_spec(self) -> EvaluatorSpec:
         raw_arguments = self.build_serialization_arguments()
 
         arguments: None | tuple[Any,] | dict[str, Any]
@@ -255,11 +263,8 @@ def serialize(self, info: SerializationInfo) -> Any:
             arguments = (next(iter(raw_arguments.values())),)
         else:
             arguments = raw_arguments
-        return to_jsonable_python(
-            EvaluatorSpec(name=self.get_serialization_name(), arguments=arguments),
-            context=info.context,
-            serialize_unknown=True,
-        )
+
+        return EvaluatorSpec(name=self.get_serialization_name(), arguments=arguments)
 
     def build_serialization_arguments(self) -> dict[str, Any]:
         """Build the arguments for serialization.