Add test

Author: DouweM

docs/api/models/function.md

@@ -36,7 +36,11 @@ async def model_function(
     print(info)
     """
     AgentInfo(
-        function_tools=[], allow_text_output=True, output_tools=[], model_settings=None
+        function_tools=[],
+        allow_text_output=True,
+        output_tools=[],
+        model_settings=None,
+        instructions=None,
     )
     """
     return ModelResponse(parts=[TextPart('hello world')])

pydantic_ai_slim/pydantic_ai/models/__init__.py

@@ -317,6 +317,12 @@ class ModelRequestParameters:
     def tool_defs(self) -> dict[str, ToolDefinition]:
         return {tool_def.name: tool_def for tool_def in [*self.function_tools, *self.output_tools]}
 
+    @cached_property
+    def prompted_output_instructions(self) -> str | None:
+        if self.output_mode == 'prompted' and self.prompted_output_template and self.output_object:
+            return PromptedOutputSchema.build_instructions(self.prompted_output_template, self.output_object)
+        return None
+
     __repr__ = _utils.dataclasses_no_defaults_repr
 
 
@@ -417,56 +423,44 @@ def prepare_request(
         """
         model_settings = merge_model_settings(self.settings, model_settings)
 
-        model_request_parameters = self.customize_request_parameters(model_request_parameters)
+        params = self.customize_request_parameters(model_request_parameters)
 
-        if builtin_tools := model_request_parameters.builtin_tools:
+        if builtin_tools := params.builtin_tools:
             # Deduplicate builtin tools
-            model_request_parameters = replace(
-                model_request_parameters,
+            params = replace(
+                params,
                 builtin_tools=list({tool.unique_id: tool for tool in builtin_tools}.values()),
             )
 
-        if model_request_parameters.output_mode == 'auto':
+        if params.output_mode == 'auto':
             output_mode = self.profile.default_structured_output_mode
-            model_request_parameters = replace(
-                model_request_parameters,
+            params = replace(
+                params,
                 output_mode=output_mode,
                 allow_text_output=output_mode in ('native', 'prompted'),
             )
 
-        if model_request_parameters.output_mode in ('native', 'prompted'):
-            assert model_request_parameters.output_object
-
-            if model_request_parameters.output_tools:
-                model_request_parameters = replace(model_request_parameters, output_tools=[])
-        else:
-            if model_request_parameters.output_object:
-                model_request_parameters = replace(model_request_parameters, output_object=None)
-
-        match model_request_parameters.output_mode:
-            case 'native':
-                if not self.profile.supports_json_schema_output:
-                    raise UserError('Native structured output is not supported by this model.')
-
-                if model_request_parameters.prompted_output_template:
-                    model_request_parameters = replace(model_request_parameters, prompted_output_template=None)
-            case 'prompted':
-                if not model_request_parameters.prompted_output_template:
-                    model_request_parameters = replace(
-                        model_request_parameters, prompted_output_template=self.profile.prompted_output_template
-                    )
-            case 'tool':
-                assert model_request_parameters.output_tools or model_request_parameters.function_tools
-
-                if not self.profile.supports_tools:
-                    raise UserError('Tool output is not supported by this model.')
-            case _:
-                pass
-
-        if model_request_parameters.allow_image_output and not self.profile.supports_image_output:
+        # Reset irrelevant fields
+        if params.output_tools and params.output_mode != 'tool':
+            params = replace(params, output_tools=[])
+        if params.output_object and params.output_mode not in ('native', 'prompted'):
+            params = replace(params, output_object=None)
+        if params.prompted_output_template and params.output_mode != 'prompted':
+            params = replace(params, prompted_output_template=None)
+
+        # Set default prompted output template
+        if params.output_mode == 'prompted' and not params.prompted_output_template:
+            params = replace(params, prompted_output_template=self.profile.prompted_output_template)
+
+        # Check if output mode is supported
+        if params.output_mode == 'native' and not self.profile.supports_json_schema_output:
+            raise UserError('Native structured output is not supported by this model.')
+        if params.output_mode == 'tool' and not self.profile.supports_tools:
+            raise UserError('Tool output is not supported by this model.')
+        if params.allow_image_output and not self.profile.supports_image_output:
             raise UserError('Image output is not supported by this model.')
 
-        return model_settings, model_request_parameters
+        return model_settings, params
 
     @property
     @abstractmethod
@@ -547,17 +541,7 @@ def _get_instructions(
             if all(p.part_kind == 'tool-return' or p.part_kind == 'retry-prompt' for p in most_recent_request.parts):
                 instructions = second_most_recent_request.instructions
 
-        # TODO (DouweM): This will now not be included in ModelRequest.instructions anymore, nor in OTel. -- especially the latter may be a problem?
-        # Unless full model_request_parameters (after processing by model) are already sent
-        if (
-            model_request_parameters
-            and model_request_parameters.output_mode == 'prompted'
-            and model_request_parameters.prompted_output_template
-            and model_request_parameters.output_object
-        ):
-            output_instructions = PromptedOutputSchema.build_instructions(
-                model_request_parameters.prompted_output_template, model_request_parameters.output_object
-            )
+        if model_request_parameters and (output_instructions := model_request_parameters.prompted_output_instructions):
             if instructions:
                 instructions = '\n\n'.join([instructions, output_instructions])
             else:

pydantic_ai_slim/pydantic_ai/models/fallback.py

@@ -80,14 +80,15 @@ async def request(
 
         for model in self.models:
             try:
+                _, prepared_parameters = model.prepare_request(model_settings, model_request_parameters)
                 response = await model.request(messages, model_settings, model_request_parameters)
             except Exception as exc:
                 if self._fallback_on(exc):
                     exceptions.append(exc)
                     continue
                 raise exc
 
-            self._set_span_attributes(model)
+            self._set_span_attributes(model, prepared_parameters)
             return response
 
         raise FallbackExceptionGroup('All models from FallbackModel failed', exceptions)
@@ -106,6 +107,7 @@ async def request_stream(
         for model in self.models:
             async with AsyncExitStack() as stack:
                 try:
+                    _, prepared_parameters = model.prepare_request(model_settings, model_request_parameters)
                     response = await stack.enter_async_context(
                         model.request_stream(messages, model_settings, model_request_parameters, run_context)
                     )
@@ -115,7 +117,7 @@ async def request_stream(
                         continue
                     raise exc  # pragma: no cover
 
-                self._set_span_attributes(model)
+                self._set_span_attributes(model, prepared_parameters)
                 yield response
                 return
 
@@ -128,13 +130,23 @@ def profile(self) -> ModelProfile:
     def customize_request_parameters(self, model_request_parameters: ModelRequestParameters) -> ModelRequestParameters:
         return model_request_parameters
 
-    def _set_span_attributes(self, model: Model):
+    def prepare_request(
+        self, model_settings: ModelSettings | None, model_request_parameters: ModelRequestParameters
+    ) -> tuple[ModelSettings | None, ModelRequestParameters]:
+        return model_settings, model_request_parameters
+
+    def _set_span_attributes(self, model: Model, model_request_parameters: ModelRequestParameters):
         with suppress(Exception):
             span = get_current_span()
             if span.is_recording():
                 attributes = getattr(span, 'attributes', {})
                 if attributes.get('gen_ai.request.model') == self.model_name:  # pragma: no branch
-                    span.set_attributes(InstrumentedModel.model_attributes(model))
+                    span.set_attributes(
+                        {
+                            **InstrumentedModel.model_attributes(model),
+                            **InstrumentedModel.model_request_parameters_attributes(model_request_parameters),
+                        }
+                    )
 
 
 def _default_fallback_condition_factory(exceptions: tuple[type[Exception], ...]) -> Callable[[Exception], bool]:

pydantic_ai_slim/pydantic_ai/models/function.py

@@ -135,6 +135,7 @@ async def request(
             allow_text_output=model_request_parameters.allow_text_output,
             output_tools=model_request_parameters.output_tools,
             model_settings=model_settings,
+            model_request_parameters=model_request_parameters,
             instructions=self._get_instructions(messages, model_request_parameters),
         )
 
@@ -169,6 +170,7 @@ async def request_stream(
             allow_text_output=model_request_parameters.allow_text_output,
             output_tools=model_request_parameters.output_tools,
             model_settings=model_settings,
+            model_request_parameters=model_request_parameters,
             instructions=self._get_instructions(messages, model_request_parameters),
         )
 
@@ -218,6 +220,8 @@ class AgentInfo:
     """The tools that can called to produce the final output of the run."""
     model_settings: ModelSettings | None
     """The model settings passed to the run call."""
+    model_request_parameters: ModelRequestParameters
+    """The model request parameters passed to the run call."""
     instructions: str | None
     """The instructions passed to model."""
 

pydantic_ai_slim/pydantic_ai/models/instrumented.py

@@ -178,17 +178,20 @@ def __init__(
             description='Monetary cost',
         )
 
-    def messages_to_otel_events(self, messages: list[ModelMessage]) -> list[Event]:
+    def messages_to_otel_events(
+        self, messages: list[ModelMessage], parameters: ModelRequestParameters | None = None
+    ) -> list[Event]:
         """Convert a list of model messages to OpenTelemetry events.
 
         Args:
             messages: The messages to convert.
+            parameters: The model request parameters.
 
         Returns:
             A list of OpenTelemetry events.
         """
         events: list[Event] = []
-        instructions = InstrumentedModel._get_instructions(messages)  # pyright: ignore [reportPrivateUsage]
+        instructions = InstrumentedModel._get_instructions(messages, parameters)  # pyright: ignore [reportPrivateUsage]
         if instructions is not None:
             events.append(
                 Event(
@@ -235,10 +238,17 @@ def messages_to_otel_messages(self, messages: list[ModelMessage]) -> list[_otel_
                 result.append(otel_message)
         return result
 
-    def handle_messages(self, input_messages: list[ModelMessage], response: ModelResponse, system: str, span: Span):
+    def handle_messages(
+        self,
+        input_messages: list[ModelMessage],
+        response: ModelResponse,
+        system: str,
+        span: Span,
+        parameters: ModelRequestParameters | None = None,
+    ):
         if self.version == 1:
-            events = self.messages_to_otel_events(input_messages)
-            for event in self.messages_to_otel_events([response]):
+            events = self.messages_to_otel_events(input_messages, parameters)
+            for event in self.messages_to_otel_events([response], parameters):
                 events.append(
                     Event(
                         'gen_ai.choice',
@@ -258,7 +268,7 @@ def handle_messages(self, input_messages: list[ModelMessage], response: ModelRes
             output_messages = self.messages_to_otel_messages([response])
             assert len(output_messages) == 1
             output_message = output_messages[0]
-            instructions = InstrumentedModel._get_instructions(input_messages)  # pyright: ignore [reportPrivateUsage]
+            instructions = InstrumentedModel._get_instructions(input_messages, parameters)  # pyright: ignore [reportPrivateUsage]
             system_instructions_attributes = self.system_instructions_attributes(instructions)
             attributes: dict[str, AttributeValue] = {
                 'gen_ai.input.messages': json.dumps(self.messages_to_otel_messages(input_messages)),
@@ -360,7 +370,7 @@ async def request(
         )
         with self._instrument(messages, prepared_settings, prepared_parameters) as finish:
             response = await self.wrapped.request(messages, model_settings, model_request_parameters)
-            finish(response)
+            finish(response, prepared_parameters)
             return response
 
     @asynccontextmanager
@@ -384,15 +394,15 @@ async def request_stream(
                     yield response_stream
             finally:
                 if response_stream:  # pragma: no branch
-                    finish(response_stream.get())
+                    finish(response_stream.get(), prepared_parameters)
 
     @contextmanager
     def _instrument(
         self,
         messages: list[ModelMessage],
         model_settings: ModelSettings | None,
         model_request_parameters: ModelRequestParameters,
-    ) -> Iterator[Callable[[ModelResponse], None]]:
+    ) -> Iterator[Callable[[ModelResponse, ModelRequestParameters], None]]:
         operation = 'chat'
         span_name = f'{operation} {self.model_name}'
         # TODO Missing attributes:
@@ -401,7 +411,7 @@ def _instrument(
         attributes: dict[str, AttributeValue] = {
             'gen_ai.operation.name': operation,
             **self.model_attributes(self.wrapped),
-            'model_request_parameters': json.dumps(InstrumentedModel.serialize_any(model_request_parameters)),
+            **self.model_request_parameters_attributes(model_request_parameters),
             'logfire.json_schema': json.dumps(
                 {
                     'type': 'object',
@@ -419,7 +429,7 @@ def _instrument(
         try:
             with self.instrumentation_settings.tracer.start_as_current_span(span_name, attributes=attributes) as span:
 
-                def finish(response: ModelResponse):
+                def finish(response: ModelResponse, parameters: ModelRequestParameters):
                     # FallbackModel updates these span attributes.
                     attributes.update(getattr(span, 'attributes', {}))
                     request_model = attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]
@@ -443,7 +453,7 @@ def _record_metrics():
                     if not span.is_recording():
                         return
 
-                    self.instrumentation_settings.handle_messages(messages, response, system, span)
+                    self.instrumentation_settings.handle_messages(messages, response, system, span, parameters)
 
                     attributes_to_set = {
                         **response.usage.opentelemetry_attributes(),
@@ -476,7 +486,7 @@ def _record_metrics():
                 record_metrics()
 
     @staticmethod
-    def model_attributes(model: Model):
+    def model_attributes(model: Model) -> dict[str, AttributeValue]:
         attributes: dict[str, AttributeValue] = {
             GEN_AI_SYSTEM_ATTRIBUTE: model.system,
             GEN_AI_REQUEST_MODEL_ATTRIBUTE: model.model_name,
@@ -494,6 +504,12 @@ def model_attributes(model: Model):
 
         return attributes
 
+    @staticmethod
+    def model_request_parameters_attributes(
+        model_request_parameters: ModelRequestParameters,
+    ) -> dict[str, AttributeValue]:
+        return {'model_request_parameters': json.dumps(InstrumentedModel.serialize_any(model_request_parameters))}
+
     @staticmethod
     def event_to_dict(event: Event) -> dict[str, Any]:
         if not event.body: