Add OpenAI built-in tools (#1327)

Author: Kludex

pydantic_ai_slim/pydantic_ai/models/openai.py

@@ -2,7 +2,7 @@
 
 import base64
 import warnings
-from collections.abc import AsyncIterable, AsyncIterator
+from collections.abc import AsyncIterable, AsyncIterator, Sequence
 from contextlib import asynccontextmanager
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
@@ -55,6 +55,7 @@
     )
     from openai.types.chat.chat_completion_content_part_image_param import ImageURL
     from openai.types.chat.chat_completion_content_part_input_audio_param import InputAudio
+    from openai.types.responses import ComputerToolParam, FileSearchToolParam, WebSearchToolParam
     from openai.types.responses.response_input_param import FunctionCallOutput, Message
     from openai.types.shared import ReasoningEffort
     from openai.types.shared_params import Reasoning
@@ -64,6 +65,14 @@
         'you can use the `openai` optional group — `pip install "pydantic-ai-slim[openai]"`'
     ) from _import_error
 
+__all__ = (
+    'OpenAIModel',
+    'OpenAIResponsesModel',
+    'OpenAIModelSettings',
+    'OpenAIResponsesModelSettings',
+    'OpenAIModelName',
+)
+
 OpenAIModelName = Union[str, ChatModel]
 """
 Possible OpenAI model names.
@@ -100,6 +109,19 @@ class OpenAIModelSettings(ModelSettings, total=False):
     """
 
 
+class OpenAIResponsesModelSettings(OpenAIModelSettings, total=False):
+    """Settings used for an OpenAI Responses model request.
+
+    ALL FIELDS MUST BE `openai_` PREFIXED SO YOU CAN MERGE THEM WITH OTHER MODELS.
+    """
+
+    openai_builtin_tools: Sequence[FileSearchToolParam | WebSearchToolParam | ComputerToolParam]
+    """The provided OpenAI built-in tools to use.
+
+    See [OpenAI's built-in tools](https://platform.openai.com/docs/guides/tools?api-mode=responses) for more details.
+    """
+
+
 @dataclass(init=False)
 class OpenAIModel(Model):
     """A model that uses the OpenAI API.
@@ -417,6 +439,8 @@ class OpenAIResponsesModel(Model):
     - [File search](https://platform.openai.com/docs/guides/tools-file-search)
     - [Computer use](https://platform.openai.com/docs/guides/tools-computer-use)
 
+    Use the `openai_builtin_tools` setting to add these tools to your model.
+
     If you are interested in the differences between the Responses API and the Chat Completions API,
     see the [OpenAI API docs](https://platform.openai.com/docs/guides/responses-vs-chat-completions).
     """
@@ -462,7 +486,7 @@ async def request(
     ) -> tuple[ModelResponse, usage.Usage]:
         check_allow_model_requests()
         response = await self._responses_create(
-            messages, False, cast(OpenAIModelSettings, model_settings or {}), model_request_parameters
+            messages, False, cast(OpenAIResponsesModelSettings, model_settings or {}), model_request_parameters
         )
         return self._process_response(response), _map_usage(response)
 
@@ -475,7 +499,7 @@ async def request_stream(
     ) -> AsyncIterator[StreamedResponse]:
         check_allow_model_requests()
         response = await self._responses_create(
-            messages, True, cast(OpenAIModelSettings, model_settings or {}), model_request_parameters
+            messages, True, cast(OpenAIResponsesModelSettings, model_settings or {}), model_request_parameters
         )
         async with response:
             yield await self._process_streamed_response(response)
@@ -511,7 +535,7 @@ async def _responses_create(
         self,
         messages: list[ModelRequest | ModelResponse],
         stream: Literal[False],
-        model_settings: OpenAIModelSettings,
+        model_settings: OpenAIResponsesModelSettings,
         model_request_parameters: ModelRequestParameters,
     ) -> responses.Response: ...
 
@@ -520,18 +544,19 @@ async def _responses_create(
         self,
         messages: list[ModelRequest | ModelResponse],
         stream: Literal[True],
-        model_settings: OpenAIModelSettings,
+        model_settings: OpenAIResponsesModelSettings,
         model_request_parameters: ModelRequestParameters,
     ) -> AsyncStream[responses.ResponseStreamEvent]: ...
 
     async def _responses_create(
         self,
         messages: list[ModelRequest | ModelResponse],
         stream: bool,
-        model_settings: OpenAIModelSettings,
+        model_settings: OpenAIResponsesModelSettings,
         model_request_parameters: ModelRequestParameters,
     ) -> responses.Response | AsyncStream[responses.ResponseStreamEvent]:
         tools = self._get_tools(model_request_parameters)
+        tools = list(model_settings.get('openai_builtin_tools', [])) + tools
 
         # standalone function to make it easier to override
         if not tools:
@@ -848,7 +873,7 @@ def _map_usage(response: chat.ChatCompletion | ChatCompletionChunk | responses.R
             },
         )
     else:
-        details: dict[str, int] = {}
+        details = {}
         if response_usage.completion_tokens_details is not None:
             details.update(response_usage.completion_tokens_details.model_dump(exclude_none=True))
         if response_usage.prompt_tokens_details is not None:

tests/models/cassettes/test_openai_responses/test_openai_responses_model_builtin_tools.yaml

@@ -0,0 +1,120 @@
+interactions:
+- request:
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '217'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+    method: POST
+    parsed_body:
+      input:
+      - content: Give me the best news about LLMs from the last 24 hours. Be short.
+        role: user
+      instructions: ''
+      model: gpt-4o
+      stream: false
+      tool_choice: auto
+      tools:
+      - type: web_search_preview
+    uri: https://api.openai.com/v1/responses
+  response:
+    headers:
+      alt-svc:
+      - h3=":443"; ma=86400
+      connection:
+      - keep-alive
+      content-length:
+      - '3210'
+      content-type:
+      - application/json
+      openai-organization:
+      - pydantic-28gund
+      openai-processing-ms:
+      - '2843'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      transfer-encoding:
+      - chunked
+    parsed_body:
+      created_at: 1743506361
+      error: null
+      id: resp_67ebcbb93728819197f923ff16e98bce04f5055a2a33abc3
+      incomplete_details: null
+      instructions: ''
+      max_output_tokens: null
+      metadata: {}
+      model: gpt-4o-2024-08-06
+      object: response
+      output:
+      - id: ws_67ebcbb9ab4481918bebf63b66dbb67c04f5055a2a33abc3
+        status: completed
+        type: web_search_call
+      - content:
+        - annotations:
+          - end_index: 571
+            start_index: 404
+            title: OpenAI plans to release open-weight language model in coming months
+            type: url_citation
+            url: https://www.reuters.com/technology/artificial-intelligence/openai-plans-release-open-weight-language-model-coming-months-2025-03-31/?utm_source=openai
+          - end_index: 846
+            start_index: 625
+            title: OpenAI plans to release open-weight language model in coming months
+            type: url_citation
+            url: https://www.reuters.com/technology/artificial-intelligence/openai-plans-release-open-weight-language-model-coming-months-2025-03-31/?utm_source=openai
+          text: "In the past 24 hours, OpenAI announced plans to release its first open-weight language model with reasoning
+            capabilities since GPT-2. This model will allow developers to fine-tune it for specific applications without needing
+            the original training data. To gather feedback and refine the model, OpenAI will host developer events starting
+            in San Francisco and expanding to Europe and Asia-Pacific regions. ([reuters.com](https://www.reuters.com/technology/artificial-intelligence/openai-plans-release-open-weight-language-model-coming-months-2025-03-31/?utm_source=openai))\n\n\n##
+            OpenAI to Release Open-Weight Language Model:\n- [OpenAI plans to release open-weight language model in coming
+            months](https://www.reuters.com/technology/artificial-intelligence/openai-plans-release-open-weight-language-model-coming-months-2025-03-31/?utm_source=openai) "
+          type: output_text
+        id: msg_67ebcbbaf988819192d44919020b82e704f5055a2a33abc3
+        role: assistant
+        status: completed
+        type: message
+      parallel_tool_calls: true
+      previous_response_id: null
+      reasoning:
+        effort: null
+        generate_summary: null
+      status: completed
+      store: true
+      temperature: 1.0
+      text:
+        format:
+          type: text
+      tool_choice: auto
+      tools:
+      - search_context_size: medium
+        type: web_search_preview
+        user_location:
+          city: null
+          country: US
+          region: null
+          timezone: null
+          type: approximate
+      top_p: 1.0
+      truncation: disabled
+      usage:
+        input_tokens: 320
+        input_tokens_details:
+          cached_tokens: 0
+        output_tokens: 200
+        output_tokens_details:
+          reasoning_tokens: 0
+        total_tokens: 520
+      user: null
+    status:
+      code: 200
+      message: OK
+version: 1

tests/models/test_openai_responses.py

@@ -22,7 +22,7 @@
 from ..conftest import IsDatetime, IsStr, TestEnv, try_import
 
 with try_import() as imports_successful:
-    from pydantic_ai.models.openai import OpenAIModelSettings, OpenAIResponsesModel
+    from pydantic_ai.models.openai import OpenAIModelSettings, OpenAIResponsesModel, OpenAIResponsesModelSettings
     from pydantic_ai.providers.openai import OpenAIProvider
 
 pytestmark = [
@@ -286,3 +286,39 @@ async def test_openai_responses_model_http_error(allow_model_requests: None, ope
     with pytest.raises(ModelHTTPError):
         async with agent.run_stream('What is the capital of France?'):
             ...
+
+
+async def test_openai_responses_model_builtin_tools(allow_model_requests: None, openai_api_key: str):
+    model = OpenAIResponsesModel('gpt-4o', provider=OpenAIProvider(api_key=openai_api_key))
+    settings = OpenAIResponsesModelSettings(openai_builtin_tools=[{'type': 'web_search_preview'}])
+    agent = Agent(model=model, model_settings=settings)
+    result = await agent.run('Give me the best news about LLMs from the last 24 hours. Be short.')
+
+    # NOTE: We don't have the tool call because OpenAI calls the tool internally.
+    assert result.all_messages() == snapshot(
+        [
+            ModelRequest(
+                parts=[
+                    UserPromptPart(
+                        content='Give me the best news about LLMs from the last 24 hours. Be short.',
+                        timestamp=IsDatetime(),
+                    )
+                ]
+            ),
+            ModelResponse(
+                parts=[
+                    TextPart(
+                        content="""\
+In the past 24 hours, OpenAI announced plans to release its first open-weight language model with reasoning capabilities since GPT-2. This model will allow developers to fine-tune it for specific applications without needing the original training data. To gather feedback and refine the model, OpenAI will host developer events starting in San Francisco and expanding to Europe and Asia-Pacific regions. ([reuters.com](https://www.reuters.com/technology/artificial-intelligence/openai-plans-release-open-weight-language-model-coming-months-2025-03-31/?utm_source=openai))
+
+
+## OpenAI to Release Open-Weight Language Model:
+- [OpenAI plans to release open-weight language model in coming months](https://www.reuters.com/technology/artificial-intelligence/openai-plans-release-open-weight-language-model-coming-months-2025-03-31/?utm_source=openai) \
+"""
+                    )
+                ],
+                model_name='gpt-4o-2024-08-06',
+                timestamp=IsDatetime(),
+            ),
+        ]
+    )