validate OpenAI responses (#2226)

samuelcolvin · web-flow · commit 772af1df41b3 · 2025-07-21T18:51:09.000-07:00
diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py
@@ -8,6 +8,7 @@
 from datetime import datetime
 from typing import Any, Literal, Union, cast, overload
 
+from pydantic import ValidationError
 from typing_extensions import assert_never
 
 from pydantic_ai._thinking_part import split_content_into_text_and_thinking
@@ -347,8 +348,19 @@ async def _completions_create(
                 raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e
             raise  # pragma: no cover
 
-    def _process_response(self, response: chat.ChatCompletion) -> ModelResponse:
+    def _process_response(self, response: chat.ChatCompletion | str) -> ModelResponse:
         """Process a non-streamed response, and prepare a message to return."""
+        # Although the OpenAI SDK claims to return a Pydantic model (`ChatCompletion`) from the chat completions function:
+        # * it hasn't actually performed validation (presumably they're creating the model with `model_construct` or something?!)
+        # * if the endpoint returns plain text, the return type is a string
+        # Thus we validate it fully here.
+        if not isinstance(response, chat.ChatCompletion):
+            raise UnexpectedModelBehavior('Invalid response from OpenAI chat completions endpoint, expected JSON data')
+
+        try:
+            response = chat.ChatCompletion.model_validate(response.model_dump())
+        except ValidationError as e:
+            raise UnexpectedModelBehavior(f'Invalid response from OpenAI chat completions endpoint: {e}') from e
         timestamp = number_to_datetime(response.created)
         choice = response.choices[0]
         items: list[ModelResponsePart] = []
diff --git a/tests/models/cassettes/test_openai/test_invalid_response.yaml b/tests/models/cassettes/test_openai/test_invalid_response.yaml
@@ -0,0 +1,51 @@
+interactions:
+- request:
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '105'
+      content-type:
+      - application/json
+      host:
+      - demo-endpoints.pydantic.workers.dev
+    method: POST
+    parsed_body:
+      messages:
+      - content: What is the capital of France?
+        role: user
+      model: gpt-4o
+      stream: false
+    uri: https://demo-endpoints.pydantic.workers.dev/bin/content-type/application/json/chat/completions
+  response:
+    headers:
+      alt-svc:
+      - h3=":443"; ma=86400
+      connection:
+      - keep-alive
+      content-length:
+      - '128'
+      content-type:
+      - application/json
+      nel:
+      - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
+      report-to:
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=vnOen4x5ThZsFrq57KAufS6JIp6%2FonMEN9WyAWXKhWzx0nNhyrIm3l5ffXE0t9yP69ay6%2Bj8TXT4jmQqDkjFOqlTXQ0lpQa7jkrZpXjuk1iD2hEyEZd5q%2F6ZKddrnPGojfa4%2FOwgp3aw2wf3DFzFZoPWYFhlEA%3D%3D"}],"group":"cf-nel","max_age":604800}'
+      server-timing:
+      - cfL4;desc="?proto=TCP&rtt=5088&min_rtt=4528&rtt_var=1666&sent=5&recv=8&lost=0&retrans=0&sent_bytes=2868&recv_bytes=1339&delivery_rate=756649&cwnd=252&unsent_bytes=0&cid=1ee35b7dfe7143b8&ts=51&x=0"
+      transfer-encoding:
+      - chunked
+      vary:
+      - Accept-Encoding
+    parsed_body:
+      contentType: application/json
+      method: POST
+      pathname: /bin/content-type/application/json/chat/completions
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/models/cassettes/test_openai/test_text_response.yaml b/tests/models/cassettes/test_openai/test_text_response.yaml
@@ -0,0 +1,49 @@
+interactions:
+- request:
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '105'
+      content-type:
+      - application/json
+      host:
+      - demo-endpoints.pydantic.workers.dev
+    method: POST
+    parsed_body:
+      messages:
+      - content: What is the capital of France?
+        role: user
+      model: gpt-4o
+      stream: false
+    uri: https://demo-endpoints.pydantic.workers.dev/bin/chat/completions
+  response:
+    body:
+      string: method=POST pathname=/bin/chat/completions Content-Type=text/plain
+    headers:
+      alt-svc:
+      - h3=":443"; ma=86400
+      connection:
+      - keep-alive
+      content-length:
+      - '66'
+      content-type:
+      - text/plain
+      nel:
+      - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
+      report-to:
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=JNLqg8RHZTY3qqAmfwzA3vjAJCnVIWrBopVnzEbxZacVCpdlDStUhB%2BnUFpk%2BK51POBOH8s6zKMJkA%2FDNORrbGZiP7MfeOrH5wmiqrw4D2F2L3L8w8GBYioreKodF%2BTsCrbqR0Y6XReZHA86T9IGo94AtnBlQg%3D%3D"}],"group":"cf-nel","max_age":604800}'
+      server-timing:
+      - cfL4;desc="?proto=TCP&rtt=24558&min_rtt=23830&rtt_var=9456&sent=5&recv=7&lost=0&retrans=0&sent_bytes=2869&recv_bytes=1309&delivery_rate=175493&cwnd=33&unsent_bytes=0&cid=4087bdc474291a40&ts=53&x=0"
+      transfer-encoding:
+      - chunked
+      vary:
+      - Accept-Encoding
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/models/cassettes/test_openai/test_valid_response.yaml b/tests/models/cassettes/test_openai/test_valid_response.yaml
@@ -0,0 +1,79 @@
+interactions:
+- request:
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '105'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+    method: POST
+    parsed_body:
+      messages:
+      - content: What is the capital of France?
+        role: user
+      model: gpt-4o
+      stream: false
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    headers:
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      connection:
+      - keep-alive
+      content-length:
+      - '832'
+      content-type:
+      - application/json
+      openai-organization:
+      - pydantic-28gund
+      openai-processing-ms:
+      - '288'
+      openai-project:
+      - proj_wlzE3wrTAwGKSsoZUKNhfDgz
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      transfer-encoding:
+      - chunked
+    parsed_body:
+      choices:
+      - finish_reason: stop
+        index: 0
+        logprobs: null
+        message:
+          annotations: []
+          content: The capital of France is Paris.
+          refusal: null
+          role: assistant
+      created: 1752720361
+      id: chatcmpl-Bu8vBIrB8kIWKRyTcpEEPncjhHtMU
+      model: gpt-4o-2024-08-06
+      object: chat.completion
+      service_tier: default
+      system_fingerprint: fp_a288987b44
+      usage:
+        completion_tokens: 7
+        completion_tokens_details:
+          accepted_prediction_tokens: 0
+          audio_tokens: 0
+          reasoning_tokens: 0
+          rejected_prediction_tokens: 0
+        prompt_tokens: 14
+        prompt_tokens_details:
+          audio_tokens: 0
+          cached_tokens: 0
+        total_tokens: 21
+    status:
+      code: 200
+      message: OK
+version: 1
diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py
@@ -46,7 +46,7 @@
 from pydantic_ai.settings import ModelSettings
 from pydantic_ai.tools import ToolDefinition
 
-from ..conftest import IsDatetime, IsInstance, IsNow, IsStr, raise_if_exception, try_import
+from ..conftest import IsDatetime, IsInstance, IsNow, IsStr, TestEnv, raise_if_exception, try_import
 from .mock_async_stream import MockAsyncStream
 
 with try_import() as imports_successful:
@@ -2539,3 +2539,43 @@ async def get_user_country() -> str:
             ),
         ]
     )
+
+
+async def test_valid_response(env: TestEnv, allow_model_requests: None):
+    """VCR recording is of a valid response."""
+    env.set('OPENAI_API_KEY', 'foobar')
+    agent = Agent('openai:gpt-4o')
+
+    result = await agent.run('What is the capital of France?')
+    assert result.output == snapshot('The capital of France is Paris.')
+
+
+async def test_invalid_response(allow_model_requests: None):
+    """VCR recording is of an invalid JSON response."""
+    m = OpenAIModel(
+        'gpt-4o',
+        provider=OpenAIProvider(
+            api_key='foobar', base_url='https://demo-endpoints.pydantic.workers.dev/bin/content-type/application/json'
+        ),
+    )
+    agent = Agent(m)
+
+    with pytest.raises(UnexpectedModelBehavior) as exc_info:
+        await agent.run('What is the capital of France?')
+    assert exc_info.value.message.startswith(
+        'Invalid response from OpenAI chat completions endpoint: 5 validation errors for ChatCompletion'
+    )
+
+
+async def test_text_response(allow_model_requests: None):
+    """VCR recording is of a text response."""
+    m = OpenAIModel(
+        'gpt-4o', provider=OpenAIProvider(api_key='foobar', base_url='https://demo-endpoints.pydantic.workers.dev/bin/')
+    )
+    agent = Agent(m)
+
+    with pytest.raises(UnexpectedModelBehavior) as exc_info:
+        await agent.run('What is the capital of France?')
+    assert exc_info.value.message == snapshot(
+        'Invalid response from OpenAI chat completions endpoint, expected JSON data'
+    )
