Add test to document OpenAI o1-mini behavior on system role (#977)

Author: Kludex

tests/conftest.py

@@ -197,3 +197,8 @@ def vcr_config():
         'filter_headers': ['authorization', 'x-api-key'],
         'decode_compressed_response': True,
     }
+
+
+@pytest.fixture(scope='session')
+def openai_key() -> str:
+    return os.getenv('OPENAI_API_KEY', 'mock-api-key')

tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[developer].yaml

@@ -0,0 +1,59 @@
+interactions:
+- request:
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '149'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=k1xoKY2.Jtgshh4aAfJbgzfCta3eg2_pKpOubx52AqY-1740396355-1.0.1.1-piAnJhOpL2bGY4paQ8VURT3k0nvStOyXvtsja2kOYPgRmSNpE1vUf87UU.AnB_di_8lc63rtIkZjSJSSZPPgDQ;
+        _cfuvid=9Q7wbAVWuVxqmo8CZYy4FZvphfjrBGzqwRIt0TKETsE-1740396355005-0.0.1.1-604800000
+      host:
+      - api.openai.com
+    method: POST
+    parsed_body:
+      messages:
+      - content: You are a helpful assistant.
+        role: developer
+      - content: Hello
+        role: user
+      model: o1-mini
+      n: 1
+      stream: false
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    headers:
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      connection:
+      - keep-alive
+      content-length:
+      - '224'
+      content-type:
+      - application/json
+      openai-organization:
+      - pydantic-28gund
+      openai-processing-ms:
+      - '23'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+    parsed_body:
+      error:
+        code: unsupported_value
+        message: 'Unsupported value: ''messages[0].role'' does not support ''developer'' with this model.'
+        param: messages[0].role
+        type: invalid_request_error
+    status:
+      code: 400
+      message: Bad Request
+version: 1

tests/models/cassettes/test_openai/test_openai_o1_mini_system_role[system].yaml

@@ -0,0 +1,61 @@
+interactions:
+- request:
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '146'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+    method: POST
+    parsed_body:
+      messages:
+      - content: You are a helpful assistant.
+        role: system
+      - content: Hello
+        role: user
+      model: o1-mini
+      n: 1
+      stream: false
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    headers:
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      connection:
+      - keep-alive
+      content-length:
+      - '221'
+      content-type:
+      - application/json
+      openai-organization:
+      - pydantic-28gund
+      openai-processing-ms:
+      - '15'
+      openai-version:
+      - '2020-10-01'
+      set-cookie:
+      - __cf_bm=k1xoKY2.Jtgshh4aAfJbgzfCta3eg2_pKpOubx52AqY-1740396355-1.0.1.1-piAnJhOpL2bGY4paQ8VURT3k0nvStOyXvtsja2kOYPgRmSNpE1vUf87UU.AnB_di_8lc63rtIkZjSJSSZPPgDQ;
+        path=/; expires=Mon, 24-Feb-25 11:55:55 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      - _cfuvid=9Q7wbAVWuVxqmo8CZYy4FZvphfjrBGzqwRIt0TKETsE-1740396355005-0.0.1.1-604800000; path=/; domain=.api.openai.com;
+        HttpOnly; Secure; SameSite=None
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+    parsed_body:
+      error:
+        code: unsupported_value
+        message: 'Unsupported value: ''messages[0].role'' does not support ''system'' with this model.'
+        param: messages[0].role
+        type: invalid_request_error
+    status:
+      code: 400
+      message: Bad Request
+version: 1

tests/models/test_openai.py

@@ -29,7 +29,7 @@
 from .mock_async_stream import MockAsyncStream
 
 with try_import() as imports_successful:
-    from openai import NOT_GIVEN, AsyncOpenAI, OpenAIError
+    from openai import NOT_GIVEN, AsyncOpenAI, BadRequestError, OpenAIError
     from openai.types import chat
     from openai.types.chat.chat_completion import Choice
     from openai.types.chat.chat_completion_chunk import (
@@ -560,6 +560,20 @@ async def test_system_prompt_role(
     ]
 
 
+@pytest.mark.parametrize('system_prompt_role', ['system', 'developer'])
+@pytest.mark.vcr
+async def test_openai_o1_mini_system_role(
+    allow_model_requests: None,
+    system_prompt_role: Literal['system', 'developer'],
+    openai_key: str,
+) -> None:
+    model = OpenAIModel('o1-mini', api_key=openai_key, system_prompt_role=system_prompt_role)
+    agent = Agent(model=model, system_prompt='You are a helpful assistant.')
+
+    with pytest.raises(BadRequestError, match=r".*Unsupported value: 'messages\[0\]\.role' does not support.*"):
+        await agent.run('Hello')
+
+
 @pytest.mark.parametrize('parallel_tool_calls', [True, False])
 async def test_parallel_tool_calls(allow_model_requests: None, parallel_tool_calls: bool) -> None:
     c = completion_message(