Skip to content

Commit e964d98

Browse files
kristapraticokristapratico
authored
update tests to target aug preview version (#37360)
* update tests to target aug preview version * adjust models where gpt-4o not supported
1 parent 58bdd75 commit e964d98

File tree

6 files changed

+102
-17
lines changed

6 files changed

+102
-17
lines changed

‎sdk/openai/azure-openai/dev_requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@
33
aiohttp
44
openai
55
pillow
6+
pydantic

‎sdk/openai/azure-openai/tests/conftest.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
# for pytest.parametrize
2323
GA = "2024-06-01"
24-
PREVIEW = "2024-07-01-preview"
24+
PREVIEW = "2024-08-01-preview"
2525
LATEST = PREVIEW
2626

2727
AZURE = "azure"
@@ -48,7 +48,7 @@
4848
ENV_AZURE_OPENAI_EMBEDDINGS_NAME = "text-embedding-ada-002"
4949
ENV_AZURE_OPENAI_AUDIO_NAME = "whisper"
5050
ENV_AZURE_OPENAI_DALLE_NAME = "dall-e-3"
51-
ENV_AZURE_OPENAI_CHAT_COMPLETIONS_GPT4_NAME = "gpt-4-1106-preview"
51+
ENV_AZURE_OPENAI_CHAT_COMPLETIONS_GPT4_NAME = "gpt-4o"
5252
ENV_AZURE_OPENAI_TTS_NAME = "tts"
5353

5454
ENV_OPENAI_KEY = "OPENAI_KEY"
@@ -57,7 +57,7 @@
5757
ENV_OPENAI_EMBEDDINGS_MODEL = "text-embedding-ada-002"
5858
ENV_OPENAI_AUDIO_MODEL = "whisper-1"
5959
ENV_OPENAI_DALLE_MODEL = "dall-e-3"
60-
ENV_OPENAI_CHAT_COMPLETIONS_GPT4_MODEL = "gpt-4-1106-preview"
60+
ENV_OPENAI_CHAT_COMPLETIONS_GPT4_MODEL = "gpt-4o-2024-08-06"
6161
ENV_OPENAI_TTS_MODEL = "tts-1"
6262

6363

‎sdk/openai/azure-openai/tests/test_assistants.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ def test_assistants_crud(self, client, api_type, api_version, **kwargs):
201201
name="python test",
202202
instructions="You are a personal math tutor. Write and run code to answer math questions.",
203203
tools=[{"type": "code_interpreter"}],
204-
**kwargs,
204+
model="gpt-4-1106-preview",
205205
)
206206
retrieved_assistant = client.beta.assistants.retrieve(
207207
assistant_id=assistant.id,
@@ -489,7 +489,7 @@ def test_assistants_runs_code(self, client, api_type, api_version, **kwargs):
489489
name="python test",
490490
instructions="You are a personal math tutor. Write and run code to answer math questions.",
491491
tools=[{"type": "code_interpreter"}],
492-
**kwargs,
492+
model="gpt-4-1106-preview",
493493
)
494494

495495
thread = client.beta.threads.create()
@@ -561,7 +561,7 @@ def test_assistants_runs_file_search(self, client, api_type, api_version, **kwar
561561
"vector_store_ids": [vector_store.id]
562562
}
563563
},
564-
**kwargs
564+
model="gpt-4-1106-preview"
565565
)
566566

567567
run = client.beta.threads.create_and_run_poll(
@@ -629,7 +629,7 @@ def test_assistants_runs_functions(self, client, api_type, api_version, **kwargs
629629
}
630630
}
631631
],
632-
**kwargs,
632+
model="gpt-4-1106-preview",
633633
)
634634

635635
run = client.beta.threads.create_and_run_poll(
@@ -710,7 +710,7 @@ def test_assistants_streaming(self, client, api_type, api_version, **kwargs):
710710
name="Math Tutor",
711711
instructions="You are a personal math tutor. Write and run code to answer math questions.",
712712
tools=[{"type": "code_interpreter"}],
713-
**kwargs,
713+
model="gpt-4-1106-preview",
714714
)
715715
try:
716716
thread = client.beta.threads.create()
@@ -738,7 +738,7 @@ def test_assistants_stream_event_handler(self, client, api_type, api_version, **
738738
name="Math Tutor",
739739
instructions="You are a personal math tutor. Write and run code to answer math questions.",
740740
tools=[{"type": "code_interpreter"}],
741-
**kwargs
741+
model="gpt-4-1106-preview"
742742
)
743743

744744
try:

‎sdk/openai/azure-openai/tests/test_assistants_async.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ async def test_assistants_crud(self, client_async, api_type, api_version, **kwar
200200
name="python test",
201201
instructions="You are a personal math tutor. Write and run code to answer math questions.",
202202
tools=[{"type": "code_interpreter"}],
203-
**kwargs,
203+
model="gpt-4-1106-preview",
204204
)
205205
retrieved_assistant = await client_async.beta.assistants.retrieve(
206206
assistant_id=assistant.id,
@@ -493,7 +493,7 @@ async def test_assistants_runs_code(self, client_async, api_type, api_version, *
493493
name="python test",
494494
instructions="You are a personal math tutor. Write and run code to answer math questions.",
495495
tools=[{"type": "code_interpreter"}],
496-
**kwargs,
496+
model="gpt-4-1106-preview",
497497
)
498498
thread = await client_async.beta.threads.create()
499499

@@ -565,7 +565,7 @@ async def test_assistants_runs_file_search(self, client_async, api_type, api_ver
565565
"vector_store_ids": [vector_store.id]
566566
}
567567
},
568-
**kwargs
568+
model="gpt-4-1106-preview"
569569
)
570570

571571
run = await client_async.beta.threads.create_and_run_poll(
@@ -634,7 +634,7 @@ async def test_assistants_runs_functions(self, client_async, api_type, api_versi
634634
}
635635
}
636636
],
637-
**kwargs,
637+
model="gpt-4-1106-preview",
638638
)
639639

640640
run = await client_async.beta.threads.create_and_run_poll(
@@ -717,7 +717,7 @@ async def test_assistants_streaming(self, client_async, api_type, api_version, *
717717
name="Math Tutor",
718718
instructions="You are a personal math tutor. Write and run code to answer math questions.",
719719
tools=[{"type": "code_interpreter"}],
720-
**kwargs,
720+
model="gpt-4-1106-preview",
721721
)
722722
try:
723723
thread = await client_async.beta.threads.create()
@@ -746,7 +746,7 @@ async def test_assistants_stream_event_handler(self, client_async, api_type, api
746746
name="Math Tutor",
747747
instructions="You are a personal math tutor. Write and run code to answer math questions.",
748748
tools=[{"type": "code_interpreter"}],
749-
**kwargs
749+
model="gpt-4-1106-preview"
750750
)
751751

752752
try:

‎sdk/openai/azure-openai/tests/test_chat_completions.py‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
import os
77
import pytest
88
import json
9+
from typing import List
10+
from pydantic import BaseModel
911
import openai
1012
from devtools_testutils import AzureRecordedTestCase
1113
from conftest import (
@@ -843,7 +845,7 @@ def test_chat_completion_block_list_term(self, client, api_type, api_version, **
843845
{"role": "user", "content": "What is the best time of year to pick pineapple?"}
844846
]
845847
with pytest.raises(openai.BadRequestError) as e:
846-
client.chat.completions.create(messages=messages, **kwargs)
848+
client.chat.completions.create(messages=messages, model="gpt-4-1106-preview")
847849
err = e.value.body
848850
assert err["code"] == "content_filter"
849851
content_filter_result = err["innererror"]["content_filter_result"]
@@ -1159,3 +1161,43 @@ def test_chat_completion_logprobs(self, client, api_type, api_version, **kwargs)
11591161
assert logprob.token is not None
11601162
assert logprob.logprob is not None
11611163
assert logprob.bytes is not None
1164+
1165+
@configure
1166+
@pytest.mark.parametrize("api_type, api_version", [(GPT_4_AZURE, PREVIEW), (GPT_4_OPENAI, "v1")])
1167+
def test_chat_completion_structured_outputs(self, client, api_type, api_version, **kwargs):
1168+
1169+
class Step(BaseModel):
1170+
explanation: str
1171+
output: str
1172+
1173+
class MathResponse(BaseModel):
1174+
steps: List[Step]
1175+
final_answer: str
1176+
1177+
completion = client.beta.chat.completions.parse(
1178+
messages=[
1179+
{"role": "system", "content": "You are a helpful math tutor."},
1180+
{"role": "user", "content": "solve 8x + 31 = 2"},
1181+
],
1182+
response_format=MathResponse,
1183+
**kwargs,
1184+
)
1185+
1186+
assert completion.id
1187+
assert completion.object == "chat.completion"
1188+
assert completion.model
1189+
assert completion.created
1190+
assert completion.usage.completion_tokens is not None
1191+
assert completion.usage.prompt_tokens is not None
1192+
assert completion.usage.total_tokens == completion.usage.completion_tokens + completion.usage.prompt_tokens
1193+
assert len(completion.choices) == 1
1194+
assert completion.choices[0].finish_reason
1195+
assert completion.choices[0].index is not None
1196+
assert completion.choices[0].message.content is not None
1197+
assert completion.choices[0].message.role
1198+
if completion.choices[0].message.parsed:
1199+
assert completion.choices[0].message.parsed.steps
1200+
for step in completion.choices[0].message.parsed.steps:
1201+
assert step.explanation
1202+
assert step.output
1203+
assert completion.choices[0].message.parsed.final_answer

‎sdk/openai/azure-openai/tests/test_chat_completions_async.py‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
import os
77
import pytest
88
import json
9+
from typing import List
10+
from pydantic import BaseModel
911
import openai
1012
from devtools_testutils import AzureRecordedTestCase
1113
from conftest import (
@@ -864,7 +866,7 @@ async def test_chat_completion_block_list_term(self, client_async, api_type, api
864866
{"role": "user", "content": "What is the best time of year to pick pineapple?"}
865867
]
866868
with pytest.raises(openai.BadRequestError) as e:
867-
await client_async.chat.completions.create(messages=messages, **kwargs)
869+
await client_async.chat.completions.create(messages=messages, model="gpt-4-1106-preview")
868870
err = e.value.body
869871
assert err["code"] == "content_filter"
870872
content_filter_result = err["innererror"]["content_filter_result"]
@@ -1185,3 +1187,43 @@ async def test_chat_completion_logprobs(self, client_async, api_type, api_versio
11851187
assert logprob.token is not None
11861188
assert logprob.logprob is not None
11871189
assert logprob.bytes is not None
1190+
1191+
@configure_async
1192+
@pytest.mark.asyncio
1193+
@pytest.mark.parametrize("api_type, api_version", [(GPT_4_AZURE, PREVIEW), (GPT_4_OPENAI, "v1")])
1194+
async def test_chat_completion_structured_outputs(self, client_async, api_type, api_version, **kwargs):
1195+
1196+
class Step(BaseModel):
1197+
explanation: str
1198+
output: str
1199+
1200+
class MathResponse(BaseModel):
1201+
steps: List[Step]
1202+
final_answer: str
1203+
1204+
completion = await client_async.beta.chat.completions.parse(
1205+
messages=[
1206+
{"role": "system", "content": "You are a helpful math tutor. You only answer about math. Refuse to answer any other question."},
1207+
{"role": "user", "content": "solve 8x + 31 = 2"},
1208+
],
1209+
response_format=MathResponse,
1210+
**kwargs,
1211+
)
1212+
assert completion.id
1213+
assert completion.object == "chat.completion"
1214+
assert completion.model
1215+
assert completion.created
1216+
assert completion.usage.completion_tokens is not None
1217+
assert completion.usage.prompt_tokens is not None
1218+
assert completion.usage.total_tokens == completion.usage.completion_tokens + completion.usage.prompt_tokens
1219+
assert len(completion.choices) == 1
1220+
assert completion.choices[0].finish_reason
1221+
assert completion.choices[0].index is not None
1222+
assert completion.choices[0].message.content is not None
1223+
assert completion.choices[0].message.role
1224+
if completion.choices[0].message.parsed:
1225+
assert completion.choices[0].message.parsed.steps
1226+
for step in completion.choices[0].message.parsed.steps:
1227+
assert step.explanation
1228+
assert step.output
1229+
assert completion.choices[0].message.parsed.final_answer

0 commit comments

Comments
 (0)