Skip to content

Commit 1986997

Browse files
kristapraticokristapratico
authored
setting up testing for both preview and GA API versions for AOAI (#34630)
* use optional env for azure openai tests and remove v0 tests * update param ToxTestEnv in live test template * enable optional * debug * fix config options * fixes to path for audio file and install aiohttp dev req * remove debug logging line * debugging test failure * more debug * undo debug testing, use pathlib for paths, target sys.executable for cli tests * setting up testing for both preview and GA versions for AOAI * remove unused code * update the rest of the test files to test against GA/preview API versions if indicated * update conftest * update model version * don't test openai for content filter tests * update for latest ga/preview api versions * fixes for api versions
1 parent 53ba5aa commit 1986997

19 files changed

+901
-615
lines changed

‎sdk/openai/azure-openai/tests/conftest.py‎

Lines changed: 23 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
import contextlib
1212
import functools
1313
import openai
14-
from devtools_testutils.sanitizers import add_header_regex_sanitizer, add_oauth_response_sanitizer
1514
from azure.identity import DefaultAzureCredential, get_bearer_token_provider
1615
from azure.identity.aio import (
1716
DefaultAzureCredential as AsyncDefaultAzureCredential,
@@ -20,27 +19,25 @@
2019

2120

2221
# for pytest.parametrize
22+
GA = "2024-02-01"
23+
PREVIEW = "2024-03-01-preview"
24+
LATEST = PREVIEW
25+
2326
AZURE = "azure"
2427
OPENAI = "openai"
2528
AZURE_AD = "azuread"
26-
ALL = [AZURE, AZURE_AD, OPENAI]
2729
WHISPER_AZURE = "whisper_azure"
2830
WHISPER_AZURE_AD = "whisper_azuread"
29-
WHISPER_ALL = [WHISPER_AZURE, WHISPER_AZURE_AD, OPENAI]
3031
TTS_OPENAI = "tts_openai"
3132
TTS_AZURE = "tts_azure"
3233
TTS_AZURE_AD = "tts_azuread"
33-
TTS_ALL = [TTS_AZURE, TTS_AZURE_AD, TTS_OPENAI]
3434
DALLE_AZURE = "dalle_azure"
3535
DALLE_AZURE_AD = "dalle_azuread"
36-
DALLE_ALL = [DALLE_AZURE, DALLE_AZURE_AD, OPENAI]
3736
GPT_4_AZURE = "gpt_4_azure"
3837
GPT_4_AZURE_AD = "gpt_4_azuread"
3938
GPT_4_OPENAI = "gpt_4_openai"
40-
GPT_4_ALL = [GPT_4_AZURE, GPT_4_AZURE_AD, GPT_4_OPENAI]
4139
ASST_AZURE = "asst_azure"
4240
ASST_AZUREAD = "asst_azuread"
43-
ASST_ALL = [ASST_AZURE, ASST_AZUREAD, GPT_4_OPENAI]
4441

4542
# Environment variable keys
4643
ENV_AZURE_OPENAI_ENDPOINT = "AZ_OPENAI_ENDPOINT"
@@ -57,9 +54,8 @@
5754
ENV_AZURE_OPENAI_SEARCH_KEY = "AZURE_OPENAI_SEARCH_KEY"
5855
ENV_AZURE_OPENAI_SEARCH_INDEX = "AZURE_OPENAI_SEARCH_INDEX"
5956

60-
ENV_AZURE_OPENAI_API_VERSION = "2024-02-15-preview"
6157
ENV_AZURE_OPENAI_COMPLETIONS_NAME = "gpt-35-turbo-instruct"
62-
ENV_AZURE_OPENAI_CHAT_COMPLETIONS_NAME = "gpt-35-turbo-16k"
58+
ENV_AZURE_OPENAI_CHAT_COMPLETIONS_NAME = "gpt-35-turbo"
6359
ENV_AZURE_OPENAI_EMBEDDINGS_NAME = "text-embedding-ada-002"
6460
ENV_AZURE_OPENAI_AUDIO_NAME = "whisper"
6561
ENV_AZURE_OPENAI_DALLE_NAME = "dall-e-3"
@@ -75,68 +71,20 @@
7571
ENV_OPENAI_CHAT_COMPLETIONS_GPT4_MODEL = "gpt-4-1106-preview"
7672
ENV_OPENAI_TTS_MODEL = "tts-1"
7773

78-
# Fake values
79-
TEST_ENDPOINT = "https://test-resource.openai.azure.com/"
80-
TEST_KEY = "0000000000000000"
81-
TEST_ID = "00000000-0000-0000-0000-000000000000"
82-
83-
84-
@pytest.fixture(scope="session", autouse=True)
85-
def add_sanitizers(test_proxy, environment_variables):
86-
sanitization_mapping = {
87-
ENV_AZURE_OPENAI_ENDPOINT: TEST_ENDPOINT,
88-
ENV_AZURE_OPENAI_KEY: TEST_KEY,
89-
ENV_AZURE_OPENAI_NORTHCENTRALUS_ENDPOINT: TEST_ENDPOINT,
90-
ENV_AZURE_OPENAI_NORTHCENTRALUS_KEY: TEST_KEY,
91-
ENV_AZURE_OPENAI_SWEDENCENTRAL_ENDPOINT: TEST_ENDPOINT,
92-
ENV_AZURE_OPENAI_SWEDENCENTRAL_KEY: TEST_KEY,
93-
ENV_SUBSCRIPTION_ID: TEST_ID,
94-
ENV_TENANT_ID: TEST_ID,
95-
ENV_CLIENT_ID: TEST_ID,
96-
ENV_CLIENT_SECRET: TEST_ID,
97-
ENV_OPENAI_KEY: TEST_KEY,
98-
ENV_AZURE_OPENAI_SEARCH_ENDPOINT: TEST_ENDPOINT,
99-
ENV_AZURE_OPENAI_SEARCH_KEY: TEST_KEY,
100-
}
101-
environment_variables.sanitize_batch(sanitization_mapping)
102-
add_oauth_response_sanitizer()
103-
add_header_regex_sanitizer(key="Set-Cookie", value="[set-cookie;]")
104-
105-
106-
@pytest.fixture(scope="session")
107-
def azure_openai_creds():
108-
yield {
109-
"completions_name": ENV_AZURE_OPENAI_COMPLETIONS_NAME,
110-
"chat_completions_name": ENV_AZURE_OPENAI_CHAT_COMPLETIONS_NAME,
111-
"embeddings_name": ENV_AZURE_OPENAI_EMBEDDINGS_NAME,
112-
"completions_model": ENV_OPENAI_COMPLETIONS_MODEL,
113-
"chat_completions_model": ENV_OPENAI_CHAT_COMPLETIONS_MODEL,
114-
"embeddings_model": ENV_OPENAI_EMBEDDINGS_MODEL,
115-
"search_endpoint": os.getenv(ENV_AZURE_OPENAI_SEARCH_ENDPOINT),
116-
"search_key": os.getenv(ENV_AZURE_OPENAI_SEARCH_KEY),
117-
"search_index": os.getenv(ENV_AZURE_OPENAI_SEARCH_INDEX),
118-
"audio_name": ENV_AZURE_OPENAI_AUDIO_NAME,
119-
"audio_model": ENV_OPENAI_AUDIO_MODEL,
120-
"dalle_name": ENV_AZURE_OPENAI_DALLE_NAME,
121-
"dalle_model": ENV_OPENAI_DALLE_MODEL,
122-
"chat_completion_gpt4_name": ENV_AZURE_OPENAI_CHAT_COMPLETIONS_GPT4_NAME,
123-
"chat_completion_gpt4_model": ENV_OPENAI_CHAT_COMPLETIONS_GPT4_MODEL,
124-
}
125-
12674

12775
@pytest.fixture
128-
def client(api_type):
76+
def client(api_type, api_version):
12977
if api_type == "azure":
13078
client = openai.AzureOpenAI(
13179
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_ENDPOINT),
13280
api_key=os.getenv(ENV_AZURE_OPENAI_KEY),
133-
api_version=ENV_AZURE_OPENAI_API_VERSION,
81+
api_version=api_version,
13482
)
13583
elif api_type == "azuread":
13684
client = openai.AzureOpenAI(
13785
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_ENDPOINT),
13886
azure_ad_token_provider=get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"),
139-
api_version=ENV_AZURE_OPENAI_API_VERSION,
87+
api_version=api_version,
14088
)
14189
elif api_type in ["openai", "gpt_4_openai", "tts_openai"]:
14290
client = openai.OpenAI(
@@ -146,42 +94,42 @@ def client(api_type):
14694
client = openai.AzureOpenAI(
14795
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_NORTHCENTRALUS_ENDPOINT),
14896
api_key=os.getenv(ENV_AZURE_OPENAI_NORTHCENTRALUS_KEY),
149-
api_version=ENV_AZURE_OPENAI_API_VERSION,
97+
api_version=api_version,
15098
)
15199
elif api_type in ["whisper_azuread", "tts_azuread"]:
152100
client = openai.AzureOpenAI(
153101
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_NORTHCENTRALUS_ENDPOINT),
154102
azure_ad_token_provider=get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"),
155-
api_version=ENV_AZURE_OPENAI_API_VERSION,
103+
api_version=api_version,
156104
)
157105
elif api_type in ["dalle_azure", "gpt_4_azure", "asst_azure"]:
158106
client = openai.AzureOpenAI(
159107
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_SWEDENCENTRAL_ENDPOINT),
160108
api_key=os.getenv(ENV_AZURE_OPENAI_SWEDENCENTRAL_KEY),
161-
api_version=ENV_AZURE_OPENAI_API_VERSION,
109+
api_version=api_version,
162110
)
163111
elif api_type in ["dalle_azuread", "gpt_4_azuread", "asst_azuread"]:
164112
client = openai.AzureOpenAI(
165113
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_SWEDENCENTRAL_ENDPOINT),
166114
azure_ad_token_provider=get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"),
167-
api_version=ENV_AZURE_OPENAI_API_VERSION,
115+
api_version=api_version,
168116
)
169117
return client
170118

171119

172120
@pytest.fixture
173-
def client_async(api_type):
121+
def client_async(api_type, api_version):
174122
if api_type == "azure":
175123
client = openai.AsyncAzureOpenAI(
176124
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_ENDPOINT),
177125
api_key=os.getenv(ENV_AZURE_OPENAI_KEY),
178-
api_version=ENV_AZURE_OPENAI_API_VERSION,
126+
api_version=api_version,
179127
)
180128
elif api_type == "azuread":
181129
client = openai.AsyncAzureOpenAI(
182130
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_ENDPOINT),
183131
azure_ad_token_provider=get_bearer_token_provider_async(AsyncDefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"),
184-
api_version=ENV_AZURE_OPENAI_API_VERSION,
132+
api_version=api_version,
185133
)
186134
elif api_type in ["openai", "gpt_4_openai", "tts_openai"]:
187135
client = openai.AsyncOpenAI(
@@ -191,25 +139,25 @@ def client_async(api_type):
191139
client = openai.AsyncAzureOpenAI(
192140
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_NORTHCENTRALUS_ENDPOINT),
193141
api_key=os.getenv(ENV_AZURE_OPENAI_NORTHCENTRALUS_KEY),
194-
api_version=ENV_AZURE_OPENAI_API_VERSION,
142+
api_version=api_version,
195143
)
196144
elif api_type in ["whisper_azuread", "tts_azuread"]:
197145
client = openai.AsyncAzureOpenAI(
198146
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_NORTHCENTRALUS_ENDPOINT),
199147
azure_ad_token_provider=get_bearer_token_provider_async(AsyncDefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"),
200-
api_version=ENV_AZURE_OPENAI_API_VERSION,
148+
api_version=api_version,
201149
)
202150
elif api_type in ["dalle_azure", "gpt_4_azure", "asst_azure"]:
203151
client = openai.AsyncAzureOpenAI(
204152
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_SWEDENCENTRAL_ENDPOINT),
205153
api_key=os.getenv(ENV_AZURE_OPENAI_SWEDENCENTRAL_KEY),
206-
api_version=ENV_AZURE_OPENAI_API_VERSION,
154+
api_version=api_version,
207155
)
208156
elif api_type in ["dalle_azuread", "gpt_4_azuread", "asst_azuread"]:
209157
client = openai.AsyncAzureOpenAI(
210158
azure_endpoint=os.getenv(ENV_AZURE_OPENAI_SWEDENCENTRAL_ENDPOINT),
211159
azure_ad_token_provider=get_bearer_token_provider_async(AsyncDefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"),
212-
api_version=ENV_AZURE_OPENAI_API_VERSION,
160+
api_version=api_version,
213161
)
214162
return client
215163

@@ -264,11 +212,11 @@ def configure_async(f):
264212
@functools.wraps(f)
265213
async def wrapper(*args, **kwargs):
266214
api_type = kwargs.pop("api_type")
215+
api_version = kwargs.pop("api_version")
267216
client_async = kwargs.pop("client_async")
268-
azure_openai_creds = kwargs.pop("azure_openai_creds")
269217
kwargs = build_kwargs(args, api_type)
270218
try:
271-
return await f(*args, client_async=client_async, azure_openai_creds=azure_openai_creds, api_type=api_type, **kwargs)
219+
return await f(*args, client_async=client_async, api_type=api_type, api_version=api_version, **kwargs)
272220
except openai.RateLimitError:
273221
pytest.skip(f"{str(f).split(' ')[1]}[{api_type}]: Skipping - Rate limit reached.")
274222

@@ -279,11 +227,11 @@ def configure(f):
279227
@functools.wraps(f)
280228
def wrapper(*args, **kwargs):
281229
api_type = kwargs.pop("api_type")
230+
api_version = kwargs.pop("api_version")
282231
client = kwargs.pop("client")
283-
azure_openai_creds = kwargs.pop("azure_openai_creds")
284232
kwargs = build_kwargs(args, api_type)
285233
try:
286-
return f(*args, client=client, azure_openai_creds=azure_openai_creds, api_type=api_type, **kwargs)
234+
return f(*args, client=client, api_type=api_type, api_version=api_version, **kwargs)
287235
except openai.RateLimitError:
288236
pytest.skip(f"{str(f).split(' ')[1]}[{api_type}]: Skipping - Rate limit reached.")
289237

‎sdk/openai/azure-openai/tests/test_assistants.py‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,18 @@
99
import pathlib
1010
import uuid
1111
from devtools_testutils import AzureRecordedTestCase
12-
from conftest import ASST_AZURE, ASST_ALL, GPT_4_OPENAI, configure
12+
from conftest import ASST_AZURE, ASST_AZUREAD, PREVIEW, GPT_4_OPENAI, configure
1313

1414
TIMEOUT = 300
1515

1616
class TestAssistants(AzureRecordedTestCase):
1717

1818
@configure
19-
@pytest.mark.parametrize("api_type", ASST_ALL)
20-
def test_assistants_crud(self, client, azure_openai_creds, api_type, **kwargs):
19+
@pytest.mark.parametrize(
20+
"api_type, api_version",
21+
[(ASST_AZURE, PREVIEW), (ASST_AZUREAD, PREVIEW), (GPT_4_OPENAI, "v1")]
22+
)
23+
def test_assistants_crud(self, client, api_type, api_version, **kwargs):
2124

2225
try:
2326
assistant = client.beta.assistants.create(
@@ -57,8 +60,8 @@ def test_assistants_crud(self, client, azure_openai_creds, api_type, **kwargs):
5760
assert delete_assistant.deleted is True
5861

5962
@configure
60-
@pytest.mark.parametrize("api_type", [ASST_AZURE, GPT_4_OPENAI])
61-
def test_assistants_files_crud(self, client, azure_openai_creds, api_type, **kwargs):
63+
@pytest.mark.parametrize("api_type, api_version", [(ASST_AZURE, PREVIEW), (GPT_4_OPENAI, "v1")])
64+
def test_assistants_files_crud(self, client, api_type, api_version, **kwargs):
6265
file_name = f"test{uuid.uuid4()}.txt"
6366
with open(file_name, "w") as f:
6467
f.write("test")
@@ -120,8 +123,8 @@ def test_assistants_files_crud(self, client, azure_openai_creds, api_type, **kwa
120123
assert delete_assistant.deleted is True
121124

122125
@configure
123-
@pytest.mark.parametrize("api_type", [ASST_AZURE, GPT_4_OPENAI])
124-
def test_assistants_threads_crud(self, client, azure_openai_creds, api_type, **kwargs):
126+
@pytest.mark.parametrize("api_type, api_version", [(ASST_AZURE, PREVIEW), (GPT_4_OPENAI, "v1")])
127+
def test_assistants_threads_crud(self, client, api_type, api_version, **kwargs):
125128
try:
126129
thread = client.beta.threads.create(
127130
messages=[
@@ -154,8 +157,8 @@ def test_assistants_threads_crud(self, client, azure_openai_creds, api_type, **k
154157
assert delete_thread.deleted is True
155158

156159
@configure
157-
@pytest.mark.parametrize("api_type", [ASST_AZURE, GPT_4_OPENAI])
158-
def test_assistants_messages_crud(self, client, azure_openai_creds, api_type, **kwargs):
160+
@pytest.mark.parametrize("api_type, api_version", [(ASST_AZURE, PREVIEW), (GPT_4_OPENAI, "v1")])
161+
def test_assistants_messages_crud(self, client, api_type, api_version, **kwargs):
159162
file_name = f"test{uuid.uuid4()}.txt"
160163
with open(file_name, "w") as f:
161164
f.write("test")
@@ -235,8 +238,8 @@ def test_assistants_messages_crud(self, client, azure_openai_creds, api_type, **
235238
assert delete_thread.deleted is True
236239

237240
@configure
238-
@pytest.mark.parametrize("api_type", [ASST_AZURE, GPT_4_OPENAI])
239-
def test_assistants_runs_code(self, client, azure_openai_creds, api_type, **kwargs):
241+
@pytest.mark.parametrize("api_type, api_version", [(ASST_AZURE, PREVIEW), (GPT_4_OPENAI, "v1")])
242+
def test_assistants_runs_code(self, client, api_type, api_version, **kwargs):
240243
try:
241244
assistant = client.beta.assistants.create(
242245
name="python test",
@@ -301,8 +304,8 @@ def test_assistants_runs_code(self, client, azure_openai_creds, api_type, **kwar
301304

302305
@pytest.mark.skip("AOAI does not support retrieval tools yet")
303306
@configure
304-
@pytest.mark.parametrize("api_type", [ASST_AZURE, GPT_4_OPENAI])
305-
def test_assistants_runs_retrieval(self, client, azure_openai_creds, api_type, **kwargs):
307+
@pytest.mark.parametrize("api_type, api_version", [(ASST_AZURE, PREVIEW), (GPT_4_OPENAI, "v1")])
308+
def test_assistants_runs_retrieval(self, client, api_type, api_version, **kwargs):
306309
file_name = f"test{uuid.uuid4()}.txt"
307310
with open(file_name, "w") as f:
308311
f.write("Contoso company policy requires that all employees take at least 10 vacation days a year.")
@@ -366,8 +369,8 @@ def test_assistants_runs_retrieval(self, client, azure_openai_creds, api_type, *
366369
assert delete_thread.deleted is True
367370

368371
@configure
369-
@pytest.mark.parametrize("api_type", [ASST_AZURE, GPT_4_OPENAI])
370-
def test_assistants_runs_functions(self, client, azure_openai_creds, api_type, **kwargs):
372+
@pytest.mark.parametrize("api_type, api_version", [(ASST_AZURE, PREVIEW), (GPT_4_OPENAI, "v1")])
373+
def test_assistants_runs_functions(self, client, api_type, api_version, **kwargs):
371374
try:
372375
assistant = client.beta.assistants.create(
373376
name="python test",

0 commit comments

Comments
 (0)