use optional env for azure openai tests (#34517)

* use optional env for azure openai tests and remove v0 tests

* update param ToxTestEnv in live test template

* enable optional

* debug

* fix config options

* fixes to path for audio file and install aiohttp dev req

* remove debug logging line

* debugging test failure

* more debug

* undo debug testing, use pathlib for paths, target sys.executable for cli tests

* cspell ignores

* no logprobs

Author: kristapratico

.vscode/cspell.json

@@ -1273,6 +1273,19 @@
         "genai"
       ]
     },
+    {
+      "filename": "sdk/openai/azure-openai/**",
+      "words": [
+        "azuread",
+        "AZUREAD",
+        "dall",
+        "Lepardus",
+        "WEBVTT",
+        "logit",
+        "byod",
+        "logprobs"
+      ]
+    },
     {
       "filename": "sdk/attestation/azure-security-attestation/tests/conftest.py",
       "words":[

eng/pipelines/templates/jobs/live.tests.yml

@@ -58,6 +58,9 @@ parameters:
   - name: TestProxy
     type: boolean
     default: false
+  - name: ToxTestEnv
+    type: string
+    default: 'whl'
 
 jobs:
   - job:
@@ -131,7 +134,7 @@ jobs:
           PostSteps: ${{ parameters.PostSteps }}
           PythonVersion: $(PythonVersion)
           OSVmImage: $(OSVmImage)
-          ToxTestEnv: "whl"
+          ToxTestEnv: ${{ parameters.ToxTestEnv }}
           AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }}
           TestMarkArgument: ${{ parameters.TestMarkArgument }}
           InjectedPackages: ${{ parameters.InjectedPackages }}

eng/pipelines/templates/stages/archetype-sdk-tests.yml

@@ -92,6 +92,9 @@ parameters:
   - name: TestProxy
     type: boolean
     default: false
+  - name: ToxTestEnv
+    type: string
+    default: 'whl'
 
 stages:
 - ${{ each cloud in parameters.CloudConfig }}:
@@ -126,6 +129,7 @@ stages:
               BuildDocs: ${{ parameters.BuildDocs }}
               TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }}
               TestProxy: ${{ parameters.TestProxy }}
+              ToxTestEnv: ${{ parameters.ToxTestEnv }}
             MatrixConfigs:
               # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix).
               - ${{ each config in parameters.MatrixConfigs }}:

sdk/openai/azure-openai/dev_requirements.txt

@@ -1,7 +1,5 @@
 -e ../../../tools/azure-sdk-tools
 -e ../../../tools/azure-devtools
 ../../identity/azure-identity
-numpy
-aiohttp
-requests
 httpx
+aiohttp

sdk/openai/azure-openai/pyproject.toml

@@ -4,3 +4,11 @@ mypy = false
 pyright = false
 type_check_samples = false
 verifytypes = false
+
+[[tool.azure-sdk-build.optional]]
+name = "main"
+install = ["git+https://github.com/openai/openai-python@main"]
+
+[[tool.azure-sdk-build.optional]]
+name = "released"
+install = ["openai"]

sdk/openai/azure-openai/tests/conftest.py

@@ -9,9 +9,6 @@
 import pytest
 import importlib
 import contextlib
-import requests
-import aiohttp
-import yarl
 import functools
 import openai
 from devtools_testutils.sanitizers import add_header_regex_sanitizer, add_oauth_response_sanitizer
@@ -21,8 +18,6 @@
     get_bearer_token_provider as get_bearer_token_provider_async,
 )
 
-# controls whether we run tests against v0 or v1. Options: v0 or v1. Default: v1
-ENV_OPENAI_TEST_MODE = "OPENAI_TEST_MODE"
 
 # for pytest.parametrize
 AZURE = "azure"
@@ -128,12 +123,9 @@ def azure_openai_creds():
         "chat_completion_gpt4_model": ENV_OPENAI_CHAT_COMPLETIONS_GPT4_MODEL,
     }
 
-# openai>=1.0.0 ---------------------------------------------------------------------------
 
 @pytest.fixture
 def client(api_type):
-    if os.getenv(ENV_OPENAI_TEST_MODE, "v1") != "v1":
-        pytest.skip("Skipping - tests set to run against v1.")
     if api_type == "azure":
         client = openai.AzureOpenAI(
             azure_endpoint=os.getenv(ENV_AZURE_OPENAI_ENDPOINT),
@@ -179,8 +171,6 @@ def client(api_type):
 
 @pytest.fixture
 def client_async(api_type):
-    if os.getenv(ENV_OPENAI_TEST_MODE, "v1") != "v1":
-        pytest.skip("Skipping - tests set to run against v1.")
     if api_type == "azure":
         client = openai.AsyncAzureOpenAI(
             azure_endpoint=os.getenv(ENV_AZURE_OPENAI_ENDPOINT),
@@ -308,70 +298,3 @@ def reload():
         yield
     finally:
         importlib.reload(openai)
-
-
-# openai<1.0.0 ---------------------------------------------------------------------------
-
-@pytest.fixture
-def set_vars(api_type):
-    if os.getenv(ENV_OPENAI_TEST_MODE, "v1") != "v0":
-        pytest.skip("Skipping - tests set to run against v0.")
-
-    if api_type == "azure":
-        openai.api_base = os.getenv(ENV_AZURE_OPENAI_ENDPOINT).rstrip("/")
-        openai.api_key = os.getenv(ENV_AZURE_OPENAI_KEY)
-        openai.api_type = "azure"
-        openai.api_version = ENV_AZURE_OPENAI_API_VERSION
-    elif api_type == "azuread":
-        openai.api_base = os.getenv(ENV_AZURE_OPENAI_ENDPOINT).rstrip("/")
-        credential = DefaultAzureCredential()
-        token = credential.get_token("https://cognitiveservices.azure.com/.default")
-        openai.api_type = "azuread"
-        openai.api_key = token.token
-        openai.api_version = ENV_AZURE_OPENAI_API_VERSION
-    elif api_type == "openai":
-        openai.api_base = "https://api.openai.com/v1"
-        openai.api_type = "openai"
-        openai.api_key = os.getenv(ENV_OPENAI_KEY)
-        openai.api_version = None
-    elif api_type == "whisper_azure":
-        openai.api_base = os.getenv(ENV_AZURE_OPENAI_NORTHCENTRALUS_ENDPOINT).rstrip("/")
-        openai.api_key = os.getenv(ENV_AZURE_OPENAI_NORTHCENTRALUS_KEY)
-        openai.api_type = "azure"
-        openai.api_version = ENV_AZURE_OPENAI_API_VERSION
-    elif api_type == "whisper_azuread":
-        openai.api_base = os.getenv(ENV_AZURE_OPENAI_NORTHCENTRALUS_ENDPOINT).rstrip("/")
-        credential = DefaultAzureCredential()
-        token = credential.get_token("https://cognitiveservices.azure.com/.default")
-        openai.api_type = "azuread"
-        openai.api_key = token.token
-        openai.api_version = ENV_AZURE_OPENAI_API_VERSION
-
-
-def configure_v0_async(f):
-    @functools.wraps(f)
-    async def wrapper(*args, **kwargs):
-        api_type = kwargs.pop("api_type")
-        set_vars = kwargs.pop("set_vars")
-        azure_openai_creds = kwargs.pop("azure_openai_creds")
-        try:
-            return await f(*args, set_vars=set_vars, azure_openai_creds=azure_openai_creds, api_type=api_type, **kwargs)
-        except openai.error.RateLimitError:
-            pytest.skip(f"{str(f).split(' ')[1]}[{api_type}]: Skipping - Rate limit reached.")
-
-    return wrapper
-
-
-def configure_v0(f):
-    @functools.wraps(f)
-    def wrapper(*args, **kwargs):
-        api_type = kwargs.pop("api_type")
-        set_vars = kwargs.pop("set_vars")
-        azure_openai_creds = kwargs.pop("azure_openai_creds")
-        try:
-            return f(*args, set_vars=set_vars, azure_openai_creds=azure_openai_creds, api_type=api_type, **kwargs)
-        except openai.error.RateLimitError:
-            pytest.skip(f"{str(f).split(' ')[1]}[{api_type}]: Skipping - Rate limit reached.")
-
-    return wrapper
-

sdk/openai/azure-openai/tests/v1_tests/test_assistants.py renamed to sdk/openai/azure-openai/tests/test_assistants.py

@@ -10,8 +10,8 @@
 from devtools_testutils import AzureRecordedTestCase
 from conftest import WHISPER_AZURE, OPENAI, WHISPER_ALL, configure, TTS_OPENAI, TTS_AZURE, TTS_AZURE_AD
 
-audio_test_file = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "./assets/hello.m4a"))
-audio_long_test_file = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "./assets/wikipediaOcelot.wav"))
+audio_test_file = pathlib.Path(__file__).parent / "./assets/hello.m4a"
+audio_long_test_file = pathlib.Path(__file__).parent / "./assets/wikipediaOcelot.wav"
 
 class TestAudio(AzureRecordedTestCase):
 

sdk/openai/azure-openai/tests/v1_tests/test_assistants_async.py renamed to sdk/openai/azure-openai/tests/test_assistants_async.py

@@ -10,8 +10,8 @@
 from devtools_testutils import AzureRecordedTestCase
 from conftest import WHISPER_AZURE, OPENAI, WHISPER_ALL, configure_async, TTS_OPENAI, TTS_AZURE, TTS_AZURE_AD
 
-audio_test_file = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "./assets/hello.m4a"))
-audio_long_test_file = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "./assets/wikipediaOcelot.wav"))
+audio_test_file = pathlib.Path(__file__).parent / "./assets/hello.m4a"
+audio_long_test_file = pathlib.Path(__file__).parent / "./assets/wikipediaOcelot.wav"
 
 
 class TestAudioAsync(AzureRecordedTestCase):