Add 'type' Key to Model Configurations and Some Cleanup (Azure#37569)

needuv · web-flow · commit 309b2e07798d · 2024-09-26T16:36:15.000Z
* Add 'type' Key to Model Configurations and Some Cleanup

* get rid of one line functions

* add retrieval chat eval

* clean up model configuration classes

* add changelog entry

* fix missing import

* fix an import

* lint and add docstrings for typed dicts

* fix docstrings

* rerecord failing tests

* add connections.json to gitignore
diff --git a/.gitignore b/.gitignore
@@ -123,6 +123,9 @@ sdk/storage/azure-storage-blob-changefeed/tests/settings/settings_real.py
 !sdk/ml/azure-ai-ml/tests/test_configs/*/*/environment*
 !sdk/ml/azure-ai-ml/tests/environment*
 
+# azure-ai-evaluation connections file
+sdk/evaluation/azure-ai-evaluation/connections.json
+
 
 # The locations below are deprecated - keep to prevent any accidental secrets leakage ==========
 sdk/storage/azure-storage-blob/tests/_shared/settings_real.py
diff --git a/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md b/sdk/evaluation/azure-ai-evaluation/CHANGELOG.md
@@ -4,6 +4,8 @@
 
 ### Features Added
 
+- Added `type` field to `AzureOpenAIModelConfiguration` and `OpenAIModelConfiguration`
+
 ### Breaking Changes
 
 ### Bugs Fixed
diff --git a/sdk/evaluation/azure-ai-evaluation/assets.json b/sdk/evaluation/azure-ai-evaluation/assets.json
@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/evaluation/azure-ai-evaluation",
-  "Tag": "python/evaluation/azure-ai-evaluation_9ac3e64c3e"
+  "Tag": "python/evaluation/azure-ai-evaluation_26cf396fa1"
 }
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py
@@ -3,12 +3,13 @@
 # ---------------------------------------------------------
 
 import threading
-from typing import List, Optional, Union
+from typing import List, Union
 
 import nltk
 import numpy as np
 
 from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from azure.ai.evaluation._constants import AZURE_OPENAI_TYPE, OPENAI_TYPE
 
 from . import constants
 
@@ -70,18 +71,32 @@ def nltk_tokenize(text: str) -> List[str]:
     return list(tokens)
 
 
-def ensure_api_version_in_aoai_model_config(
+def parse_model_config_type(
     model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
-    default_api_version: str,
 ) -> None:
     if "azure_endpoint" in model_config or "azure_deployment" in model_config:
-        model_config["api_version"] = model_config.get("api_version", default_api_version)
+        model_config["type"] = AZURE_OPENAI_TYPE
+    else:
+        model_config["type"] = OPENAI_TYPE
 
 
-def ensure_user_agent_in_aoai_model_config(
+def construct_prompty_model_config(
     model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
-    prompty_model_config: dict,
-    user_agent: Optional[str] = None,
-) -> None:
-    if user_agent and ("azure_endpoint" in model_config or "azure_deployment" in model_config):
+    default_api_version: str,
+    user_agent: str,
+) -> dict:
+    parse_model_config_type(model_config)
+
+    if model_config["type"] == AZURE_OPENAI_TYPE:
+        model_config["api_version"] = model_config.get("api_version", default_api_version)
+
+    prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
+
+    # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
+    # https://github.com/encode/httpx/discussions/2959
+    prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
+
+    if model_config["type"] == AZURE_OPENAI_TYPE and user_agent:
         prompty_model_config["parameters"]["extra_headers"].update({"x-ms-useragent": user_agent})
+
+    return prompty_model_config
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_constants.py
@@ -57,3 +57,7 @@ class DefaultOpenEncoding:
 
 OTEL_EXPORTER_OTLP_TRACES_TIMEOUT = "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT"
 OTEL_EXPORTER_OTLP_TRACES_TIMEOUT_DEFAULT = 60
+
+AZURE_OPENAI_TYPE = "azure_openai"
+
+OPENAI_TYPE = "openai"
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_chat/retrieval/_retrieval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_chat/retrieval/_retrieval.py
@@ -11,7 +11,7 @@
 from promptflow._utils.async_utils import async_run_allowing_running_loop
 from promptflow.core import AsyncPrompty
 
-from ...._common.utils import ensure_api_version_in_aoai_model_config, ensure_user_agent_in_aoai_model_config
+from ...._common.utils import construct_prompty_model_config
 
 logger = logging.getLogger(__name__)
 
@@ -28,17 +28,9 @@ class _AsyncRetrievalChatEvaluator:
     DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
     def __init__(self, model_config: dict):
-        ensure_api_version_in_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
-
-        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
-
-        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
-        # https://github.com/encode/httpx/discussions/2959
-        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
-
-        ensure_user_agent_in_aoai_model_config(
+        prompty_model_config = construct_prompty_model_config(
             model_config,
-            prompty_model_config,
+            self.DEFAULT_OPEN_API_VERSION,
             USER_AGENT,
         )
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_coherence/_coherence.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_coherence/_coherence.py
@@ -11,7 +11,7 @@
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
-from ..._common.utils import ensure_api_version_in_aoai_model_config, ensure_user_agent_in_aoai_model_config
+from ..._common.utils import construct_prompty_model_config
 
 try:
     from ..._user_agent import USER_AGENT
@@ -26,17 +26,9 @@ class _AsyncCoherenceEvaluator:
     DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
     def __init__(self, model_config: dict):
-        ensure_api_version_in_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
-
-        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
-
-        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
-        # https://github.com/encode/httpx/discussions/2959
-        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
-
-        ensure_user_agent_in_aoai_model_config(
+        prompty_model_config = construct_prompty_model_config(
             model_config,
-            prompty_model_config,
+            self.DEFAULT_OPEN_API_VERSION,
             USER_AGENT,
         )
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_fluency/_fluency.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_fluency/_fluency.py
@@ -11,7 +11,7 @@
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
-from ..._common.utils import ensure_api_version_in_aoai_model_config, ensure_user_agent_in_aoai_model_config
+from ..._common.utils import construct_prompty_model_config
 
 try:
     from ..._user_agent import USER_AGENT
@@ -26,17 +26,9 @@ class _AsyncFluencyEvaluator:
     DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
     def __init__(self, model_config: dict):
-        ensure_api_version_in_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
-
-        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
-
-        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
-        # https://github.com/encode/httpx/discussions/2959
-        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
-
-        ensure_user_agent_in_aoai_model_config(
+        prompty_model_config = construct_prompty_model_config(
             model_config,
-            prompty_model_config,
+            self.DEFAULT_OPEN_API_VERSION,
             USER_AGENT,
         )
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py
@@ -11,7 +11,7 @@
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
-from ..._common.utils import ensure_api_version_in_aoai_model_config, ensure_user_agent_in_aoai_model_config
+from ..._common.utils import construct_prompty_model_config
 
 try:
     from ..._user_agent import USER_AGENT
@@ -26,17 +26,9 @@ class _AsyncGroundednessEvaluator:
     DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
     def __init__(self, model_config: dict):
-        ensure_api_version_in_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
-
-        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
-
-        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
-        # https://github.com/encode/httpx/discussions/2959
-        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
-
-        ensure_user_agent_in_aoai_model_config(
+        prompty_model_config = construct_prompty_model_config(
             model_config,
-            prompty_model_config,
+            self.DEFAULT_OPEN_API_VERSION,
             USER_AGENT,
         )
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_relevance/_relevance.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_relevance/_relevance.py
@@ -11,7 +11,7 @@
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
-from ..._common.utils import ensure_api_version_in_aoai_model_config, ensure_user_agent_in_aoai_model_config
+from ..._common.utils import construct_prompty_model_config
 
 try:
     from ..._user_agent import USER_AGENT
@@ -26,17 +26,9 @@ class _AsyncRelevanceEvaluator:
     DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
     def __init__(self, model_config: dict):
-        ensure_api_version_in_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
-
-        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
-
-        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
-        # https://github.com/encode/httpx/discussions/2959
-        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
-
-        ensure_user_agent_in_aoai_model_config(
+        prompty_model_config = construct_prompty_model_config(
             model_config,
-            prompty_model_config,
+            self.DEFAULT_OPEN_API_VERSION,
             USER_AGENT,
         )
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_similarity/_similarity.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_similarity/_similarity.py
@@ -11,7 +11,7 @@
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
-from ..._common.utils import ensure_api_version_in_aoai_model_config, ensure_user_agent_in_aoai_model_config
+from ..._common.utils import construct_prompty_model_config
 
 try:
     from ..._user_agent import USER_AGENT
@@ -26,17 +26,9 @@ class _AsyncSimilarityEvaluator:
     DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
     def __init__(self, model_config: dict):
-        ensure_api_version_in_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
-
-        prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
-
-        # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
-        # https://github.com/encode/httpx/discussions/2959
-        prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
-
-        ensure_user_agent_in_aoai_model_config(
+        prompty_model_config = construct_prompty_model_config(
             model_config,
-            prompty_model_config,
+            self.DEFAULT_OPEN_API_VERSION,
             USER_AGENT,
         )
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_model_configurations.py
@@ -2,26 +2,47 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from typing import TypedDict
+from typing import Literal, TypedDict
 
+from typing_extensions import NotRequired
 
-class AzureOpenAIModelConfigurationBase(TypedDict):
+
+class AzureOpenAIModelConfiguration(TypedDict, total=False):
+    """Model Configuration for Azure OpenAI Model"""
+
+    type: Literal["azure_openai"]
+    """The type of the model configuration. Should be 'azure_openai' for AzureOpenAIModelConfiguration"""
     azure_deployment: str
+    """Name of Azure OpenAI deployment to make request to"""
     azure_endpoint: str
+    """Endpoint of Azure OpenAI resource to make request to"""
     api_key: str
+    """API key of Azure OpenAI resource"""
+    api_version: NotRequired[str]
+    """(Optional) API version to use in request to Azure OpenAI deployment"""
 
 
-class AzureOpenAIModelConfiguration(AzureOpenAIModelConfigurationBase, total=False):
-    api_version: str
+class OpenAIModelConfiguration(TypedDict, total=False):
+    """Model Configuration for OpenAI Model"""
 
-
-class OpenAIModelConfiguration(TypedDict):
+    type: Literal["openai"]
+    """The type of the model configuration. Should be 'openai' for OpenAIModelConfiguration"""
     api_key: str
-    base_url: str
-    organization: str
+    "API key needed to make request to model"
+    model: str
+    """Name of model to be used in OpenAI request"""
+    base_url: NotRequired[str]
+    """(Optional) Base URL to be used in OpenAI request"""
+    organization: NotRequired[str]
+    """(Optional) OpenAI organization"""
 
 
 class AzureAIProject(TypedDict):
+    """Azure AI Project Information"""
+
     subscription_id: str
+    """Azure subscription id of the project"""
     resource_group_name: str
+    """Azure resource group name of the project"""
     project_name: str
+    """Azure project name"""
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/conftest.py b/sdk/evaluation/azure-ai-evaluation/tests/conftest.py
@@ -469,7 +469,6 @@ def pytest_collection_modifyitems(items):
 
 
 def pytest_sessionfinish() -> None:
-
     def stop_promptflow_service() -> None:
         """Ensure that the promptflow service is stopped when pytest exits.
 
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_non_adv_simulator.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_non_adv_simulator.py
@@ -37,7 +37,6 @@ def invalid_project():
 
 @pytest.mark.unittest
 class TestNonAdvSimulator:
-
     def test_init_valid_project(self, valid_project):
         simulator = Simulator(azure_ai_project=valid_project)
         assert simulator.azure_ai_project["subscription_id"] == "test_subscription"

Original file line number	Diff line number	Diff line change
`@@ -2,5 +2,5 @@`
`2`	`2`	`"AssetsRepo": "Azure/azure-sdk-assets",`
`3`	`3`	`"AssetsRepoPrefixPath": "python",`
`4`	`4`	`"TagPrefix": "python/evaluation/azure-ai-evaluation",`
`5`		`- "Tag": "python/evaluation/azure-ai-evaluation_9ac3e64c3e"`
	`5`	`+ "Tag": "python/evaluation/azure-ai-evaluation_26cf396fa1"`
`6`	`6`	`}`