Change Model Configurations to TypedDict (#37429)

* remove public evaluators and evaluate modules

* rename synthetic to simulator and expose only one namespace

* clean up some references

* fix some broken imports

* add details on breaking change

* fix changelog grammar issue

* fix changelog grammar issue

* attempt at fixing tests

* change patch

* disable verifytypes

* start work on adding model config classes

* add model configuration classes

* change type hint for azure_ai_project to AzureAIProject typeddict

* convert model configs to dictionaries

* update readme and changelog

* fix spell check issue

Author: needuv

sdk/evaluation/azure-ai-evaluation/CHANGELOG.md

@@ -7,10 +7,12 @@
 - The `synthetic` namespace has been renamed to `simulator`, and sub-namespaces under this module have been removed
 - The `evaluate` and `evaluators` namespaces have been removed, and everything previously exposed in those modules has been added to the root namespace `azure.ai.evaluation`  
 - The parameter name `project_scope` in content safety evaluators have been renamed to `azure_ai_project` for consistency with evaluate API and simulators.
+- Model configurations classes are now of type `TypedDict` and are exposed in the `azure.ai.evaluation` module instead of coming from `promptflow.core`.  
 - Updated the parameter names for `question` and `answer` in built-in evaluators to more generic terms: `query` and `response`.
 
 
 ### Features Added
 
 - First preview
 - This package is port of `promptflow-evals`. New features will be added only to this package moving forward.
+- Added a `TypedDict` for `AzureAIProject` that allows for better intellisense and type checking when passing in project information

sdk/evaluation/azure-ai-evaluation/README.md

@@ -8,7 +8,6 @@ Install the Azure AI Evaluation library for Python with:
 
 ```bash
 pip install azure-ai-evaluation
-pip install azure-identity
 ```
 
 ## Key concepts
@@ -23,8 +22,6 @@ Users can create evaluator runs on the local machine as shown in the example bel
 import os
 from pprint import pprint
 
-from promptflow.core import AzureOpenAIModelConfiguration
-
 from azure.ai.evaluation import evaluate, RelevanceEvaluator, ViolenceEvaluator
 
 
@@ -34,12 +31,12 @@ def response_length(response, **kwargs):
 
 if __name__ == "__main__":
     # Built-in evaluators
-    # Initialize Azure OpenAI Connection
-    model_config = AzureOpenAIModelConfiguration(
-        azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
-        api_key=os.environ.get("AZURE_OPENAI_KEY"),
-        azure_deployment=os.environ.get("AZURE_OPENAI_DEPLOYMENT"),
-    )
+    # Initialize Azure OpenAI Model Configuration
+    model_config = {
+        "azure_endpoint": os.environ.get("AZURE_OPENAI_ENDPOINT"),
+        "api_key": os.environ.get("AZURE_OPENAI_KEY"),
+        "azure_deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT"),
+    }
 
     # Initialzing Relevance Evaluator
     relevance_eval = RelevanceEvaluator(model_config)
@@ -91,8 +88,8 @@ if __name__ == "__main__":
     pprint(result)
 ```
 
-Simulator expects the user to have a callback method that invokes their AI application.
-Here's a sample of a callback which invokes AsyncAzureOpenAI:
+Simulators allow users to generate synthentic data using their application. Simulator expects the user to have a callback method that invokes
+their AI application. Here's a sample of a callback which invokes AsyncAzureOpenAI:
 
 ```python
 from from azure.ai.evaluation.simulator import AdversarialSimulator, AdversarialScenario

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py

@@ -25,6 +25,11 @@
 from ._evaluators._rouge import RougeScoreEvaluator, RougeType
 from ._evaluators._similarity import SimilarityEvaluator
 from ._evaluators._xpia import IndirectAttackEvaluator
+from ._model_configurations import (
+    AzureAIProject,
+    AzureOpenAIModelConfiguration,
+    OpenAIModelConfiguration,
+)
 
 __all__ = [
     "evaluate",
@@ -49,4 +54,7 @@
     "RougeScoreEvaluator",
     "RougeType",
     "ProtectedMaterialEvaluator",
+    "AzureAIProject",
+    "AzureOpenAIModelConfiguration",
+    "OpenAIModelConfiguration",
 ]

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/rai_service.py

@@ -15,6 +15,7 @@
 from azure.identity import DefaultAzureCredential
 
 from azure.ai.evaluation._http_utils import get_async_http_client
+from azure.ai.evaluation._model_configurations import AzureAIProject
 
 from .constants import (
     CommonConstants,
@@ -327,11 +328,11 @@ def _parse_content_harm_response(batch_response: List[Dict], metric_name: str) -
     return result
 
 
-async def _get_service_discovery_url(azure_ai_project: dict, token: str) -> str:
+async def _get_service_discovery_url(azure_ai_project: AzureAIProject, token: str) -> str:
     """Get the discovery service URL for the Azure AI project
 
     :param azure_ai_project: The Azure AI project details.
-    :type azure_ai_project: Dict
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param token: The Azure authentication token.
     :type token: str
     :return: The discovery service URL.
@@ -410,7 +411,7 @@ async def fetch_or_reuse_token(credential: TokenCredential, token: str = None) -
 
 
 async def evaluate_with_rai_service(
-    query: str, response: str, metric_name: str, project_scope: dict, credential: TokenCredential
+    query: str, response: str, metric_name: str, project_scope: AzureAIProject, credential: TokenCredential
 ):
     """ "Evaluate the content safety of the response using Responsible AI service
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py

@@ -1,6 +1,11 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+
+from typing import Optional, Union
+
+from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+
 try:
     from . import constants
 except ImportError:
@@ -54,3 +59,22 @@ def nltk_tokenize(text: str) -> List[str]:
         tokens = nltk.word_tokenize(text)
 
     return list(tokens)
+
+
+def check_and_add_api_version_for_aoai_model_config(
+    model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+    default_api_version: str,
+) -> None:
+    if (
+        "azure_endpoint" in model_config or "azure_deployment" in model_config
+    ):
+        model_config["api_version"] = model_config.get("api_version", default_api_version)
+
+
+def check_and_add_user_agent_for_aoai_model_config(
+    model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+    prompty_model_config: dict,
+    user_agent: Optional[str] = None,
+) -> None:
+    if user_agent and ("azure_endpoint" in model_config or "azure_deployment" in model_config):
+        prompty_model_config["parameters"]["extra_headers"].update({"x-ms-useragent": user_agent})

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py

@@ -12,6 +12,7 @@
 from promptflow._sdk._constants import LINE_NUMBER
 from promptflow.client import PFClient
 
+from .._model_configurations import AzureAIProject
 from .._constants import (
     CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT,
     EvaluationMetrics,
@@ -360,7 +361,7 @@ def evaluate(
     data: Optional[str] = None,
     evaluators: Optional[Dict[str, Callable]] = None,
     evaluator_config: Optional[Dict[str, Dict[str, str]]] = None,
-    azure_ai_project: Optional[Dict] = None,
+    azure_ai_project: Optional[AzureAIProject] = None,
     output_path: Optional[str] = None,
     **kwargs,
 ):
@@ -386,7 +387,7 @@ def evaluate(
           the results will be saved to a file named `evaluation_results.json` in the folder.
     :paramtype output_path: Optional[str]
     :keyword azure_ai_project: Logs evaluation results to AI Studio if set.
-    :paramtype azure_ai_project: Optional[Dict]
+    :paramtype azure_ai_project: Optional[~azure.ai.evaluation.AzureAIProject]
     :return: Evaluation results.
     :rtype: dict
 
@@ -396,15 +397,14 @@ def evaluate(
 
     .. code-block:: python
 
-            from promptflow.core import AzureOpenAIModelConfiguration
             from azure.ai.evaluation import evaluate, RelevanceEvaluator, CoherenceEvaluator
 
 
-            model_config = AzureOpenAIModelConfiguration(
-                azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
-                api_key=os.environ.get("AZURE_OPENAI_KEY"),
-                azure_deployment=os.environ.get("AZURE_OPENAI_DEPLOYMENT")
-            )
+            model_config = {
+                "azure_endpoint": os.environ.get("AZURE_OPENAI_ENDPOINT"),
+                "api_key": os.environ.get("AZURE_OPENAI_KEY"),
+                "azure_deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT")
+            }
 
             coherence_eval = CoherenceEvaluator(model_config=model_config)
             relevance_eval = RelevanceEvaluator(model_config=model_config)
@@ -466,7 +466,7 @@ def _evaluate(  # pylint: disable=too-many-locals
     data: Optional[str] = None,
     evaluators: Optional[Dict[str, Callable]] = None,
     evaluator_config: Optional[Dict[str, Dict[str, str]]] = None,
-    azure_ai_project: Optional[Dict] = None,
+    azure_ai_project: Optional[AzureAIProject] = None,
     output_path: Optional[str] = None,
     **kwargs,
 ):

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_chat/_chat.py

@@ -8,12 +8,12 @@
 
 import numpy as np
 
-from promptflow.core import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 
 from .._coherence import CoherenceEvaluator
 from .._fluency import FluencyEvaluator
 from .._groundedness import GroundednessEvaluator
+from ..._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 from .._relevance import RelevanceEvaluator
 from .retrieval import RetrievalChatEvaluator
 
@@ -25,8 +25,8 @@ class ChatEvaluator:
     Initialize a chat evaluator configured for a specific Azure OpenAI model.
 
     :param model_config: Configuration for the Azure OpenAI model.
-    :type model_config: Union[~promptflow.core.AzureOpenAIModelConfiguration,
-        ~promptflow.core.OpenAIModelConfiguration]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
     :param eval_last_turn: Set to True to evaluate only the most recent exchange in the dialogue,
         focusing on the latest user inquiry and the assistant's corresponding response. Defaults to False
     :type eval_last_turn: bool
@@ -46,7 +46,7 @@ class ChatEvaluator:
             {"role": "assistant", "content": "2 + 2 = 4", "context": {
                 "citations": [
                         {"id": "math_doc.md", "content": "Information about additions: 1 + 2 = 3, 2 + 2 = 4"}
-                        ]
+                    ]
                 }
             }
         ]
@@ -74,7 +74,7 @@ class ChatEvaluator:
 
     def __init__(
         self,
-        model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration],
+        model_config: dict,
         eval_last_turn: bool = False,
         parallel: bool = True,
     ):

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_chat/retrieval/_retrieval.py

@@ -11,7 +11,13 @@
 import numpy as np
 
 from promptflow._utils.async_utils import async_run_allowing_running_loop
-from promptflow.core import AsyncPrompty, AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from promptflow.core import AsyncPrompty
+
+from ...._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from ...._common.utils import (
+    check_and_add_api_version_for_aoai_model_config,
+    check_and_add_user_agent_for_aoai_model_config,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -27,21 +33,20 @@ class _AsyncRetrievalChatEvaluator:
     LLM_CALL_TIMEOUT = 600
     DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
-    def __init__(self, model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]):
-        if (
-            isinstance(model_config, AzureOpenAIModelConfiguration)
-            and (not hasattr(model_config, "api_version") or model_config.api_version) is None
-        ):
-            model_config.api_version = self.DEFAULT_OPEN_API_VERSION
+    def __init__(self, model_config: dict):
+        check_and_add_api_version_for_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
 
         prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
 
         # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
         # https://github.com/encode/httpx/discussions/2959
         prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
 
-        if USER_AGENT and isinstance(model_config, AzureOpenAIModelConfiguration):
-            prompty_model_config["parameters"]["extra_headers"].update({"x-ms-useragent": USER_AGENT})
+        check_and_add_user_agent_for_aoai_model_config(
+            model_config,
+            prompty_model_config,
+            USER_AGENT,
+        )
 
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
@@ -107,8 +112,8 @@ class RetrievalChatEvaluator:
     Initialize an evaluator configured for a specific Azure OpenAI model.
 
     :param model_config: Configuration for the Azure OpenAI model.
-    :type model_config: Union[~promptflow.core.AzureOpenAIModelConfiguration,
-        ~promptflow.core.OpenAIModelConfiguration]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
     :return: A function that evaluates and generates metrics for "chat" scenario.
     :rtype: Callable
     **Usage**
@@ -141,7 +146,7 @@ class RetrievalChatEvaluator:
     }
     """
 
-    def __init__(self, model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]):
+    def __init__(self, model_config: dict):
         self._async_evaluator = _AsyncRetrievalChatEvaluator(model_config)
 
     def __call__(self, *, conversation, **kwargs):

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_coherence/_coherence.py

@@ -9,7 +9,13 @@
 import numpy as np
 
 from promptflow._utils.async_utils import async_run_allowing_running_loop
-from promptflow.core import AsyncPrompty, AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from promptflow.core import AsyncPrompty
+
+from ..._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
+from ..._common.utils import (
+    check_and_add_api_version_for_aoai_model_config,
+    check_and_add_user_agent_for_aoai_model_config,
+)
 
 try:
     from ..._user_agent import USER_AGENT
@@ -23,21 +29,20 @@ class _AsyncCoherenceEvaluator:
     LLM_CALL_TIMEOUT = 600
     DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
-    def __init__(self, model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]):
-        if (
-            isinstance(model_config, AzureOpenAIModelConfiguration)
-            and (not hasattr(model_config, "api_version") or model_config.api_version) is None
-        ):
-            model_config.api_version = self.DEFAULT_OPEN_API_VERSION
+    def __init__(self, model_config: dict):
+        check_and_add_api_version_for_aoai_model_config(model_config, self.DEFAULT_OPEN_API_VERSION)
 
         prompty_model_config = {"configuration": model_config, "parameters": {"extra_headers": {}}}
 
         # Handle "RuntimeError: Event loop is closed" from httpx AsyncClient
         # https://github.com/encode/httpx/discussions/2959
         prompty_model_config["parameters"]["extra_headers"].update({"Connection": "close"})
 
-        if USER_AGENT and isinstance(model_config, AzureOpenAIModelConfiguration):
-            prompty_model_config["parameters"]["extra_headers"].update({"x-ms-useragent": USER_AGENT})
+        check_and_add_user_agent_for_aoai_model_config(
+            model_config,
+            prompty_model_config,
+            USER_AGENT,
+        )
 
         current_dir = os.path.dirname(__file__)
         prompty_path = os.path.join(current_dir, self.PROMPTY_FILE)
@@ -68,8 +73,8 @@ class CoherenceEvaluator:
     Initialize a coherence evaluator configured for a specific Azure OpenAI model.
 
     :param model_config: Configuration for the Azure OpenAI model.
-    :type model_config: Union[~promptflow.core.AzureOpenAIModelConfiguration,
-        ~promptflow.core.OpenAIModelConfiguration]
+    :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration,
+        ~azure.ai.evaluation.OpenAIModelConfiguration]
 
     **Usage**
 
@@ -89,7 +94,7 @@ class CoherenceEvaluator:
         }
     """
 
-    def __init__(self, model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]):
+    def __init__(self, model_config: dict):
         self._async_evaluator = _AsyncCoherenceEvaluator(model_config)
 
     def __call__(self, *, query: str, response: str, **kwargs):

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py

@@ -5,6 +5,8 @@
 
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 
+from azure.ai.evaluation._model_configurations import AzureAIProject
+
 try:
     from ._hate_unfairness import HateUnfairnessEvaluator
     from ._self_harm import SelfHarmEvaluator
@@ -23,7 +25,7 @@ class ContentSafetyEvaluator:
 
     :param azure_ai_project: The scope of the Azure AI project.
         It contains subscription id, resource group, and project name.
-    :type azure_ai_project: dict
+    :type azure_ai_project: ~azure.ai.evaluation.AzureAIProject
     :param parallel: If True, use parallel execution for evaluators. Else, use sequential execution.
         Default is True.
     :param credential: The credential for connecting to Azure AI project.