Flatten Namespaces for Evaluation SDK (#37398)

* remove public evaluators and evaluate modules

* rename synthetic to simulator and expose only one namespace

* clean up some references

* fix some broken imports

* add details on breaking change

* fix changelog grammar issue

* fix changelog grammar issue

* attempt at fixing tests

* change patch

* disable verifytypes

Author: needuv

sdk/evaluation/azure-ai-evaluation/CHANGELOG.md

@@ -1,3 +1,52 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+
+from ._evaluate._evaluate import evaluate
+from ._evaluators._bleu import BleuScoreEvaluator
+from ._evaluators._chat import ChatEvaluator
+from ._evaluators._coherence import CoherenceEvaluator
+from ._evaluators._content_safety import (
+    ContentSafetyChatEvaluator,
+    ContentSafetyEvaluator,
+    HateUnfairnessEvaluator,
+    SelfHarmEvaluator,
+    SexualEvaluator,
+    ViolenceEvaluator,
+)
+from ._evaluators._f1_score import F1ScoreEvaluator
+from ._evaluators._fluency import FluencyEvaluator
+from ._evaluators._gleu import GleuScoreEvaluator
+from ._evaluators._groundedness import GroundednessEvaluator
+from ._evaluators._meteor import MeteorScoreEvaluator
+from ._evaluators._protected_material import ProtectedMaterialEvaluator
+from ._evaluators._qa import QAEvaluator
+from ._evaluators._relevance import RelevanceEvaluator
+from ._evaluators._rouge import RougeScoreEvaluator, RougeType
+from ._evaluators._similarity import SimilarityEvaluator
+from ._evaluators._xpia import IndirectAttackEvaluator
+
+__all__ = [
+    "evaluate",
+    "CoherenceEvaluator",
+    "F1ScoreEvaluator",
+    "FluencyEvaluator",
+    "GroundednessEvaluator",
+    "RelevanceEvaluator",
+    "SimilarityEvaluator",
+    "QAEvaluator",
+    "ChatEvaluator",
+    "ViolenceEvaluator",
+    "SexualEvaluator",
+    "SelfHarmEvaluator",
+    "HateUnfairnessEvaluator",
+    "ContentSafetyEvaluator",
+    "ContentSafetyChatEvaluator",
+    "IndirectAttackEvaluator",
+    "BleuScoreEvaluator",
+    "GleuScoreEvaluator",
+    "MeteorScoreEvaluator",
+    "RougeScoreEvaluator",
+    "RougeType",
+    "ProtectedMaterialEvaluator",
+]

sdk/evaluation/azure-ai-evaluation/README.md

@@ -1,7 +1,3 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-
-from ._evaluate import evaluate
-
-__all__ = ["evaluate"]

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py

@@ -24,8 +24,8 @@ class BatchRunContext:
 
     :param client: The client to run in the context.
     :type client: Union[
-        ~azure.ai.evaluation.evaluate.code_client.CodeClient,
-        ~azure.ai.evaluation.evaluate.proxy_client.ProxyClient
+        ~azure.ai.evaluation._evaluate._batch_run_client.code_client.CodeClient,
+        ~azure.ai.evaluation._evaluate._batch_run_client.proxy_client.ProxyClient
     ]
     """
 

…azure/ai/evaluation/evaluate/__init__.py …zure/ai/evaluation/_evaluate/__init__.pysdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/evaluate/__init__.py renamed to sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/__init__.py sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/evaluate/__init__.py renamed to sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/__init__.py

@@ -8,7 +8,7 @@
 import pandas as pd
 
 from promptflow.contracts.types import AttrDict
-from azure.ai.evaluation.evaluate._utils import _apply_column_mapping, _has_aggregator, get_int_env_var, load_jsonl
+from azure.ai.evaluation._evaluate._utils import _apply_column_mapping, _has_aggregator, get_int_env_var, load_jsonl
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 
 from ..._constants import PF_BATCH_TIMEOUT_SEC, PF_BATCH_TIMEOUT_SEC_DEFAULT

…n/evaluate/_batch_run_client/__init__.py …/_evaluate/_batch_run_client/__init__.pysdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/evaluate/_batch_run_client/__init__.py renamed to sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run_client/__init__.py sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/evaluate/_batch_run_client/__init__.py renamed to sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run_client/__init__.py

@@ -59,7 +59,7 @@ def generate(run_name: Optional[str]) -> "RunInfo":
         :param run_name: The name of a run.
         :type run_name: Optional[str]
         :return: The RunInfo instance.
-        :rtype: azure.ai.evaluation.evaluate.RunInfo
+        :rtype: azure.ai.evaluation._evaluate._eval_run.RunInfo
         """
         return RunInfo(str(uuid.uuid4()), str(uuid.uuid4()), run_name or "")
 
@@ -229,7 +229,7 @@ def __enter__(self):
         """The Context Manager enter call.
 
         :return: The instance of the class.
-        :rtype: azure.ai.evaluation.evaluate.EvalRun
+        :rtype: azure.ai.evaluation._evaluate._eval_run.EvalRun
         """
         self._start_run()
         return self
@@ -360,7 +360,7 @@ def log_artifact(self, artifact_folder: str, artifact_name: str = EVALUATION_ART
         :param artifact_folder: The folder with artifacts to be uploaded.
         :type artifact_folder: str
         :param artifact_name: The name of the artifact to be uploaded. Defaults to
-            azure.ai.evaluation.evaluate.EvalRun.EVALUATION_ARTIFACT.
+            azure.ai.evaluation._evaluate._eval_run.EvalRun.EVALUATION_ARTIFACT.
         :type artifact_name: str
         """
         if not self._check_state_and_log("log artifact", {RunStatus.BROKEN, RunStatus.NOT_STARTED}, False):

…e/_batch_run_client/batch_run_context.py …e/_batch_run_client/batch_run_context.pysdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/evaluate/_batch_run_client/batch_run_context.py renamed to sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run_client/batch_run_context.py sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/evaluate/_batch_run_client/batch_run_context.py renamed to sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run_client/batch_run_context.py

@@ -60,7 +60,7 @@ def _aggregate_content_safety_metrics(
             module = inspect.getmodule(evaluators[evaluator_name])
             if (
                 module
-                and module.__name__.startswith("azure.ai.evaluation.evaluators.")
+                and module.__name__.startswith("azure.ai.evaluation.")
                 and metric_name.endswith("_score")
                 and metric_name.replace("_score", "") in content_safety_metrics
             ):
@@ -397,8 +397,7 @@ def evaluate(
     .. code-block:: python
 
             from promptflow.core import AzureOpenAIModelConfiguration
-            from azure.ai.evaluation.evaluate import evaluate
-            from azure.ai.evaluation.evaluators import RelevanceEvaluator, CoherenceEvaluator
+            from azure.ai.evaluation import evaluate, RelevanceEvaluator, CoherenceEvaluator
 
 
             model_config = AzureOpenAIModelConfiguration(