Placeholder Ids for evaluators (Azure#38330)

singankit · web-flow · commit 9e3b0e97cc1a · 2024-11-06T19:40:49.000Z
* Placeholer Ids for evaluators

* Adding id field

* Adding experimental decorator

* Trying out doc string

* Updating evaluator ids

* Fxing tests

* Removing unused imports

* Removing unused imports

* Deleting unused code

* Fixing formatting issues
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_bleu/_bleu.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_bleu/_bleu.py
@@ -47,6 +47,9 @@ class BleuScoreEvaluator:
             :caption: Initialize and call an BleuScoreEvaluator.
     """
 
+    id = "bleu"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self):
         self._async_evaluator = _AsyncBleuScoreEvaluator()
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_coherence/_coherence.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_coherence/_coherence.py
@@ -41,6 +41,9 @@ class CoherenceEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _PROMPTY_FILE = "coherence.prompty"
     _RESULT_KEY = "coherence"
 
+    id = "coherence"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(self, model_config):
         current_dir = os.path.dirname(__file__)
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py
@@ -7,9 +7,9 @@
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
 from typing_extensions import overload, override
 
-from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._evaluators._common import EvaluatorBase
 from azure.ai.evaluation._model_configurations import Conversation
+from azure.ai.evaluation._common._experimental import experimental
 
 from ._hate_unfairness import HateUnfairnessEvaluator
 from ._self_harm import SelfHarmEvaluator
@@ -41,6 +41,9 @@ class ContentSafetyEvaluator(EvaluatorBase[Union[str, float]]):
             :caption: Initialize and call a ContentSafetyEvaluator.
     """
 
+    id = "content_safety"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     # TODO address 3579092 to re-enabled parallel evals.
     def __init__(self, credential, azure_ai_project, **kwargs):
         super().__init__()
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py
@@ -58,6 +58,9 @@ class HateUnfairnessEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :caption: Initialize and call a HateUnfairnessEvaluator.
     """
 
+    id = "hate_unfairness"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(
         self,
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py
@@ -52,6 +52,9 @@ class SelfHarmEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :caption: Initialize and call a SelfHarmEvaluator.
     """
 
+    id = "self_harm"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(
         self,
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py
@@ -54,6 +54,9 @@ class SexualEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :caption: Initialize and call a SexualEvaluator.
     """
 
+    id = "sexual"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(
         self,
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py
@@ -54,6 +54,9 @@ class ViolenceEvaluator(RaiServiceEvaluatorBase[Union[str, float]]):
             :caption: Initialize and call a ViolenceEvaluator.
     """
 
+    id = "violence"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(
         self,
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_eci/_eci.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_eci/_eci.py
@@ -50,6 +50,9 @@ class ECIEvaluator(RaiServiceEvaluatorBase):
         }
     """
 
+    id = "eci"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(
         self,
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_f1_score/_f1_score.py
@@ -131,6 +131,9 @@ class F1ScoreEvaluator:
             :caption: Initialize and call an F1ScoreEvaluator.
     """
 
+    id = "f1_score"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self):
         self._async_evaluator = _AsyncF1ScoreEvaluator()
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_fluency/_fluency.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_fluency/_fluency.py
@@ -43,6 +43,9 @@ class FluencyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _PROMPTY_FILE = "fluency.prompty"
     _RESULT_KEY = "fluency"
 
+    id = "fluency"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(self, model_config):
         current_dir = os.path.dirname(__file__)
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_gleu/_gleu.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_gleu/_gleu.py
@@ -44,6 +44,9 @@ class GleuScoreEvaluator:
             :caption: Initialize and call a GleuScoreEvaluator.
     """
 
+    id = "gleu_score"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self):
         self._async_evaluator = _AsyncGleuScoreEvaluator()
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py
@@ -55,6 +55,9 @@ class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     _RESULT_KEY = "groundedness"
     _OPTIONAL_PARAMS = ["query"]
 
+    id = "groundedness"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(self, model_config):
         current_dir = os.path.dirname(__file__)
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_meteor/_meteor.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_meteor/_meteor.py
@@ -65,6 +65,9 @@ class MeteorScoreEvaluator:
             :caption: Initialize and call a MeteorScoreEvaluator with alpha of 0.8.
     """
 
+    id = "meteor"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, alpha: float = 0.9, beta: float = 3.0, gamma: float = 0.5):
         self._async_evaluator = _AsyncMeteorScoreEvaluator(alpha=alpha, beta=beta, gamma=gamma)
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_content_safety_multimodal.py
@@ -90,6 +90,9 @@ class ContentSafetyMultimodalEvaluator:
 
     """
 
+    id = "content_safety_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, credential, azure_ai_project, **kwargs):
         self._parallel = kwargs.pop("_parallel", False)
         self._evaluators: List[Callable[..., Dict[str, Union[str, float]]]] = [
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_hate_unfairness.py
@@ -66,6 +66,9 @@ class HateUnfairnessMultimodalEvaluator:
 
     """
 
+    id = "hate_unfairness_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, credential, azure_ai_project):
         self._async_evaluator = _AsyncHateUnfairnessMultimodalEvaluator(credential, azure_ai_project)
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_protected_material.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_protected_material.py
@@ -2,10 +2,10 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._common._experimental import experimental
 from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._common.utils import validate_conversation
 from azure.ai.evaluation._common.rai_service import evaluate_with_rai_service_multimodal
+from azure.ai.evaluation._common._experimental import experimental
 
 
 @experimental
@@ -70,6 +70,9 @@ class ProtectedMaterialMultimodalEvaluator:
 
     """
 
+    id = "protected_material_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(
         self,
         credential,
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_self_harm.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_self_harm.py
@@ -66,6 +66,9 @@ class SelfHarmMultimodalEvaluator:
 
     """
 
+    id = "self_harm_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, credential, azure_ai_project):
         self._async_evaluator = _AsyncSelfHarmMultimodalEvaluator(credential, azure_ai_project)
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_sexual.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_sexual.py
@@ -66,6 +66,9 @@ class SexualMultimodalEvaluator:
 
     """
 
+    id = "sexual_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, credential, azure_ai_project):
         self._async_evaluator = _AsyncSexualMultimodalEvaluator(credential, azure_ai_project)
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_violence.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_multimodal/_violence.py
@@ -66,6 +66,9 @@ class ViolenceMultimodalEvaluator:
 
     """
 
+    id = "violence_multimodal"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, credential, azure_ai_project):
         self._async_evaluator = _AsyncViolenceMultimodalEvaluator(credential, azure_ai_project)
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py
@@ -39,6 +39,9 @@ class ProtectedMaterialEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
             :caption: Initialize and call a ProtectedMaterialEvaluator.
     """
 
+    id = "protected_material"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(
         self,
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_qa/_qa.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_qa/_qa.py
@@ -42,6 +42,9 @@ class QAEvaluator:
         however, it is recommended to use the new keys moving forward as the old keys will be deprecated in the future.
     """
 
+    id = "qa"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, model_config, **kwargs):
         self._parallel = kwargs.pop("_parallel", False)
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_relevance/_relevance.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_relevance/_relevance.py
@@ -48,6 +48,9 @@ class RelevanceEvaluator(PromptyEvaluatorBase):
     _PROMPTY_FILE = "relevance.prompty"
     _RESULT_KEY = "relevance"
 
+    id = "relevance"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(self, model_config):
         current_dir = os.path.dirname(__file__)
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py
@@ -147,6 +147,9 @@ class RetrievalEvaluator(PromptyEvaluatorBase[Union[str, float]]):
         however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
+    id = "retrieval"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, model_config):  # pylint: disable=super-init-not-called
         self._async_evaluator = _AsyncRetrievalScoreEvaluator(model_config)
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_rouge/_rouge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_rouge/_rouge.py
@@ -73,6 +73,9 @@ class RougeScoreEvaluator:
             :caption: Initialize and call a RougeScoreEvaluator with a four-gram rouge type.
     """
 
+    id = "rouge"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, rouge_type: RougeType):
         self._async_evaluator = _AsyncRougeScoreEvaluator(rouge_type)
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_service_groundedness/_service_groundedness.py
@@ -45,6 +45,9 @@ class GroundednessProEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
         for the groundedness pro label will be "groundedness_pro_passing_rate".
     """
 
+    id = "groundedness_pro"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(
         self,
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_similarity/_similarity.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_similarity/_similarity.py
@@ -113,6 +113,9 @@ class SimilarityEvaluator:
         however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
+    id = "similarity"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     def __init__(self, model_config):
         self._async_evaluator = _AsyncSimilarityEvaluator(model_config)
 
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_xpia/xpia.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_xpia/xpia.py
@@ -54,6 +54,9 @@ class IndirectAttackEvaluator(RaiServiceEvaluatorBase[Union[str, bool]]):
             :caption: Initialize and call an IndirectAttackEvaluator.
     """
 
+    id = "xpia"
+    """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
+
     @override
     def __init__(
         self,
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluator_ids.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluator_ids.py
@@ -0,0 +1,71 @@
+import pytest
+
+from azure.ai.evaluation import (
+    CoherenceEvaluator,
+    F1ScoreEvaluator,
+    FluencyEvaluator,
+    F1ScoreEvaluator,
+    GroundednessEvaluator,
+    RelevanceEvaluator,
+    SimilarityEvaluator,
+    QAEvaluator,
+    ViolenceEvaluator,
+    SexualEvaluator,
+    HateUnfairnessEvaluator,
+    SelfHarmEvaluator,
+    ContentSafetyEvaluator,
+    IndirectAttackEvaluator,
+    BleuScoreEvaluator,
+    GleuScoreEvaluator,
+    MeteorScoreEvaluator,
+    RetrievalEvaluator,
+    RougeScoreEvaluator,
+    RougeType,
+    ProtectedMaterialEvaluator,
+    HateUnfairnessMultimodalEvaluator,
+    SelfHarmMultimodalEvaluator,
+    ViolenceMultimodalEvaluator,
+    SexualMultimodalEvaluator,
+    IndirectAttackEvaluator,
+    GroundednessProEvaluator,
+    ProtectedMaterialMultimodalEvaluator,
+    RetrievalEvaluator,
+)
+
+
+# Evaluators are dummy ids for testing purposes to be replaced once the actual ids are available
+class TestEvaluatorIds:
+
+    def test_content_safety_evaluator_ids(self):
+        assert ViolenceEvaluator.id == "violence"
+        assert SexualEvaluator.id == "sexual"
+        assert SelfHarmEvaluator.id == "self_harm"
+        assert HateUnfairnessEvaluator.id == "hate_unfairness"
+
+    def test_content_safety_multimodal_evaluator_ids(self):
+        assert ViolenceMultimodalEvaluator.id == "violence_multimodal"
+        assert SexualMultimodalEvaluator.id == "sexual_multimodal"
+        assert SelfHarmMultimodalEvaluator.id == "self_harm_multimodal"
+        assert HateUnfairnessMultimodalEvaluator.id == "hate_unfairness_multimodal"
+        assert ProtectedMaterialMultimodalEvaluator.id == "protected_material_multimodal"
+
+    def test_service_based_evaluator_ids(self):
+        assert GroundednessProEvaluator.id == "groundedness_pro"
+        assert IndirectAttackEvaluator.id == "xpia"
+        assert ProtectedMaterialEvaluator.id == "protected_material"
+
+    def test_quality_evaluator_ids(self):
+        assert RelevanceEvaluator.id == "relevance"
+        assert CoherenceEvaluator.id == "coherence"
+        assert FluencyEvaluator.id == "fluency"
+        assert SimilarityEvaluator.id == "similarity"
+        assert QAEvaluator.id == "qa"
+        assert GroundednessEvaluator.id == "groundedness"
+        assert RetrievalEvaluator.id == "retrieval"
+
+    def test_mathematical_evaluator_ids(self):
+        assert BleuScoreEvaluator.id == "bleu"
+        assert GleuScoreEvaluator.id == "gleu_score"
+        assert MeteorScoreEvaluator.id == "meteor"
+        assert RougeScoreEvaluator.id == "rouge"
+        assert F1ScoreEvaluator.id == "f1_score"

Original file line number	Diff line number	Diff line change
`@@ -50,6 +50,9 @@ class ECIEvaluator(RaiServiceEvaluatorBase):`
`50`	`50`	`}`
`51`	`51`	`"""`
`52`	`52`
	`53`	`+ id = "eci"`
	`54`	`+ """Evaluator identifier, experimental and to be used only with evaluation in cloud."""`
	`55`	`+`
`53`	`56`	`@override`
`54`	`57`	`def __init__(`
`55`	`58`	`self,`