Improve e2e evaluator testing. (Azure#38758)

* enable glue

* add similarity to e2e testing and refactor to use base class

* analysis and unit test fixes

* run black

* re-enabled no PF QA and image multi

* run black

* re record new tests

* undo image with targets

* run black

* undo not using pf client

* remove qa again

* run black

* lowere PF performance reqs

* update recordings

Author: MilesHolland

sdk/evaluation/azure-ai-evaluation/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/evaluation/azure-ai-evaluation",
-  "Tag": "python/evaluation/azure-ai-evaluation_5ad4de0f7c"
+  "Tag": "python/evaluation/azure-ai-evaluation_4f3f9f39dc"
 }

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_eval.py

@@ -25,6 +25,7 @@ class DerivedEvalInput(TypedDict, total=False):
     query: Dict[str, Any]
     response: Dict[str, Any]
     context: str
+    ground_truth: str
 
 
 AggregateResult: TypeAlias = Dict[str, Union[float, Dict[str, List[T]]]]
@@ -158,6 +159,7 @@ def _derive_conversation_converter(self) -> Callable[[Dict], List[DerivedEvalInp
         include_context = "context" in self._singleton_inputs
         include_query = "query" in self._singleton_inputs
         include_response = "response" in self._singleton_inputs
+        include_ground_truth = "ground_truth" in self._singleton_inputs
 
         def converter(conversation: Dict) -> List[DerivedEvalInput]:
             messages = cast(List[Dict[str, Any]], conversation["messages"])
@@ -198,6 +200,8 @@ def converter(conversation: Dict) -> List[DerivedEvalInput]:
                     eval_input["response"] = response.get("content", "")
                 if include_context:
                     eval_input["context"] = str(context)
+                if include_ground_truth:
+                    eval_input["ground_truth"] = response.get("ground_truth", "")
                 eval_inputs.append(eval_input)
             return eval_inputs
 
@@ -402,7 +406,9 @@ def __init__(self, real_call):  # DO NOT ADD TYPEHINT PROMPT FLOW WILL SCREAM AT
     # are just not passed into this function instead of ending up in kwargs.
     # Since we want this to be relatively call-agnostic, we just account for every input that any children
     # are known to throw at this, mash them into kwargs, and then pass them into the real call.
-    async def __call__(self, *, query=None, response=None, context=None, conversation=None, **kwargs):
+    async def __call__(
+        self, *, query=None, response=None, context=None, conversation=None, ground_truth=None, **kwargs
+    ):
         if conversation is not None:
             kwargs["conversation"] = conversation
         if query is not None:
@@ -411,4 +417,6 @@ async def __call__(self, *, query=None, response=None, context=None, conversatio
             kwargs["response"] = response
         if context is not None:
             kwargs["context"] = context
+        if ground_truth is not None:
+            kwargs["ground_truth"] = ground_truth
         return await self._real_call(**kwargs)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_similarity/_similarity.py

@@ -2,83 +2,15 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-import math
 import os
-import re
+from typing import Dict
 
-from promptflow._utils.async_utils import async_run_allowing_running_loop
-from promptflow.core import AsyncPrompty
+from typing_extensions import overload, override
 
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 
-from ..._common.utils import construct_prompty_model_config, validate_model_config
 
-try:
-    from ..._user_agent import USER_AGENT
-except ImportError:
-    USER_AGENT = "None"
-
-
-class _AsyncSimilarityEvaluator:
-    # Constants must be defined within eval's directory to be save/loadable
-    _PROMPTY_FILE = "similarity.prompty"
-    _LLM_CALL_TIMEOUT = 600
-    _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
-
-    def __init__(self, model_config: dict):
-        prompty_model_config = construct_prompty_model_config(
-            validate_model_config(model_config),
-            self._DEFAULT_OPEN_API_VERSION,
-            USER_AGENT,
-        )
-
-        current_dir = os.path.dirname(__file__)
-        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
-        self._flow = AsyncPrompty.load(source=prompty_path, model=prompty_model_config)
-
-    async def __call__(self, *, query: str, response: str, ground_truth: str, **kwargs):
-        """
-        Evaluate similarity.
-
-        :keyword query: The query to be evaluated.
-        :paramtype query: str
-        :keyword response: The response to be evaluated.
-        :paramtype response: str
-        :keyword ground_truth: The ground truth to be evaluated.
-        :paramtype ground_truth: str
-        :return: The similarity score.
-        :rtype: Dict[str, float]
-        """
-        # Validate input parameters
-        query = str(query or "")
-        response = str(response or "")
-        ground_truth = str(ground_truth or "")
-
-        if not (query.strip() and response.strip() and ground_truth.strip()):
-            msg = "'query', 'response' and 'ground_truth' must be non-empty strings."
-            raise EvaluationException(
-                message=msg,
-                internal_message=msg,
-                error_category=ErrorCategory.MISSING_FIELD,
-                error_blame=ErrorBlame.USER_ERROR,
-                error_target=ErrorTarget.SIMILARITY_EVALUATOR,
-            )
-
-        # Run the evaluation flow
-        llm_output = await self._flow(
-            query=query, response=response, ground_truth=ground_truth, timeout=self._LLM_CALL_TIMEOUT, **kwargs
-        )
-
-        score = math.nan
-        if llm_output:
-            match = re.search(r"\d", llm_output)
-            if match:
-                score = float(match.group())
-
-        return {"similarity": float(score), "gpt_similarity": float(score)}
-
-
-class SimilarityEvaluator:
+class SimilarityEvaluator(PromptyEvaluatorBase):
     """
     Evaluates similarity score for a given query, response, and ground truth.
 
@@ -113,13 +45,27 @@ class SimilarityEvaluator:
         however, it is recommended to use the new key moving forward as the old key will be deprecated in the future.
     """
 
-    id = "azureml://registries/azureml/models/Similarity-Evaluator/versions/3"
+    # Constants must be defined within eval's directory to be save/loadable
+
+    _PROMPTY_FILE = "similarity.prompty"
+    _RESULT_KEY = "similarity"
+
+    id = "similarity"
     """Evaluator identifier, experimental and to be used only with evaluation in cloud."""
 
+    @override
     def __init__(self, model_config):
-        self._async_evaluator = _AsyncSimilarityEvaluator(model_config)
-
-    def __call__(self, *, query: str, response: str, ground_truth: str, **kwargs):
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, self._PROMPTY_FILE)
+        super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY)
+
+    # Ignoring a mypy error about having only 1 overload function.
+    # We want to use the overload style for all evals, even single-inputs. This is both to make
+    # refactoring to multi-input styles easier, stylistic consistency consistency across evals,
+    # and due to the fact that non-overloaded syntax now causes various parsing issues that
+    # we don't want to deal with.
+    @overload  # type: ignore
+    def __call__(self, *, query: str, response: str, ground_truth: str) -> Dict[str, float]:
         """
         Evaluate similarity.
 
@@ -132,9 +78,23 @@ def __call__(self, *, query: str, response: str, ground_truth: str, **kwargs):
         :return: The similarity score.
         :rtype: Dict[str, float]
         """
-        return async_run_allowing_running_loop(
-            self._async_evaluator, query=query, response=response, ground_truth=ground_truth, **kwargs
-        )
 
-    def _to_async(self):
-        return self._async_evaluator
+    @override
+    def __call__(  # pylint: disable=docstring-missing-param
+        self,
+        *args,
+        **kwargs,
+    ):
+        """
+        Evaluate similarity.
+
+        :keyword query: The query to be evaluated.
+        :paramtype query: str
+        :keyword response: The response to be evaluated.
+        :paramtype response: str
+        :keyword ground_truth: The ground truth to be evaluated.
+        :paramtype ground_truth: str
+        :return: The similarity score.
+        :rtype: Dict[str, float]
+        """
+        return super().__call__(*args, **kwargs)

sdk/evaluation/azure-ai-evaluation/tests/e2etests/test_mass_evaluate.py

@@ -5,18 +5,19 @@
 import pathlib
 import pandas as pd
 import pytest
+from regex import F
 
 
 from azure.ai.evaluation import (
     F1ScoreEvaluator,
-    # GleuScoreEvaluator,
+    GleuScoreEvaluator,
     BleuScoreEvaluator,
     RougeScoreEvaluator,
     MeteorScoreEvaluator,
     CoherenceEvaluator,
     FluencyEvaluator,
     RelevanceEvaluator,
-    # SimilarityEvaluator,
+    SimilarityEvaluator,
     GroundednessEvaluator,
     # QAEvaluator,
     ContentSafetyEvaluator,
@@ -74,21 +75,20 @@ class TestMassEvaluate:
     """
 
     def test_evaluate_singleton_inputs(self, model_config, azure_cred, project_scope, data_file):
-        # qa and similarity disabled due to being playback-unfriendly due to URL sanitization problems.
-        # glue disabled due to being unfriendly to CI playback for some reason.
-        # content safety disabled temporarily to test CI PF teardown race condition
+        # qa fails in playback but ONLY when using the pf proxy for some reason, and
+        # using it without pf proxy causes CI to hang and timeout after 3 hours.
         evaluators = {
             "f1_score": F1ScoreEvaluator(),
-            # "gleu": GleuScoreEvaluator(),
+            "gleu": GleuScoreEvaluator(),
             "bleu": BleuScoreEvaluator(),
             "rouge": RougeScoreEvaluator(RougeType.ROUGE_L),
             "meteor": MeteorScoreEvaluator(),
             "grounded": GroundednessEvaluator(model_config),
             "coherence": CoherenceEvaluator(model_config),
             "fluency": FluencyEvaluator(model_config),
             "relevance": RelevanceEvaluator(model_config),
-            # "similarity": SimilarityEvaluator(model_config),
-            # "qa" : QAEvaluator(model_config),
+            "similarity": SimilarityEvaluator(model_config),
+            # "qa": QAEvaluator(model_config),
             "grounded_pro": GroundednessProEvaluator(azure_cred, project_scope),
             "protected_material": ProtectedMaterialEvaluator(azure_cred, project_scope),
             "indirect_attack": IndirectAttackEvaluator(azure_cred, project_scope),
@@ -105,13 +105,13 @@ def test_evaluate_singleton_inputs(self, model_config, azure_cred, project_scope
         row_result_df = pd.DataFrame(result["rows"])
         metrics = result["metrics"]
 
-        assert len(row_result_df.keys()) == 45  #  63 with gleu, qa/similarity
+        assert len(row_result_df.keys()) == 48  # 63 with qa
         assert len(row_result_df["inputs.query"]) == 3
         assert len(row_result_df["inputs.context"]) == 3
         assert len(row_result_df["inputs.response"]) == 3
         assert len(row_result_df["inputs.ground_truth"]) == 3
         assert len(row_result_df["outputs.f1_score.f1_score"]) == 3
-        # assert len(row_result_df["outputs.gleu.gleu_score"]) == 3
+        assert len(row_result_df["outputs.gleu.gleu_score"]) == 3
         assert len(row_result_df["outputs.bleu.bleu_score"]) == 3
         assert len(row_result_df["outputs.rouge.rouge_precision"]) == 3
         assert len(row_result_df["outputs.rouge.rouge_recall"]) == 3
@@ -129,23 +129,8 @@ def test_evaluate_singleton_inputs(self, model_config, azure_cred, project_scope
         assert len(row_result_df["outputs.relevance.relevance"]) == 3
         assert len(row_result_df["outputs.relevance.gpt_relevance"]) == 3
         assert len(row_result_df["outputs.relevance.relevance_reason"]) == 3
-        # assert len(row_result_df['outputs.similarity.similarity']) == 3
-        # assert len(row_result_df['outputs.similarity.gpt_similarity']) == 3
-        # assert len(row_result_df['outputs.qa.f1_score']) == 3
-        # assert len(row_result_df['outputs.qa.groundedness']) == 3
-        # assert len(row_result_df['outputs.qa.gpt_groundedness']) == 3
-        # assert len(row_result_df['outputs.qa.groundedness_reason']) == 3
-        # assert len(row_result_df['outputs.qa.coherence']) == 3
-        # assert len(row_result_df['outputs.qa.gpt_coherence']) == 3
-        # assert len(row_result_df['outputs.qa.coherence_reason']) == 3
-        # assert len(row_result_df['outputs.qa.fluency']) == 3
-        # assert len(row_result_df['outputs.qa.gpt_fluency']) == 3
-        # assert len(row_result_df['outputs.qa.fluency_reason']) == 3
-        # assert len(row_result_df['outputs.qa.relevance']) == 3
-        # assert len(row_result_df['outputs.qa.gpt_relevance']) == 3
-        # assert len(row_result_df['outputs.qa.relevance_reason']) == 3
-        # assert len(row_result_df['outputs.qa.similarity']) == 3
-        # assert len(row_result_df['outputs.qa.gpt_similarity']) == 3
+        assert len(row_result_df["outputs.similarity.similarity"]) == 3
+        assert len(row_result_df["outputs.similarity.gpt_similarity"]) == 3
         assert len(row_result_df["outputs.grounded_pro.groundedness_pro_label"]) == 3
         assert len(row_result_df["outputs.grounded_pro.groundedness_pro_reason"]) == 3
         assert len(row_result_df["outputs.protected_material.protected_material_label"]) == 3
@@ -169,10 +154,25 @@ def test_evaluate_singleton_inputs(self, model_config, azure_cred, project_scope
         assert len(row_result_df["outputs.content_safety.violence"]) == 3
         assert len(row_result_df["outputs.content_safety.violence_score"]) == 3
         assert len(row_result_df["outputs.content_safety.violence_reason"]) == 3
+        # assert len(row_result_df["outputs.qa.f1_score"]) == 3
+        # assert len(row_result_df["outputs.qa.groundedness"]) == 3
+        # assert len(row_result_df["outputs.qa.gpt_groundedness"]) == 3
+        # assert len(row_result_df["outputs.qa.groundedness_reason"]) == 3
+        # assert len(row_result_df["outputs.qa.coherence"]) == 3
+        # assert len(row_result_df["outputs.qa.gpt_coherence"]) == 3
+        # assert len(row_result_df["outputs.qa.coherence_reason"]) == 3
+        # assert len(row_result_df["outputs.qa.fluency"]) == 3
+        # assert len(row_result_df["outputs.qa.gpt_fluency"]) == 3
+        # assert len(row_result_df["outputs.qa.fluency_reason"]) == 3
+        # assert len(row_result_df["outputs.qa.relevance"]) == 3
+        # assert len(row_result_df["outputs.qa.gpt_relevance"]) == 3
+        # assert len(row_result_df["outputs.qa.relevance_reason"]) == 3
+        # assert len(row_result_df["outputs.qa.similarity"]) == 3
+        # assert len(row_result_df["outputs.qa.gpt_similarity"]) == 3
 
-        assert len(metrics.keys()) == 25  # 39 with gleu, qa, similarity
+        assert len(metrics.keys()) == 28  # 39 with qa
         assert metrics["f1_score.f1_score"] >= 0
-        # assert metrics["gleu.gleu_score"] >= 0
+        assert metrics["gleu.gleu_score"] >= 0
         assert metrics["bleu.bleu_score"] >= 0
         assert metrics["rouge.rouge_precision"] >= 0
         assert metrics["rouge.rouge_recall"] >= 0
@@ -186,8 +186,8 @@ def test_evaluate_singleton_inputs(self, model_config, azure_cred, project_scope
         assert metrics["fluency.gpt_fluency"] >= 0
         assert metrics["relevance.relevance"] >= 0
         assert metrics["relevance.gpt_relevance"] >= 0
-        # assert metrics['similarity.similarity'] >= 0
-        # assert metrics['similarity.gpt_similarity'] >= 0
+        assert metrics["similarity.similarity"] >= 0
+        assert metrics["similarity.gpt_similarity"] >= 0
         assert metrics["indirect_attack.xpia_manipulated_content"] >= 0
         assert metrics["indirect_attack.xpia_intrusion"] >= 0
         assert metrics["indirect_attack.xpia_information_gathering"] >= 0
@@ -199,17 +199,17 @@ def test_evaluate_singleton_inputs(self, model_config, azure_cred, project_scope
         assert metrics["protected_material.protected_material_defect_rate"] >= 0
         assert metrics["indirect_attack.xpia_defect_rate"] >= 0
         assert metrics["eci.eci_defect_rate"] >= 0
-        # assert metrics['qa.f1_score'] >= 0
-        # assert metrics['qa.groundedness'] >= 0
-        # assert metrics['qa.gpt_groundedness'] >= 0
-        # assert metrics['qa.coherence'] >= 0
-        # assert metrics['qa.gpt_coherence'] >= 0
-        # assert metrics['qa.fluency'] >= 0
-        # assert metrics['qa.gpt_fluency'] >= 0
-        # assert metrics['qa.relevance'] >= 0
-        # assert metrics['qa.gpt_relevance'] >= 0
-        # assert metrics['qa.similarity'] >= 0
-        # assert metrics['qa.gpt_similarity'] >= 0
+        # assert metrics["qa.f1_score"] >= 0
+        # assert metrics["qa.groundedness"] >= 0
+        # assert metrics["qa.gpt_groundedness"] >= 0
+        # assert metrics["qa.coherence"] >= 0
+        # assert metrics["qa.gpt_coherence"] >= 0
+        # assert metrics["qa.fluency"] >= 0
+        # assert metrics["qa.gpt_fluency"] >= 0
+        # assert metrics["qa.relevance"] >= 0
+        # assert metrics["qa.gpt_relevance"] >= 0
+        # assert metrics["qa.similarity"] >= 0
+        # assert metrics["qa.gpt_similarity"] >= 0
 
     def test_evaluate_conversation(self, model_config, data_convo_file, azure_cred, project_scope):
         evaluators = {
@@ -291,7 +291,7 @@ def test_evaluate_conversation(self, model_config, data_convo_file, azure_cred,
         assert metrics["indirect_attack.xpia_defect_rate"] >= 0
         assert metrics["eci.eci_defect_rate"] >= 0
 
-    # Imagee urls with target is disabled due to being unstable in CI
+    # Image urls with target is disabled due to being unstable in CI
     @pytest.mark.parametrize(
         "multi_modal_input_type,pm_evaluator_class,cs_evaluator_class",
         [

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_built_in_evaluator.py

@@ -57,7 +57,7 @@ def test_fluency_evaluator_empty_string(self, mock_model_config):
 
     def test_similarity_evaluator_keys(self, mock_model_config):
         similarity_eval = SimilarityEvaluator(model_config=mock_model_config)
-        similarity_eval._async_evaluator._flow = MagicMock(return_value=quality_no_response_async_mock())
+        similarity_eval._flow = MagicMock(return_value=quality_no_response_async_mock())
 
         result = similarity_eval(
             query="What is the capital of Japan?",

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate_performance.py

@@ -50,7 +50,7 @@ def test_bulk_evaluate(self, big_f1_data_file, use_pf_client):
         if in_ci():
             max_duration += 25
         if use_pf_client:  # PF client doesn't seem to parallelize, and takes about a second or 2 to start
-            max_duration += 6.5
+            max_duration += 7.5
         assert diff < max_duration
         row_result_df = pd.DataFrame(result["rows"])
         assert "outputs.f1.f1_score" in row_result_df.columns
@@ -76,7 +76,7 @@ def test_evaluate_parallelism(self, ten_queries_file, use_pf_client):
         # 2 batches at most, so it should take between 1 and 1.5 seconds.
         max_duration = 1.5
         if use_pf_client:  # PF client doesn't seem to parallelize, and takes about a second to start.
-            max_duration += 7.5
+            max_duration += 8.5
         assert diff < max_duration
         row_result_df = pd.DataFrame(result["rows"])
         assert "outputs.slow.result" in row_result_df.columns