Eval/feature/unified eval inputs (#37523)

* mid-progress

* more base eval stuff

* async stuff

* change 1 eval to use base

* fix initial bugs

* apply new base to rai evals

* add reworked evals, and remove content safety chat

* update tests

* renaming evals

* refactor to derive conversation converter, add 2 evals, surface retrieval

* changelog and init

* run black

* pylint

* cspell

* re-skip

* optional cred

* change async passthrough

* test full

* comments

* cspell

* cspell

* run black

* typehint

* fix types for saving

* upddate conftest and recorddigns

* pylint

* fix fluency unit tests

* last new assets

Author: MilesHolland

sdk/evaluation/azure-ai-evaluation/CHANGELOG.md

@@ -5,9 +5,23 @@
 ### Features Added
 
 - Added `type` field to `AzureOpenAIModelConfiguration` and `OpenAIModelConfiguration`
+- The following evaluators now support `conversation` as an alternative input to their usual single-turn inputs:
+  - `ViolenceEvaluator`
+  - `SexualEvaluator`
+  - `SelfHarmEvaluator`
+  - `HateUnfairnessEvaluator`
+  - `ProtectedMaterialEvaluator`
+  - `IndirectAttackEvaluator`
+  - `CoherenceEvaluator`
+  - `RelevanceEvaluator`
+  - `FluencyEvaluator`
+  - `GroundednessEvaluator`
+- Surfaced `RetrievalScoreEvaluator`, formally an internal part of `ChatEvaluator` as a standalone conversation-only evaluator.
 
 ### Breaking Changes
 
+
+- Removed `ContentSafetyChatEvaluator` and `ChatEvaluator`
 - The `evaluator_config` parameter of `evaluate` now maps in evaluator name to a dictionary `EvaluatorConfig`, which is a `TypedDict`. The
 `column_mapping` between `data` or `target` and evaluator field names should now be specified inside this new dictionary:
 

sdk/evaluation/azure-ai-evaluation/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/evaluation/azure-ai-evaluation",
-  "Tag": "python/evaluation/azure-ai-evaluation_26cf396fa1"
+  "Tag": "python/evaluation/azure-ai-evaluation_051cb9dfbd"
 }

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py

@@ -4,10 +4,8 @@
 
 from ._evaluate._evaluate import evaluate
 from ._evaluators._bleu import BleuScoreEvaluator
-from ._evaluators._chat import ChatEvaluator
 from ._evaluators._coherence import CoherenceEvaluator
 from ._evaluators._content_safety import (
-    ContentSafetyChatEvaluator,
     ContentSafetyEvaluator,
     HateUnfairnessEvaluator,
     SelfHarmEvaluator,
@@ -22,6 +20,7 @@
 from ._evaluators._protected_material import ProtectedMaterialEvaluator
 from ._evaluators._qa import QAEvaluator
 from ._evaluators._relevance import RelevanceEvaluator
+from ._evaluators._retrieval import RetrievalEvaluator
 from ._evaluators._rouge import RougeScoreEvaluator, RougeType
 from ._evaluators._similarity import SimilarityEvaluator
 from ._evaluators._xpia import IndirectAttackEvaluator
@@ -41,17 +40,16 @@
     "RelevanceEvaluator",
     "SimilarityEvaluator",
     "QAEvaluator",
-    "ChatEvaluator",
     "ViolenceEvaluator",
     "SexualEvaluator",
     "SelfHarmEvaluator",
     "HateUnfairnessEvaluator",
     "ContentSafetyEvaluator",
-    "ContentSafetyChatEvaluator",
     "IndirectAttackEvaluator",
     "BleuScoreEvaluator",
     "GleuScoreEvaluator",
     "MeteorScoreEvaluator",
+    "RetrievalEvaluator",
     "RougeScoreEvaluator",
     "RougeType",
     "ProtectedMaterialEvaluator",

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py

@@ -158,6 +158,12 @@ def _validate_input_data_for_evaluator(evaluator, evaluator_name, df_data, is_ta
     ]
 
     missing_inputs = [col for col in required_inputs if col not in df_data.columns]
+    if missing_inputs and "conversation" in required_inputs:
+        non_conversation_inputs = [val for val in required_inputs if val != "conversation"]
+        if len(missing_inputs) == len(non_conversation_inputs) and [
+            input in non_conversation_inputs for input in missing_inputs
+        ]:
+            missing_inputs = []
     if missing_inputs:
         if not is_target_fn:
             msg = f"Missing required inputs for evaluator {evaluator_name} : {missing_inputs}."