Removing Multimodal evals and fixes few tests (Azure#39558)

* Removing Multimodal evals and fixes few tests

* fix

* release date

* release date

* release date

Author: w-javed

sdk/evaluation/azure-ai-evaluation/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release History
 
+## 1.3.0 (Unreleased)
+
+### Breaking Changes
+- Multimodal specific evaluators `ContentSafetyMultimodalEvaluator`, `ViolenceMultimodalEvaluator`, `SexualMultimodalEvaluator`, `SelfHarmMultimodalEvaluator`, `HateUnfairnessMultimodalEvaluator` and `ProtectedMaterialMultimodalEvaluator` has been removed. Please use `ContentSafetyEvaluator`, `ViolenceEvaluator`, `SexualEvaluator`, `SelfHarmEvaluator`, `HateUnfairnessEvaluator` and `ProtectedMaterialEvaluator` instead.
+- Metric name in ProtectedMaterialEvaluator's output is changed from `protected_material.fictional_characters_label` to `protected_material.fictional_characters_defect_rate`. It's now consistent with other evaluator's metric names (ending with `_defect_rate`).
+
 ## 1.2.0 (2025-01-27)
 
 ### Features Added

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py

@@ -12,14 +12,6 @@
     SexualEvaluator,
     ViolenceEvaluator,
 )
-from ._evaluators._multimodal._content_safety_multimodal import (
-    ContentSafetyMultimodalEvaluator,
-    HateUnfairnessMultimodalEvaluator,
-    SelfHarmMultimodalEvaluator,
-    SexualMultimodalEvaluator,
-    ViolenceMultimodalEvaluator,
-)
-from ._evaluators._multimodal._protected_material import ProtectedMaterialMultimodalEvaluator
 from ._evaluators._f1_score import F1ScoreEvaluator
 from ._evaluators._fluency import FluencyEvaluator
 from ._evaluators._gleu import GleuScoreEvaluator
@@ -72,11 +64,5 @@
     "EvaluatorConfig",
     "Conversation",
     "Message",
-    "EvaluationResult",
-    "ContentSafetyMultimodalEvaluator",
-    "HateUnfairnessMultimodalEvaluator",
-    "SelfHarmMultimodalEvaluator",
-    "SexualMultimodalEvaluator",
-    "ViolenceMultimodalEvaluator",
-    "ProtectedMaterialMultimodalEvaluator",
+    "EvaluationResult"
 ]

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py

@@ -366,7 +366,7 @@ def raise_exception(msg, target):
     if not isinstance(messages, list):
         raise_exception(
             "'messages' parameter must be a JSON-compatible list of chat messages",
-            ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+            ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
         )
     expected_roles = {"user", "assistant", "system"}
     image_found = False
@@ -393,7 +393,7 @@ def raise_exception(msg, target):
             ):
                 raise_exception(
                     f"Messages must be a strongly typed class of ChatRequestMessage. Message number: {num}",
-                    ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+                    ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
                 )
             if isinstance(message, AssistantMessage):
                 assistant_message_count += 1
@@ -407,7 +407,7 @@ def raise_exception(msg, target):
         if message.get("role") not in expected_roles:
             raise_exception(
                 f"Invalid role provided: {message.get('role')}. Message number: {num}",
-                ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+                ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
             )
         if message.get("role") == "assistant":
             assistant_message_count += 1
@@ -417,29 +417,29 @@ def raise_exception(msg, target):
         if not isinstance(content, (str, list)):
             raise_exception(
                 f"Content in each turn must be a string or array. Message number: {num}",
-                ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+                ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
             )
         if isinstance(content, list):
             if any(item.get("type") == "image_url" and "url" in item.get("image_url", {}) for item in content):
                 image_found = True
     if not image_found:
         raise_exception(
             "Message needs to have multi-modal input like images.",
-            ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+            ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
         )
     if assistant_message_count == 0:
         raise_exception(
             "Assistant role required in one of the messages.",
-            ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+            ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
         )
     if user_message_count == 0:
         raise_exception(
             "User role required in one of the messages.",
-            ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+            ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
         )
     if assistant_message_count > 1:
         raise_exception(
             "Evaluators for multimodal conversations only support single turn. "
             "User and assistant role expected as the only role in each message.",
-            ErrorTarget.CONTENT_SAFETY_MULTIMODAL_EVALUATOR,
+            ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
         )