redteam aoai instance results updates v3 (#43256)

* aoai instance results updates v3

* add label to RedTeamRunOutputItemResult

* reformatting

Author: slister1001

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_mlflow_integration.py

@@ -204,11 +204,6 @@ async def log_redteam_results_to_mlflow(
                         raise ValueError("aoai_summary parameter is required but was not provided")
 
                     payload = dict(aoai_summary)  # Make a copy
-                    # Ensure conversations are included for scan output
-                    if "conversations" not in payload:
-                        payload["conversations"] = (
-                            redteam_result.attack_details or redteam_result.scan_result.get("attack_details") or []
-                        )
                     json.dump(payload, f)
 
                 # Save legacy format as instance_results.json

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_red_team_result.py

@@ -271,19 +271,23 @@ class EvaluationRunOutputItemMessage(TypedDict, total=False):
 
 @experimental
 class RedTeamRunOutputItemResult(TypedDict, total=False):
-    """Flattened evaluation result for a single risk category."""
+    """Flattened evaluation result for a single risk category.
+
+    :param label: String label "pass" or "fail" that aligns with the passed field
+    :type label: Optional[str]
+    """
 
     # Should extend EvaluationRunOutputItemResult
 
     object: str
     type: str
     name: str
     passed: Optional[bool]
+    label: Optional[str]
     score: Optional[float]
     metric: Optional[str]
     threshold: Optional[float]
     reason: Optional[str]
-    sample: "RedTeamRunOutputItemSample"
     properties: RedTeamOutputResultProperties
 
 
@@ -376,18 +380,21 @@ class ResultCount(TypedDict):
 
 
 @experimental
-class PerTestingCriteriaResult(TypedDict):
+class PerTestingCriteriaResult(TypedDict, total=False):
     """Result count for a specific testing criteria.
 
     :param testing_criteria: The name of the testing criteria (e.g., risk category)
     :type testing_criteria: str
+    :param attack_strategy: The attack strategy used (optional, for attack strategy summaries)
+    :type attack_strategy: Optional[str]
     :param passed: Number of passed results for this criteria
     :type passed: int
     :param failed: Number of failed results for this criteria
     :type failed: int
     """
 
     testing_criteria: str
+    attack_strategy: Optional[str]
     passed: int
     failed: int
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/_result_processor.py

@@ -375,7 +375,6 @@ def to_red_team_result(
             output_items=ordered_output_items,
             eval_run=eval_run,
             red_team_info=red_team_info,
-            include_conversations=False,
             scan_name=scan_name,
             run_id_override=run_id_override,
             eval_id_override=eval_id_override,
@@ -413,7 +412,7 @@ def _build_output_item(
         results = self._build_output_result(
             conversation,
             eval_row,
-            sample_payload=sample_payload,
+            sample_payload=None,
         )
         output_item_id = self._resolve_output_item_id(
             eval_row, datasource_item_id, conversation_key, conversation_index
@@ -431,6 +430,7 @@ def _build_output_item(
             "id": output_item_id,
             "created_time": created_time,
             "status": status,
+            "sample": sample_payload,
             "results": results,
         }
 
@@ -584,6 +584,7 @@ def _build_output_result(
                 "name": risk_value,
                 "metric": risk_value,
                 "passed": passed,
+                "label": "pass" if passed is True else ("fail" if passed is False else None),
                 "score": score,
                 "threshold": threshold,
                 "reason": reason,
@@ -592,9 +593,6 @@ def _build_output_result(
             if properties:
                 result_entry["properties"] = properties
 
-            if sample_payload:
-                result_entry["sample"] = sample_payload
-
             results.append(result_entry)
 
         if not results:
@@ -624,6 +622,7 @@ def _build_output_result(
                 "name": risk_value,
                 "metric": risk_value,
                 "passed": None,
+                "label": None,
                 "score": None,
                 "threshold": attack_threshold,
                 "reason": fallback_reason,
@@ -632,9 +631,6 @@ def _build_output_result(
             if properties:
                 fallback_result["properties"] = properties
 
-            if sample_payload:
-                fallback_result["sample"] = sample_payload
-
             results.append(fallback_result)
 
         return results
@@ -1096,9 +1092,12 @@ def _compute_result_count(output_items: List[Dict[str, Any]]) -> Dict[str, int]:
 
     @staticmethod
     def _compute_per_testing_criteria(output_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        """Build aggregated pass/fail counts per testing criteria (risk category)."""
+        """Build aggregated pass/fail counts per testing criteria (risk category and attack strategy)."""
 
+        # Track by risk category (testing_criteria)
         criteria: Dict[str, Dict[str, int]] = {}
+        # Track by attack strategy
+        strategy_criteria: Dict[str, Dict[str, int]] = {}
 
         for item in output_items:
             for result in item.get("results", []):
@@ -1111,13 +1110,28 @@ def _compute_per_testing_criteria(output_items: List[Dict[str, Any]]) -> List[Di
                 if passed_value is None:
                     continue
 
+                # Track by risk category
                 bucket = criteria.setdefault(str(name), {"passed": 0, "failed": 0})
                 if passed_value:
                     bucket["passed"] += 1
                 else:
                     bucket["failed"] += 1
 
-        return [
+                # Track by attack strategy from properties
+                properties = result.get("properties", {})
+                if isinstance(properties, dict):
+                    attack_technique = properties.get("attack_technique")
+                    if attack_technique:
+                        strategy_bucket = strategy_criteria.setdefault(
+                            str(attack_technique), {"passed": 0, "failed": 0}
+                        )
+                        if passed_value:
+                            strategy_bucket["passed"] += 1
+                        else:
+                            strategy_bucket["failed"] += 1
+
+        # Build results list with risk categories
+        results = [
             {
                 "testing_criteria": criteria_name,
                 "passed": counts["passed"],
@@ -1126,6 +1140,19 @@ def _compute_per_testing_criteria(output_items: List[Dict[str, Any]]) -> List[Di
             for criteria_name, counts in sorted(criteria.items())
         ]
 
+        # Add attack strategy summaries
+        for strategy_name, counts in sorted(strategy_criteria.items()):
+            results.append(
+                {
+                    "testing_criteria": strategy_name,
+                    "attack_strategy": strategy_name,
+                    "passed": counts["passed"],
+                    "failed": counts["failed"],
+                }
+            )
+
+        return results
+
     @staticmethod
     def _build_data_source_section(parameters: Dict[str, Any], red_team_info: Optional[Dict]) -> Dict[str, Any]:
         """Build the data_source portion of the run payload for red-team scans."""
@@ -1179,7 +1206,6 @@ def _build_results_payload(
         output_items: List[Dict[str, Any]],
         eval_run: Optional[Any] = None,
         red_team_info: Optional[Dict] = None,
-        include_conversations: bool = False,
         scan_name: Optional[str] = None,
         run_id_override: Optional[str] = None,
         eval_id_override: Optional[str] = None,
@@ -1191,7 +1217,6 @@ def _build_results_payload(
         :param output_items: List of output items containing results for each conversation
         :param eval_run: The MLFlow run object (optional)
         :param red_team_info: Red team tracking information (optional)
-        :param include_conversations: Whether to include conversation details (optional)
         :param scan_name: Name of the scan (optional)
         :param run_id_override: Override for run ID (optional)
         :param eval_id_override: Override for eval ID (optional)
@@ -1290,7 +1315,4 @@ def _build_results_payload(
             "output_items": list_wrapper,
         }
 
-        if include_conversations:
-            run_payload["conversations"] = redteam_result.attack_details or scan_result.get("attack_details") or []
-
         return run_payload