Update redteam complexity mapping (Azure#40321)

* Update complexity mapping

* remove unused debug_mode param

* update strategy converter mapping

* get base64 tense working properly

* Add api version if its available in model config

* fix printing errors

* make red_team separate sub-package and fix unit tests

* update sample

* Update setup.py to pin pyrit and termcolor

* Update setup.py

* fix unit tests

---------

Co-authored-by: Nagkumar Arkalgud <[email protected]>
Co-authored-by: Nagkumar Arkalgud <[email protected]>

Author: 3 people

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py

@@ -52,22 +52,6 @@
 except ImportError:
     print("[INFO] Could not import AIAgentConverter. Please install the dependency with `pip install azure-ai-projects`.")
 
-# RedTeam requires a dependency on pyrit, but python 3.9 is not supported by pyrit.
-# So we only import it if it's available and the user has pyrit.
-try:
-    from ._red_team._red_team import RedTeam
-    from ._red_team._attack_strategy import AttackStrategy
-    from ._red_team._attack_objective_generator import RiskCategory
-    from ._red_team._red_team_result import RedTeamOutput
-    _patch_all.extend([
-        "RedTeam",
-        "RedTeamOutput",
-        "AttackStrategy",
-        "RiskCategory",
-    ])
-except ImportError:
-    print("[INFO] Could not import RedTeam. Please install the dependency with `pip install azure-ai-evaluation[redteam]`.")
-
 
 __all__ = [
     "evaluate",

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_red_team/_utils/__init__.py

@@ -0,0 +1,15 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from ._red_team import RedTeam
+from ._attack_strategy import AttackStrategy
+from ._attack_objective_generator import RiskCategory
+from ._red_team_result import RedTeamOutput
+
+__all__ = [
+    "RedTeam",
+    "AttackStrategy",
+    "RiskCategory",
+    "RedTeamOutput",
+]

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/red_team/__init__.py

@@ -441,11 +441,11 @@ async def _get_attack_objectives(
                 self.logger.debug(f"API call: get_attack_objectives({risk_cat_value}, app: {application_scenario}, strategy: {strategy})")
                 # strategy param specifies whether to get a strategy-specific dataset from the RAI service
                 # right now, only tense requires strategy-specific dataset
-                if strategy == "tense":
+                if "tense" in strategy:
                     objectives_response = await self.generated_rai_client.get_attack_objectives(
                         risk_category=risk_cat_value,
                         application_scenario=application_scenario or "",
-                        strategy=strategy
+                        strategy="tense"
                     )
                 else: 
                     objectives_response = await self.generated_rai_client.get_attack_objectives(
@@ -679,7 +679,7 @@ async def _prompt_sending_orchestrator(
                         continue
                     except Exception as e:
                         log_error(self.logger, f"Error processing batch {batch_idx+1}", e, f"{strategy_name}/{risk_category}")
-                        print(f"❌ ERROR: Strategy {strategy_name}, Risk {risk_category}, Batch {batch_idx+1}: {str(e)}")
+                        self.logger.debug(f"ERROR: Strategy {strategy_name}, Risk {risk_category}, Batch {batch_idx+1}: {str(e)}")
                         # Continue with other batches even if one fails
                         continue
             else:
@@ -701,14 +701,14 @@ async def _prompt_sending_orchestrator(
                     self.task_statuses[single_batch_task_key] = TASK_STATUS["TIMEOUT"]
                 except Exception as e:
                     log_error(self.logger, "Error processing prompts", e, f"{strategy_name}/{risk_category}")
-                    print(f"❌ ERROR: Strategy {strategy_name}, Risk {risk_category}: {str(e)}")
+                    self.logger.debug(f"ERROR: Strategy {strategy_name}, Risk {risk_category}: {str(e)}")
 
             self.task_statuses[task_key] = TASK_STATUS["COMPLETED"]
             return orchestrator
 
         except Exception as e:
             log_error(self.logger, "Failed to initialize orchestrator", e, f"{strategy_name}/{risk_category}")
-            print(f"❌ CRITICAL: Failed to create orchestrator for {strategy_name}/{risk_category}: {str(e)}")
+            self.logger.debug(f"CRITICAL: Failed to create orchestrator for {strategy_name}/{risk_category}: {str(e)}")
             self.task_statuses[task_key] = TASK_STATUS["FAILED"]
             raise
 
@@ -1344,7 +1344,7 @@ async def _process_attack(
                 orchestrator = await call_orchestrator(self.chat_target, all_prompts, converter, strategy_name, risk_category.value, timeout)
             except PyritException as e:
                 log_error(self.logger, f"Error calling orchestrator for {strategy_name} strategy", e)
-                print(f"❌ Orchestrator error for {strategy_name}/{risk_category.value}: {str(e)}")
+                self.logger.debug(f"Orchestrator error for {strategy_name}/{risk_category.value}: {str(e)}")
                 self.task_statuses[task_key] = TASK_STATUS["FAILED"]
                 self.failed_tasks += 1
 
@@ -1399,7 +1399,7 @@ async def _process_attack(
 
         except Exception as e:
             log_error(self.logger, f"Unexpected error processing {strategy_name} strategy for {risk_category.value}", e)
-            print(f"❌ Critical error in task {strategy_name}/{risk_category.value}: {str(e)}")
+            self.logger.debug(f"Critical error in task {strategy_name}/{risk_category.value}: {str(e)}")
             self.task_statuses[task_key] = TASK_STATUS["FAILED"]
             self.failed_tasks += 1
 
@@ -1419,7 +1419,6 @@ async def scan(
             application_scenario: Optional[str] = None,
             parallel_execution: bool = True,
             max_parallel_tasks: int = 5,
-            debug_mode: bool = False,
             timeout: int = 120) -> RedTeamOutput:
         """Run a red team scan against the target using the specified strategies.
         
@@ -1441,8 +1440,6 @@ async def scan(
         :type parallel_execution: bool
         :param max_parallel_tasks: Maximum number of parallel orchestrator tasks to run (default: 5)
         :type max_parallel_tasks: int
-        :param debug_mode: Whether to run in debug mode (more verbose output)
-        :type debug_mode: bool
         :param timeout: The timeout in seconds for API calls (default: 120)
         :type timeout: int
         :return: The output from the red team scan
@@ -1522,7 +1519,7 @@ def filter(self, record):
         if not self.attack_objective_generator:
             error_msg = "Attack objective generator is required for red team agent."
             log_error(self.logger, error_msg)
-            print(f"❌ {error_msg}")
+            self.logger.debug(f"{error_msg}")
             raise EvaluationException(
                 message=error_msg,
                 internal_message="Attack objective generator is not provided.",
@@ -1676,17 +1673,13 @@ def filter(self, record):
                 for risk_category in self.risk_categories:
                     progress_bar.set_postfix({"current": f"fetching {strategy_name}/{risk_category.value}"})
                     self.logger.debug(f"Fetching objectives for {strategy_name} strategy and {risk_category.value} risk category")
-                    
                     objectives = await self._get_attack_objectives(
                         risk_category=risk_category,
                         application_scenario=application_scenario,
                         strategy=strategy_name
                     )
                     all_objectives[strategy_name][risk_category.value] = objectives
 
-                    # Print status about objective count for this strategy/risk
-                    if debug_mode:
-                        print(f"  - {risk_category.value}: {len(objectives)} objectives")
 
             self.logger.info("Completed fetching all attack objectives")
 
@@ -1754,7 +1747,7 @@ def filter(self, record):
                         continue
                     except Exception as e:
                         log_error(self.logger, f"Error processing batch {i//max_parallel_tasks+1}", e)
-                        print(f"❌ Error in batch {i//max_parallel_tasks+1}: {str(e)}")
+                        self.logger.debug(f"Error in batch {i//max_parallel_tasks+1}: {str(e)}")
                         continue
             else:
                 # Sequential execution 
@@ -1776,7 +1769,7 @@ def filter(self, record):
                         continue
                     except Exception as e:
                         log_error(self.logger, f"Error processing task {i+1}/{len(orchestrator_tasks)}", e)
-                        print(f"❌ Error in task {i+1}: {str(e)}")
+                        self.logger.debug(f"Error in task {i+1}: {str(e)}")
                         continue
 
             progress_bar.close()