Removing private parameter from __call__ of AdversarialSimulator (Azure#37709)

* Update task_query_response.prompty

remove required keys

* Update task_simulate.prompty

* Update task_query_response.prompty

* Update task_simulate.prompty

* Remove private variable and use kwargs

* Add experimental tag to adv sim

---------

Co-authored-by: Nagkumar Arkalgud <[email protected]>

Author: nagkumar91

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_adversarial_simulator.py

@@ -29,10 +29,12 @@
     TokenScope,
 )
 from ._utils import JsonLineList
+from ._helpers import experimental
 
 logger = logging.getLogger(__name__)
 
 
+@experimental
 class AdversarialSimulator:
     """
     Initializes the adversarial simulator with a project scope.
@@ -92,7 +94,7 @@ def _ensure_service_dependencies(self):
                 blame=ErrorBlame.USER_ERROR,
             )
 
-    # @monitor_adversarial_scenario
+    # pylint: disable=too-many-locals
     async def __call__(
         self,
         *,
@@ -106,10 +108,10 @@ async def __call__(
         api_call_retry_sleep_sec: int = 1,
         api_call_delay_sec: int = 0,
         concurrent_async_task: int = 3,
-        _jailbreak_type: Optional[str] = None,
         language: SupportedLanguages = SupportedLanguages.English,
         randomize_order: bool = True,
         randomization_seed: Optional[int] = None,
+        **kwargs,
     ):
         """
         Executes the adversarial simulation against a specified target function asynchronously.
@@ -216,6 +218,7 @@ async def __call__(
                 total_tasks,
             )
         total_tasks = min(total_tasks, max_simulation_results)
+        _jailbreak_type = kwargs.get("_jailbreak_type", None)
         if _jailbreak_type:
             jailbreak_dataset = await self.rai_client.get_jailbreaks_dataset(type=_jailbreak_type)
         progress_bar = tqdm(

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_direct_attack_simulator.py

@@ -1,54 +1,25 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+# pylint: disable=C0301,C0114,R0913,R0903
 # noqa: E501
-import functools
 import logging
 from random import randint
 from typing import Callable, Optional
 
-from promptflow._sdk._telemetry import ActivityType, monitor_operation
-
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._model_configurations import AzureAIProject
 from azure.ai.evaluation.simulator import AdversarialScenario
 from azure.identity import DefaultAzureCredential
 
 from ._adversarial_simulator import AdversarialSimulator
 from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient, TokenScope
+from ._helpers import experimental
 
 logger = logging.getLogger(__name__)
 
 
-def monitor_adversarial_scenario(func) -> Callable:
-    """Decorator to monitor adversarial scenario.
-
-    :param func: The function to be decorated.
-    :type func: Callable
-    :return: The decorated function.
-    :rtype: Callable
-    """
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        scenario = str(kwargs.get("scenario", None))
-        max_conversation_turns = kwargs.get("max_conversation_turns", None)
-        max_simulation_results = kwargs.get("max_simulation_results", None)
-        decorated_func = monitor_operation(
-            activity_name="jailbreak.adversarial.simulator.call",
-            activity_type=ActivityType.PUBLICAPI,
-            custom_dimensions={
-                "scenario": scenario,
-                "max_conversation_turns": max_conversation_turns,
-                "max_simulation_results": max_simulation_results,
-            },
-        )(func)
-
-        return decorated_func(*args, **kwargs)
-
-    return wrapper
-
-
+@experimental
 class DirectAttackSimulator:
     """
     Initialize a UPIA (user prompt injected attack) jailbreak adversarial simulator with a project scope.
@@ -110,7 +81,6 @@ def _ensure_service_dependencies(self):
                 blame=ErrorBlame.USER_ERROR,
             )
 
-    # @monitor_adversarial_scenario
     async def __call__(
         self,
         *,

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_indirect_attack_simulator.py

@@ -1,53 +1,23 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+# pylint: disable=C0301,C0114,R0913,R0903
 # noqa: E501
-import functools
 import logging
 from typing import Callable
-
-from promptflow._sdk._telemetry import ActivityType, monitor_operation
-
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._model_configurations import AzureAIProject
 from azure.ai.evaluation.simulator import AdversarialScenario
 from azure.identity import DefaultAzureCredential
 
 from ._adversarial_simulator import AdversarialSimulator
 from ._model_tools import AdversarialTemplateHandler, ManagedIdentityAPITokenManager, RAIClient, TokenScope
+from ._helpers import experimental
 
 logger = logging.getLogger(__name__)
 
 
-def monitor_adversarial_scenario(func) -> Callable:
-    """Decorator to monitor adversarial scenario.
-
-    :param func: The function to be decorated.
-    :type func: Callable
-    :return: The decorated function.
-    :rtype: Callable
-    """
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        scenario = str(kwargs.get("scenario", None))
-        max_conversation_turns = kwargs.get("max_conversation_turns", None)
-        max_simulation_results = kwargs.get("max_simulation_results", None)
-        decorated_func = monitor_operation(
-            activity_name="xpia.adversarial.simulator.call",
-            activity_type=ActivityType.PUBLICAPI,
-            custom_dimensions={
-                "scenario": scenario,
-                "max_conversation_turns": max_conversation_turns,
-                "max_simulation_results": max_simulation_results,
-            },
-        )(func)
-
-        return decorated_func(*args, **kwargs)
-
-    return wrapper
-
-
+@experimental
 class IndirectAttackSimulator:
     """
     Initializes the XPIA (cross domain prompt injected attack) jailbreak adversarial simulator with a project scope.
@@ -107,7 +77,6 @@ def _ensure_service_dependencies(self):
                 blame=ErrorBlame.USER_ERROR,
             )
 
-    # @monitor_adversarial_scenario
     async def __call__(
         self,
         *,

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_prompty/task_query_response.prompty

@@ -6,7 +6,6 @@ model:
   configuration:
     type: azure_openai
     azure_deployment: ${env:AZURE_DEPLOYMENT}
-    api_key: ${env:AZURE_OPENAI_API_KEY}
     azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
   parameters:
     temperature: 0.0

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py

@@ -1,5 +1,5 @@
 # flake8: noqa
-# pylint: disable=W0102,W0613,R0914,C0301,E0401,E0611
+# pylint: disable=W0102,W0613,R0914,C0301,E0401,E0611,C0114,R0913,E0702,R0903
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------