[redteam] agent risk categories and refactor XPIA into attack strategy (#43054)

* Refactor XPIA into AttackStrategy

* RedTeam context handling improvements

* fix formatting for xpia prompts

* updates

* updates

* updates working multiple contexts

* context without tool_name / context_type

* updates

* risk subtypes

* sync step 1

* risk subtypes pt 2

* updates

* undo sync updates until updated typespec

* add new risk categories

* custom attack objectives behavioral change

* fix target for attack objectives

* update client

* formatting

* ensure consistency between orchestrators behavior

* old flow working

* create sync eval working

* eval result being parsed properly!!

* temp commit to call vienna endpoint for sync evals

* sdl pipeliene working local

* fix num objectives for xpia

* taxonomy support

* remove local rai service config logic

* formatting

* undo int url hardcoding

* copilot comment

* revert change to sample

* updated projects api version to 11-15

* spell check errors

* make xpia easy

* fix import

* fix all imports of projectsclient

* last import fix I hope

* update pandas for 3.14

* start fix tests, remove experimental tags from safety eval, undo setup changes

* run formatter

* app insights changes commented out

* formatting

* updates to client

* fix evaluator name

* fix unit tests

* fix some e2e tests

* fix formatting

* fix some e2e tests

* update recordings again

* add back experimental tags for content safety evaluators for now

Author: slister1001

sdk/evaluation/azure-ai-evaluation/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/evaluation/azure-ai-evaluation",
-  "Tag": "python/evaluation/azure-ai-evaluation_d7b00f22b8"
+  "Tag": "python/evaluation/azure-ai-evaluation_b613e35220"
 }

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/__init__.py

@@ -6,14 +6,15 @@
 # that would have otherwise been a relative import scoped to single evaluator directories.
 
 from . import constants
-from .rai_service import evaluate_with_rai_service
+from .rai_service import evaluate_with_rai_service, evaluate_with_rai_service_sync
 from .utils import get_harm_severity_level
 from .evaluation_onedp_client import EvaluationServiceOneDPClient
 from .onedp.models import EvaluationUpload, EvaluationResult, RedTeamUpload, ResultType
 
 __all__ = [
     "get_harm_severity_level",
     "evaluate_with_rai_service",
+    "evaluate_with_rai_service_sync",
     "constants",
     "EvaluationServiceOneDPClient",
     "EvaluationResult",

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/constants.py

@@ -50,6 +50,9 @@ class Tasks:
     GROUNDEDNESS = "groundedness"
     CODE_VULNERABILITY = "code vulnerability"
     UNGROUNDED_ATTRIBUTES = "inference sensitive attributes"
+    SENSITIVE_DATA_LEAKAGE = "sensitive_data_leakage"
+    TASK_ADHERENCE = "task_adherence"
+    PROHIBITED_ACTIONS = "prohibited_actions"
 
 
 class _InternalAnnotationTasks:
@@ -74,6 +77,9 @@ class EvaluationMetrics(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     GROUNDEDNESS = "generic_groundedness"
     CODE_VULNERABILITY = "code_vulnerability"
     UNGROUNDED_ATTRIBUTES = "ungrounded_attributes"
+    SENSITIVE_DATA_LEAKAGE = "sensitive_data_leakage"
+    TASK_ADHERENCE = "task_adherence"
+    PROHIBITED_ACTIONS = "prohibited_actions"
 
 
 class _InternalEvaluationMetrics(str, Enum, metaclass=CaseInsensitiveEnumMeta):

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/evaluation_onedp_client.py

@@ -5,7 +5,7 @@
 import logging
 from typing import Union, Any, Dict
 from azure.core.credentials import AzureKeyCredential, TokenCredential
-from azure.ai.evaluation._common.onedp import AIProjectClient as RestEvaluationServiceClient
+from azure.ai.evaluation._common.onedp import ProjectsClient as RestEvaluationServiceClient
 from azure.ai.evaluation._common.onedp.models import (
     PendingUploadRequest,
     PendingUploadType,
@@ -71,7 +71,7 @@ def create_evaluation_result(
         )
         start_pending_upload_response = self.rest_client.evaluation_results.start_pending_upload(
             name=name,
-            version=version,
+            version=str(version),
             body=PendingUploadRequest(pending_upload_type=PendingUploadType.TEMPORARY_BLOB_REFERENCE),
             **kwargs,
         )
@@ -84,15 +84,15 @@ def create_evaluation_result(
 
         LOGGER.debug(f"Creating evaluation result version for {name} with version {version}")
         create_version_response = self.rest_client.evaluation_results.create_or_update_version(
-            body=EvaluationResult(
+            evaluation_result=EvaluationResult(
                 blob_uri=start_pending_upload_response.blob_reference_for_consumption.blob_uri,
                 result_type=result_type,
                 name=name,
-                version=version,
+                version=str(version),
                 metrics=metrics,
             ),
             name=name,
-            version=version,
+            version=str(version),
             **kwargs,
         )
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/onedp/__init__.py

@@ -12,7 +12,7 @@
 if TYPE_CHECKING:
     from ._patch import *  # pylint: disable=unused-wildcard-import
 
-from ._client import AIProjectClient  # type: ignore
+from ._client import ProjectsClient  # type: ignore
 from ._version import VERSION
 
 __version__ = VERSION
@@ -25,7 +25,7 @@
 from ._patch import patch_sdk as _patch_sdk
 
 __all__ = [
-    "AIProjectClient",
+    "ProjectsClient",
 ]
 __all__.extend([p for p in _patch_all if p not in __all__])  # pyright: ignore
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/onedp/_client.py

@@ -14,58 +14,78 @@
 from azure.core.pipeline import policies
 from azure.core.rest import HttpRequest, HttpResponse
 
-from ._configuration import AIProjectClientConfiguration
+from ._configuration import ProjectsClientConfiguration
 from ._utils.serialization import Deserializer, Serializer
 from .operations import (
     ConnectionsOperations,
     DatasetsOperations,
     DeploymentsOperations,
     EvaluationResultsOperations,
+    EvaluationRulesOperations,
+    EvaluationTaxonomiesOperations,
     EvaluationsOperations,
+    EvaluatorsOperations,
     IndexesOperations,
+    InsightsOperations,
     RedTeamsOperations,
+    SchedulesOperations,
+    SyncEvalsOperations,
 )
 
 if TYPE_CHECKING:
     from azure.core.credentials import TokenCredential
 
 
-class AIProjectClient:  # pylint: disable=too-many-instance-attributes
-    """AIProjectClient.
+class ProjectsClient:  # pylint: disable=too-many-instance-attributes
+    """ProjectsClient.
 
     :ivar connections: ConnectionsOperations operations
-    :vartype connections: azure.ai.projects.onedp.operations.ConnectionsOperations
+    :vartype connections: azure.ai.projects.operations.ConnectionsOperations
+    :ivar sync_evals: SyncEvalsOperations operations
+    :vartype sync_evals: azure.ai.projects.operations.SyncEvalsOperations
     :ivar evaluations: EvaluationsOperations operations
-    :vartype evaluations: azure.ai.projects.onedp.operations.EvaluationsOperations
+    :vartype evaluations: azure.ai.projects.operations.EvaluationsOperations
+    :ivar evaluators: EvaluatorsOperations operations
+    :vartype evaluators: azure.ai.projects.operations.EvaluatorsOperations
     :ivar datasets: DatasetsOperations operations
-    :vartype datasets: azure.ai.projects.onedp.operations.DatasetsOperations
+    :vartype datasets: azure.ai.projects.operations.DatasetsOperations
     :ivar indexes: IndexesOperations operations
-    :vartype indexes: azure.ai.projects.onedp.operations.IndexesOperations
+    :vartype indexes: azure.ai.projects.operations.IndexesOperations
+    :ivar insights: InsightsOperations operations
+    :vartype insights: azure.ai.projects.operations.InsightsOperations
     :ivar deployments: DeploymentsOperations operations
-    :vartype deployments: azure.ai.projects.onedp.operations.DeploymentsOperations
+    :vartype deployments: azure.ai.projects.operations.DeploymentsOperations
     :ivar red_teams: RedTeamsOperations operations
-    :vartype red_teams: azure.ai.projects.onedp.operations.RedTeamsOperations
+    :vartype red_teams: azure.ai.projects.operations.RedTeamsOperations
+    :ivar evaluation_taxonomies: EvaluationTaxonomiesOperations operations
+    :vartype evaluation_taxonomies: azure.ai.projects.operations.EvaluationTaxonomiesOperations
+    :ivar schedules: SchedulesOperations operations
+    :vartype schedules: azure.ai.projects.operations.SchedulesOperations
     :ivar evaluation_results: EvaluationResultsOperations operations
-    :vartype evaluation_results: azure.ai.projects.onedp.operations.EvaluationResultsOperations
+    :vartype evaluation_results: azure.ai.projects.operations.EvaluationResultsOperations
+    :ivar evaluation_rules: EvaluationRulesOperations operations
+    :vartype evaluation_rules: azure.ai.projects.operations.EvaluationRulesOperations
     :param endpoint: Project endpoint. In the form
-     "https://<your-ai-services-account-name>.services.ai.azure.com/api/projects/_project"
+     "`https://your-ai-services-account-name.services.ai.azure.com/api/projects/_project
+     <https://your-ai-services-account-name.services.ai.azure.com/api/projects/_project>`_"
      if your Foundry Hub has only one Project, or to use the default Project in your Hub. Or in the
      form
-     "https://<your-ai-services-account-name>.services.ai.azure.com/api/projects/<your-project-name>"
+     "`https://your-ai-services-account-name.services.ai.azure.com/api/projects/your-project-name
+     <https://your-ai-services-account-name.services.ai.azure.com/api/projects/your-project-name>`_"
      if you want to explicitly
      specify the Foundry Project name. Required.
     :type endpoint: str
     :param credential: Credential used to authenticate requests to the service. Required.
     :type credential: ~azure.core.credentials.TokenCredential
     :keyword api_version: The API version to use for this operation. Default value is
-     "2025-05-15-preview". Note that overriding this default value may result in unsupported
+     "2025-11-15-preview". Note that overriding this default value may result in unsupported
      behavior.
     :paramtype api_version: str
     """
 
     def __init__(self, endpoint: str, credential: "TokenCredential", **kwargs: Any) -> None:
         _endpoint = "{endpoint}"
-        self._config = AIProjectClientConfiguration(endpoint=endpoint, credential=credential, **kwargs)
+        self._config = ProjectsClientConfiguration(endpoint=endpoint, credential=credential, **kwargs)
 
         _policies = kwargs.pop("policies", None)
         if _policies is None:
@@ -90,14 +110,24 @@ def __init__(self, endpoint: str, credential: "TokenCredential", **kwargs: Any)
         self._deserialize = Deserializer()
         self._serialize.client_side_validation = False
         self.connections = ConnectionsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.sync_evals = SyncEvalsOperations(self._client, self._config, self._serialize, self._deserialize)
         self.evaluations = EvaluationsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.evaluators = EvaluatorsOperations(self._client, self._config, self._serialize, self._deserialize)
         self.datasets = DatasetsOperations(self._client, self._config, self._serialize, self._deserialize)
         self.indexes = IndexesOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.insights = InsightsOperations(self._client, self._config, self._serialize, self._deserialize)
         self.deployments = DeploymentsOperations(self._client, self._config, self._serialize, self._deserialize)
         self.red_teams = RedTeamsOperations(self._client, self._config, self._serialize, self._deserialize)
+        self.evaluation_taxonomies = EvaluationTaxonomiesOperations(
+            self._client, self._config, self._serialize, self._deserialize
+        )
+        self.schedules = SchedulesOperations(self._client, self._config, self._serialize, self._deserialize)
         self.evaluation_results = EvaluationResultsOperations(
             self._client, self._config, self._serialize, self._deserialize
         )
+        self.evaluation_rules = EvaluationRulesOperations(
+            self._client, self._config, self._serialize, self._deserialize
+        )
 
     def send_request(self, request: HttpRequest, *, stream: bool = False, **kwargs: Any) -> HttpResponse:
         """Runs the network request through the client's chained policies.

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/onedp/_configuration.py

@@ -16,30 +16,32 @@
     from azure.core.credentials import TokenCredential
 
 
-class AIProjectClientConfiguration:  # pylint: disable=too-many-instance-attributes
-    """Configuration for AIProjectClient.
+class ProjectsClientConfiguration:  # pylint: disable=too-many-instance-attributes
+    """Configuration for ProjectsClient.
 
     Note that all parameters used to create this instance are saved as instance
     attributes.
 
     :param endpoint: Project endpoint. In the form
-     "https://<your-ai-services-account-name>.services.ai.azure.com/api/projects/_project"
+     "`https://your-ai-services-account-name.services.ai.azure.com/api/projects/_project
+     <https://your-ai-services-account-name.services.ai.azure.com/api/projects/_project>`_"
      if your Foundry Hub has only one Project, or to use the default Project in your Hub. Or in the
      form
-     "https://<your-ai-services-account-name>.services.ai.azure.com/api/projects/<your-project-name>"
+     "`https://your-ai-services-account-name.services.ai.azure.com/api/projects/your-project-name
+     <https://your-ai-services-account-name.services.ai.azure.com/api/projects/your-project-name>`_"
      if you want to explicitly
      specify the Foundry Project name. Required.
     :type endpoint: str
     :param credential: Credential used to authenticate requests to the service. Required.
     :type credential: ~azure.core.credentials.TokenCredential
     :keyword api_version: The API version to use for this operation. Default value is
-     "2025-05-15-preview". Note that overriding this default value may result in unsupported
+     "2025-11-15-preview". Note that overriding this default value may result in unsupported
      behavior.
     :paramtype api_version: str
     """
 
     def __init__(self, endpoint: str, credential: "TokenCredential", **kwargs: Any) -> None:
-        api_version: str = kwargs.pop("api_version", "2025-05-15-preview")
+        api_version: str = kwargs.pop("api_version", "2025-11-15-preview")
 
         if endpoint is None:
             raise ValueError("Parameter 'endpoint' must not be None.")
@@ -50,7 +52,7 @@ def __init__(self, endpoint: str, credential: "TokenCredential", **kwargs: Any)
         self.credential = credential
         self.api_version = api_version
         self.credential_scopes = kwargs.pop("credential_scopes", ["https://ai.azure.com/.default"])
-        kwargs.setdefault("sdk_moniker", "ai-projects-onedp/{}".format(VERSION))
+        kwargs.setdefault("sdk_moniker", "ai-projects/{}".format(VERSION))
         self.polling_interval = kwargs.get("polling_interval", 30)
         self._configure(**kwargs)
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/onedp/_utils/model_base.py

@@ -1,4 +1,4 @@
-# pylint: disable=too-many-lines
+# pylint: disable=line-too-long,useless-suppression,too-many-lines
 # coding=utf-8
 # --------------------------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/onedp/_validation.py

@@ -10,6 +10,22 @@
 def api_version_validation(**kwargs):
     params_added_on = kwargs.pop("params_added_on", {})
     method_added_on = kwargs.pop("method_added_on", "")
+    api_versions_list = kwargs.pop("api_versions_list", [])
+
+    def _index_with_default(value: str, default: int = -1) -> int:
+        """Get the index of value in lst, or return default if not found.
+
+        :param value: The value to search for in the api_versions_list.
+        :type value: str
+        :param default: The default value to return if the value is not found.
+        :type default: int
+        :return: The index of the value in the list, or the default value if not found.
+        :rtype: int
+        """
+        try:
+            return api_versions_list.index(value)
+        except ValueError:
+            return default
 
     def decorator(func):
         @functools.wraps(func)
@@ -21,7 +37,7 @@ def wrapper(*args, **kwargs):
             except AttributeError:
                 return func(*args, **kwargs)
 
-            if method_added_on > client_api_version:
+            if _index_with_default(method_added_on) > _index_with_default(client_api_version):
                 raise ValueError(
                     f"'{func.__name__}' is not available in API version "
                     f"{client_api_version}. Pass service API version {method_added_on} or newer to your client."
@@ -31,7 +47,7 @@ def wrapper(*args, **kwargs):
                 parameter: api_version
                 for api_version, parameters in params_added_on.items()
                 for parameter in parameters
-                if parameter in kwargs and api_version > client_api_version
+                if parameter in kwargs and _index_with_default(api_version) > _index_with_default(client_api_version)
             }
             if unsupported:
                 raise ValueError(

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/onedp/aio/__init__.py

@@ -12,7 +12,7 @@
 if TYPE_CHECKING:
     from ._patch import *  # pylint: disable=unused-wildcard-import
 
-from ._client import AIProjectClient  # type: ignore
+from ._client import ProjectsClient  # type: ignore
 
 try:
     from ._patch import __all__ as _patch_all
@@ -22,7 +22,7 @@
 from ._patch import patch_sdk as _patch_sdk
 
 __all__ = [
-    "AIProjectClient",
+    "ProjectsClient",
 ]
 __all__.extend([p for p in _patch_all if p not in __all__])  # pyright: ignore