diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py index 75e9067e9b47..d34fe28762a4 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py @@ -23,6 +23,7 @@ from ._evaluators._qa import QAEvaluator from ._evaluators._response_completeness import ResponseCompletenessEvaluator from ._evaluators._task_adherence import TaskAdherenceEvaluator +from ._evaluators._task_success import TaskSuccessEvaluator from ._evaluators._relevance import RelevanceEvaluator from ._evaluators._retrieval import RetrievalEvaluator from ._evaluators._rouge import RougeScoreEvaluator, RougeType @@ -104,6 +105,7 @@ def lazy_import(): "GroundednessProEvaluator", "ResponseCompletenessEvaluator", "TaskAdherenceEvaluator", + "TaskSuccessEvaluator", "IntentResolutionEvaluator", "RelevanceEvaluator", "SimilarityEvaluator", diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_eval_mapping.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_eval_mapping.py index 6a4690ccf4eb..2fb716ed48db 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_eval_mapping.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_eval_mapping.py @@ -36,6 +36,7 @@ SexualEvaluator, SimilarityEvaluator, TaskAdherenceEvaluator, + TaskSuccessEvaluator, ToolCallAccuracyEvaluator, UngroundedAttributesEvaluator, ViolenceEvaluator, @@ -67,6 +68,7 @@ SexualEvaluator: "sexual", SimilarityEvaluator: "similarity", TaskAdherenceEvaluator: "task_adherence", + TaskSuccessEvaluator: "task_success", ToolCallAccuracyEvaluator: "tool_call_accuracy", UngroundedAttributesEvaluator: "ungrounded_attributes", ViolenceEvaluator: "violence", diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/__init__.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/__init__.py new file mode 100644 index 000000000000..a20c2d4a8f72 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/__init__.py @@ -0,0 +1,7 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +from ._task_success import TaskSuccessEvaluator + +__all__ = ["TaskSuccessEvaluator"] diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/_task_success.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/_task_success.py new file mode 100644 index 000000000000..92e2f669adcf --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/_task_success.py @@ -0,0 +1,165 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- +import os +import math +import logging +from typing import Dict, Union, List, Optional + +from typing_extensions import overload, override + +from azure.ai.evaluation._exceptions import EvaluationException, ErrorBlame, ErrorCategory, ErrorTarget +from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase +from ..._common.utils import reformat_conversation_history, reformat_agent_response, reformat_tool_definitions +from azure.ai.evaluation._model_configurations import Message +from azure.ai.evaluation._common._experimental import experimental + +logger = logging.getLogger(__name__) + + +@experimental +class TaskSuccessEvaluator(PromptyEvaluatorBase[Union[str, bool]]): + """The Task Success evaluator determines whether an AI agent successfully completed the requested task based on: + + - Final outcome and deliverable of the task + - Completeness of task requirements + + This evaluator focuses solely on task completion and success, not on task adherence or intent understanding. + + Scoring is binary: + - TRUE: Task fully completed with usable deliverable that meets all user requirements + - FALSE: Task incomplete, partially completed, or deliverable does not meet requirements + + The evaluation includes task requirement analysis, outcome assessment, and completion gap identification. + + + :param model_config: Configuration for the Azure OpenAI model. + :type model_config: Union[~azure.ai.evaluation.AzureOpenAIModelConfiguration, + ~azure.ai.evaluation.OpenAIModelConfiguration] + + .. admonition:: Example: + .. literalinclude:: ../samples/evaluation_samples_evaluate.py + :start-after: [START task_success_evaluator] + :end-before: [END task_success_evaluator] + :language: python + :dedent: 8 + :caption: Initialize and call a TaskSuccessEvaluator with a query and response. + + .. admonition:: Example using Azure AI Project URL: + + .. literalinclude:: ../samples/evaluation_samples_evaluate_fdp.py + :start-after: [START task_success_evaluator] + :end-before: [END task_success_evaluator] + :language: python + :dedent: 8 + :caption: Initialize and call TaskSuccessEvaluator using Azure AI Project URL in the following format + https://{resource_name}.services.ai.azure.com/api/projects/{project_name} + + """ + + _PROMPTY_FILE = "task_success.prompty" + _RESULT_KEY = "task_success" + _OPTIONAL_PARAMS = ["tool_definitions"] + + id = "azureai://built-in/evaluators/task_success" + """Evaluator identifier, experimental and to be used only with evaluation in cloud.""" + + @override + def __init__(self, model_config, **kwargs): + current_dir = os.path.dirname(__file__) + prompty_path = os.path.join(current_dir, self._PROMPTY_FILE) + super().__init__(model_config=model_config, prompty_file=prompty_path, result_key=self._RESULT_KEY, **kwargs) + + @overload + def __call__( + self, + *, + query: Union[str, List[dict]], + response: Union[str, List[dict]], + tool_definitions: Optional[Union[dict, List[dict]]] = None, + ) -> Dict[str, Union[str, bool]]: + """Evaluate task success for a given query, response, and optional tool definitions. + The query and response can be either a string or a list of messages. + + + Example with string inputs and no tools: + evaluator = TaskSuccessEvaluator(model_config) + query = "Plan a 3-day itinerary for Paris with cultural landmarks and local cuisine." + response = "**Day 1:** Morning: Louvre Museum, Lunch: Le Comptoir du Relais..." + + result = evaluator(query=query, response=response) + + Example with list of messages: + evaluator = TaskSuccessEvaluator(model_config) + query = [{'role': 'system', 'content': 'You are a helpful travel planning assistant.'}, {'createdAt': 1700000060, 'role': 'user', 'content': [{'type': 'text', 'text': 'Plan a 3-day Paris itinerary with cultural landmarks and cuisine'}]}] + response = [{'createdAt': 1700000070, 'run_id': '0', 'role': 'assistant', 'content': [{'type': 'text', 'text': '**Day 1:** Morning: Visit Louvre Museum (9 AM - 12 PM)...'}]}] + tool_definitions = [{'name': 'get_attractions', 'description': 'Get tourist attractions for a city.', 'parameters': {'type': 'object', 'properties': {'city': {'type': 'string', 'description': 'The city name.'}}}}] + + result = evaluator(query=query, response=response, tool_definitions=tool_definitions) + + :keyword query: The query being evaluated, either a string or a list of messages. + :paramtype query: Union[str, List[dict]] + :keyword response: The response being evaluated, either a string or a list of messages (full agent response potentially including tool calls) + :paramtype response: Union[str, List[dict]] + :keyword tool_definitions: An optional list of messages containing the tool definitions the agent is aware of. + :paramtype tool_definitions: Optional[Union[dict, List[dict]]] + :return: A dictionary with the task success evaluation results. + :rtype: Dict[str, Union[str, bool]] + """ + + @override + def __call__( # pylint: disable=docstring-missing-param + self, + *args, + **kwargs, + ): + """ + Invokes the instance using the overloaded __call__ signature. + + For detailed parameter types and return value documentation, see the overloaded __call__ definition. + """ + return super().__call__(*args, **kwargs) + + @override + async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[bool, str]]: # type: ignore[override] + """Do Task Success evaluation. + :param eval_input: The input to the evaluator. Expected to contain whatever inputs are needed for the _flow method + :type eval_input: Dict + :return: The evaluation result. + :rtype: Dict + """ + # we override the _do_eval method as we want the output to be a dictionary, + # which is a different schema than _base_prompty_eval.py + if "query" not in eval_input and "response" not in eval_input: + raise EvaluationException( + message=f"Both query and response must be provided as input to the Task Success evaluator.", + internal_message=f"Both query and response must be provided as input to the Task Success evaluator.", + blame=ErrorBlame.USER_ERROR, + category=ErrorCategory.MISSING_FIELD, + target=ErrorTarget.TASK_SUCCESS_EVALUATOR, + ) + eval_input["query"] = reformat_conversation_history(eval_input["query"], logger, include_system_messages=True) + eval_input["response"] = reformat_agent_response(eval_input["response"], logger, include_tool_messages=True) + if "tool_definitions" in eval_input and eval_input["tool_definitions"] is not None: + eval_input["tool_definitions"] = reformat_tool_definitions(eval_input["tool_definitions"], logger) + + llm_output = await self._flow(timeout=self._LLM_CALL_TIMEOUT, **eval_input) + if isinstance(llm_output, dict): + success = llm_output.get("success", False) + if isinstance(success, str): + success = success.upper() == "TRUE" + + success_result = "pass" if success == True else "fail" + reason = llm_output.get("explanation", "") + print(f"LLM output: {llm_output}") # Debugging line to check the output structure + return { + f"{self._result_key}": success, + f"{self._result_key}_result": success_result, + f"{self._result_key}_reason": reason, + f"{self._result_key}_task_requirements": llm_output.get("task_requirements", ""), + f"{self._result_key}_delivered_outcome": llm_output.get("delivered_outcome", ""), + f"{self._result_key}_completion_gaps": llm_output.get("completion_gaps", ""), + } + if logger: + logger.warning("LLM output is not a dictionary, returning False for the success.") + return {self._result_key: False} diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/task_success.prompty b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/task_success.prompty new file mode 100644 index 000000000000..1e3e59f08e09 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/task_success.prompty @@ -0,0 +1,213 @@ +--- +name: Task Success +description: Evaluates whether a task was successfully completed +model: + api: chat + parameters: + max_completion_tokens: 1500 + top_p: 1.0 + presence_penalty: 0 + frequency_penalty: 0 + response_format: + type: json_object +inputs: + query: + type: string + response: + type: string + tool_definitions: + default: "[]" +--- +system: +You are an expert evaluator who determines if an agent has successfully completed the task required by the user based on the final outcome. + +user: +ROLE +==== +You are a judge on Task Success who assesses the final outcome of a user-agent interaction. Your single focus is: **Was the user's task successfully and completely accomplished?** + +You are NOT evaluating: +- How well the agent followed instructions (that's Task Adherence) +- How well the agent understood the user's intent (that's Intent Resolution) + +You ARE evaluating: +- Whether the task is actually completed in the final outcome +- Whether the deliverable meets the user's requirements +- Whether the end result is actionable and usable + +INPUT +===== +CONVERSATION_HISTORY: {{query}} +AGENT_RESPONSE: {{response}} + +CONVERSATION_HISTORY includes the full dialogue. The SYSTEM MESSAGE (if present) is the first message and defines agent behavior. +AGENT_RESPONSE is the agent's reply to the latest user query. + +EVALUATION FRAMEWORK +==================== + +A. Identify the Task Requirements: +- What specific outcome did the user request? +- What deliverables were expected? +- What criteria define "completion" for this task? + +B. Analyze the Final Outcome: +- What did the agent actually deliver? +- Is the deliverable complete and usable? +- Does it meet the user's specified requirements? + +C. Assess Task Completion: +- **Fully Complete**: All requirements met, deliverable is ready for use +- **Partially Complete**: Some requirements met, but missing key components +- **Incomplete**: No usable deliverable or major requirements unmet + +D. Assign a Score: + - **TRUE**: The agent delivered a complete and correct solution that accomplishes the user's entire goal. The user does not need to take further action or ask follow-up questions to get what they originally asked for. + - **FALSE**: The agent failed to complete one or more parts of the task, provided an incorrect/incomplete result, or left the user's goal unresolved. + +OUTPUT FORMAT +============= +Output a JSON object with these keys: +{ + "explanation": "<15-60 words explaining the completion status>", + "task_requirements": "<15-60 words on what the user specifically requested>", + "delivered_outcome": "<15-60 words on what the agent actually provided>", + "completion_gaps": "<50-60 words on missing elements if task is incomplete>", + "success": +} + +SCORING EXAMPLES +================ + +### SUCCESS: TRUE - Example A + +CONVERSATION_HISTORY: +User: Plan a 3-day itinerary for Paris with cultural landmarks and local cuisine. + +AGENT_RESPONSE: +**Day 1:** +- Morning: Visit Louvre Museum (9 AM - 12 PM) +- Lunch: Le Comptoir du Relais (traditional French bistro) +- Afternoon: Walk through Tuileries Garden +- Dinner: L'As du Fallafel (famous in Marais district) + +**Day 2:** +- Morning: Tour Eiffel Tower and Trocadéro +- Lunch: Du Pain et des Idées (artisanal bakery) +- Afternoon: Explore Montmartre and Sacré-Cœur +- Dinner: Le Procope (historic restaurant) + +**Day 3:** +- Morning: Visit Notre-Dame Cathedral area +- Lunch: Marché des Enfants Rouges (food market) +- Afternoon: Musée d'Orsay +- Dinner: Guy Savoy (Michelin-starred French cuisine) + +Transportation: Metro Day Pass recommended. Book Louvre tickets online in advance. + +EXPECTED OUTPUT: +{ + "explanation": "Complete 3-day Paris itinerary delivered with cultural landmarks and local cuisine for each day, including practical transportation advice.", + "task_requirements": "3-day Paris itinerary with cultural landmarks and local cuisine", + "delivered_outcome": "Detailed 3-day schedule with specific landmarks, restaurants, and practical tips", + "completion_gaps": "None", + "success": TRUE +} + +### SUCCESS: TRUE - Example B + +CONVERSATION_HISTORY: +User: My internet connection keeps dropping. Help me troubleshoot and then upgrade my data plan to unlimited. + +Agent: Let me help you with both issues. + +First, let's troubleshoot your connection: +[TOOL_CALL] run_network_diagnostic() +[TOOL_RESULT] Issue found: Router firmware outdated, DNS servers slow + +I've identified the problems. Here's what to do: +1. Update router firmware: Go to 192.168.1.1, login, update firmware to version 2.1.4 +2. Change DNS to 8.8.8.8 and 8.8.4.4 + +Now for your data plan upgrade: +[TOOL_CALL] upgrade_plan(customer_id="12345", plan="unlimited") +[TOOL_RESULT] Plan upgraded successfully. Unlimited data active immediately. + +AGENT_RESPONSE: +Your internet issues are resolved - update firmware and change DNS as instructed. Your unlimited data plan is now active. You should see stable connection and no data limits immediately. + +EXPECTED OUTPUT: +{ + "explanation": "Both issues resolved: specific troubleshooting steps provided for internet drops, and unlimited data plan successfully activated.", + "task_requirements": "Fix internet connection issues and upgrade to unlimited data plan", + "delivered_outcome": "Diagnostic completed with specific fix instructions, data plan upgrade confirmed active", + "completion_gaps": "None", + "success": TRUE +} + +### SUCCESS: FALSE - Example A + +CONVERSATION_HISTORY: +User: Plan a 3-day itinerary for Paris with cultural landmarks and local cuisine. + +AGENT_RESPONSE: +I'd recommend visiting the Louvre and Eiffel Tower. Paris has great food - try French pastries and wine. The Metro is convenient for getting around. + +EXPECTED OUTPUT: +{ + "explanation": "Vague suggestions provided but no actual day-by-day itinerary created. Missing specific landmarks, restaurants, and detailed schedule.", + "task_requirements": "3-day Paris itinerary with cultural landmarks and local cuisine", + "delivered_outcome": "General recommendations without structured itinerary", + "completion_gaps": "No day-by-day schedule, no specific restaurant recommendations, no detailed cultural landmarks", + "success": FALSE +} + +### SUCCESS: FALSE - Example B + +CONVERSATION_HISTORY: +User: My internet connection keeps dropping. Help me troubleshoot and then upgrade my data plan to unlimited. + +AGENT_RESPONSE: +[TOOL_CALL] run_network_diagnostic() +[TOOL_RESULT] Issue found: Router firmware outdated + +I found that your router firmware needs updating. You can usually do this through your router's admin panel. Let me know if you need help with the data plan upgrade after you fix the connection. + +EXPECTED OUTPUT: +{ + "explanation": "Internet troubleshooting started but not completed with specific steps. Data plan upgrade not attempted. User cannot proceed without more help.", + "task_requirements": "Fix internet connection issues and upgrade to unlimited data plan", + "delivered_outcome": "Identified router firmware issue but no specific fix instructions, data upgrade not completed", + "completion_gaps": "No specific firmware update steps, data plan upgrade not processed", + "success": FALSE +} + +### SUCCESS: FALSE - Example C + +CONVERSATION_HISTORY: +User: Analyze our Q3 sales data and generate a summary report. + +AGENT_RESPONSE: +I can help you analyze sales data! Sales analysis is important for business growth. You should look at metrics like revenue, customer acquisition, and seasonal trends. Would you like me to explain different types of sales analysis techniques? + +EXPECTED OUTPUT: +{ + "explanation": "No actual analysis performed and no report generated. Agent provided general information about sales analysis instead of completing the requested task.", + "task_requirements": "Analyze Q3 sales data and generate summary report", + "delivered_outcome": "General information about sales analysis concepts", + "completion_gaps": "No data analysis performed, no summary report created", + "success": FALSE +} + +KEY PRINCIPLES +============== + +1. **Outcome Focus**: Judge only the final deliverable, not the process +2. **User Readiness**: Can the user proceed with what was delivered? +3. **Requirement Matching**: Does the outcome match what was specifically requested? +4. **Completeness**: Are all components of the task addressed? +5. **Actionability**: Is the deliverable usable in its current form? + +Remember: A task can be understood correctly and approached properly but still fail if the final outcome doesn't meet requirements. + +# Output \ No newline at end of file diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py index 7850d2f49683..102d0885272c 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py @@ -85,6 +85,7 @@ class ErrorTarget(Enum): FLUENCY_EVALUATOR = "FluencyEvaluator" RETRIEVAL_EVALUATOR = "RetrievalEvaluator" TASK_ADHERENCE_EVALUATOR = "TaskAdherenceEvaluator" + TASK_SUCCESS_EVALUATOR = "TaskSuccessEvaluator" INDIRECT_ATTACK_EVALUATOR = "IndirectAttackEvaluator" INDIRECT_ATTACK_SIMULATOR = "IndirectAttackSimulator" ADVERSARIAL_SIMULATOR = "AdversarialSimulator" diff --git a/sdk/evaluation/azure-ai-evaluation/cspell.json b/sdk/evaluation/azure-ai-evaluation/cspell.json index 10214c4c5b1f..6456144f99f4 100644 --- a/sdk/evaluation/azure-ai-evaluation/cspell.json +++ b/sdk/evaluation/azure-ai-evaluation/cspell.json @@ -19,14 +19,19 @@ "qrels", "ollama", "prompty", - "Likert" + "Likert", + "Comptoir", + "Tuileries", + "Fallafel" ], "ignorePaths": [ "sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/onedp/models/_enums.py", "sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_retrieval/retrieval.prompty", "sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_intent_resolution/intent_resolution.prompty", "sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_data_sources/grounding.json", - "sdk/evaluation/azure-ai-evaluation/samples/data/evaluate_test_data.jsonl" + "sdk/evaluation/azure-ai-evaluation/samples/data/evaluate_test_data.jsonl", + "sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/task_success.prompty", + "sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_success/_task_success.py" ], "words": [ "Aoai", diff --git a/sdk/evaluation/azure-ai-evaluation/samples/agent_evaluators/agent_evaluation.ipynb b/sdk/evaluation/azure-ai-evaluation/samples/agent_evaluators/agent_evaluation.ipynb index d306021a8c84..e648552b8381 100644 --- a/sdk/evaluation/azure-ai-evaluation/samples/agent_evaluators/agent_evaluation.ipynb +++ b/sdk/evaluation/azure-ai-evaluation/samples/agent_evaluators/agent_evaluation.ipynb @@ -355,7 +355,7 @@ "metadata": {}, "outputs": [], "source": [ - "from azure.ai.evaluation import ToolCallAccuracyEvaluator , AzureOpenAIModelConfiguration, IntentResolutionEvaluator, TaskAdherenceEvaluator, ViolenceEvaluator\n", + "from azure.ai.evaluation import ToolCallAccuracyEvaluator , AzureOpenAIModelConfiguration, IntentResolutionEvaluator, TaskAdherenceEvaluator, TaskSuccessEvaluator\n", "from pprint import pprint\n", "\n", "model_config = AzureOpenAIModelConfiguration(\n", @@ -373,7 +373,8 @@ "\n", "tool_call_accuracy = ToolCallAccuracyEvaluator(model_config=model_config)\n", "intent_resolution = IntentResolutionEvaluator(model_config=model_config)\n", - "task_adherence = TaskAdherenceEvaluator(model_config=model_config)\n" + "task_adherence = TaskAdherenceEvaluator(model_config=model_config)\n", + "task_success = TaskSuccessEvaluator(model_config=model_config)" ] }, { @@ -406,6 +407,7 @@ " \"tool_call_accuracy\": tool_call_accuracy,\n", " \"intent_resolution\": intent_resolution,\n", " \"task_adherence\": task_adherence,\n", + " \"task_success\": task_success\n", " },\n", " azure_ai_project={\n", " \"subscription_id\": os.environ[\"AZURE_SUBSCRIPTION_ID\"],\n", diff --git a/sdk/evaluation/azure-ai-evaluation/samples/agent_evaluators/task_success.ipynb b/sdk/evaluation/azure-ai-evaluation/samples/agent_evaluators/task_success.ipynb new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/evaluation/azure-ai-evaluation/samples/evaluation_samples_evaluate.py b/sdk/evaluation/azure-ai-evaluation/samples/evaluation_samples_evaluate.py index f9099c2e706e..db2e194ff581 100644 --- a/sdk/evaluation/azure-ai-evaluation/samples/evaluation_samples_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/samples/evaluation_samples_evaluate.py @@ -95,7 +95,7 @@ def evaluation_evaluate_classes_methods(self): # [START intent_resolution_evaluator] import os - from azure.ai.evaluation import CoherenceEvaluator + from azure.ai.evaluation import IntentResolutionEvaluator model_config = { "azure_endpoint": os.environ.get("AZURE_OPENAI_ENDPOINT"), @@ -422,6 +422,80 @@ def evaluation_evaluate_classes_methods(self): task_adherence_evaluator(query=query, response=response, tool_definitions=tool_definitions) # [END task_adherence_evaluator] + # [START task_success_evaluator] + import os + from azure.ai.evaluation import TaskSuccessEvaluator + + model_config = { + "azure_endpoint": os.environ.get("AZURE_OPENAI_ENDPOINT"), + "api_key": os.environ.get("AZURE_OPENAI_KEY"), + "azure_deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT"), + } + + task_success_evaluator = TaskSuccessEvaluator(model_config=model_config) + + query = [ + {"role": "system", "content": "You are a travel booking assistant. Help users find and book flights."}, + {"role": "user", "content": [{"type": "text", "text": "I need to book a flight from London to Paris for tomorrow"}]}, + ] + + response = [ + { + "role": "assistant", + "content": [ + { + "type": "tool_call", + "tool_call": { + "id": "search_001", + "type": "function", + "function": { + "name": "search_flights", + "arguments": { + "origin": "London", + "destination": "Paris", + "departure_date": "2025-08-13" + } + }, + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "search_001", + "content": [ + {"type": "tool_result", "tool_result": '{"flights": [{"flight_id": "BA309", "price": "£89", "departure": "10:30", "arrival": "13:45"}, {"flight_id": "AF1234", "price": "£95", "departure": "14:20", "arrival": "17:35"}]}'} + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I found 2 flights from London to Paris for tomorrow:\n\n1. BA309 departing 10:30, arriving 13:45 - £89\n2. AF1234 departing 14:20, arriving 17:35 - £95\n\nWould you like me to book one of these flights for you?" + } + ], + }, + ] + + tool_definitions = [ + { + "name": "search_flights", + "description": "Search for available flights between two cities.", + "parameters": { + "type": "object", + "properties": { + "origin": {"type": "string", "description": "Departure city"}, + "destination": {"type": "string", "description": "Arrival city"}, + "departure_date": {"type": "string", "description": "Departure date in YYYY-MM-DD format"} + } + }, + } + ] + + task_success_evaluator(query=query, response=response, tool_definitions=tool_definitions) + # [END task_success_evaluator] + # [START indirect_attack_evaluator] import os from azure.identity import DefaultAzureCredential diff --git a/sdk/evaluation/azure-ai-evaluation/samples/evaluation_samples_evaluate_fdp.py b/sdk/evaluation/azure-ai-evaluation/samples/evaluation_samples_evaluate_fdp.py index f50fe0e77231..91e64cfb6b88 100644 --- a/sdk/evaluation/azure-ai-evaluation/samples/evaluation_samples_evaluate_fdp.py +++ b/sdk/evaluation/azure-ai-evaluation/samples/evaluation_samples_evaluate_fdp.py @@ -432,6 +432,80 @@ def evaluation_evaluate_classes_methods(self): task_adherence_evaluator(query=query, response=response, tool_definitions=tool_definitions) # [END task_adherence_evaluator] + # [START task_success_evaluator] + import os + from azure.ai.evaluation import TaskSuccessEvaluator + + model_config = { + "azure_endpoint": os.environ.get("AZURE_OPENAI_ENDPOINT"), # https://.services.ai.azure.com + "api_key": os.environ.get("AZURE_OPENAI_KEY"), + "azure_deployment": os.environ.get("AZURE_OPENAI_DEPLOYMENT"), + } + + task_success_evaluator = TaskSuccessEvaluator(model_config=model_config) + + query = [ + {"role": "system", "content": "You are a travel booking assistant. Help users find and book flights."}, + {"role": "user", "content": [{"type": "text", "text": "I need to book a flight from London to Paris for tomorrow"}]}, + ] + + response = [ + { + "role": "assistant", + "content": [ + { + "type": "tool_call", + "tool_call": { + "id": "search_001", + "type": "function", + "function": { + "name": "search_flights", + "arguments": { + "origin": "London", + "destination": "Paris", + "departure_date": "2025-08-13" + } + }, + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "search_001", + "content": [ + {"type": "tool_result", "tool_result": '{"flights": [{"flight_id": "BA309", "price": "£89", "departure": "10:30", "arrival": "13:45"}, {"flight_id": "AF1234", "price": "£95", "departure": "14:20", "arrival": "17:35"}]}'} + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I found 2 flights from London to Paris for tomorrow:\n\n1. BA309 departing 10:30, arriving 13:45 - £89\n2. AF1234 departing 14:20, arriving 17:35 - £95\n\nWould you like me to book one of these flights for you?" + } + ], + }, + ] + + tool_definitions = [ + { + "name": "search_flights", + "description": "Search for available flights between two cities.", + "parameters": { + "type": "object", + "properties": { + "origin": {"type": "string", "description": "Departure city"}, + "destination": {"type": "string", "description": "Arrival city"}, + "departure_date": {"type": "string", "description": "Departure date in YYYY-MM-DD format"} + } + }, + } + ] + + task_success_evaluator(query=query, response=response, tool_definitions=tool_definitions) + # [END task_success_evaluator] + # [START indirect_attack_evaluator] import os from azure.identity import DefaultAzureCredential