Add support for Built-in Tools for the Tool Call Accuracy Evaluator (#42359)

* support 5 levels, evaluate all tools at once

* Update sample notebook and change log

* Add missing import

* Modify test cases to match the new output format

* Modify other test file to match the new output format

* Fixed parsing of results

* Change key name in output

* Spell check fixes

* Minor prompt update

* Update result key to tool_call_accuracy

* Delete test_new_evaluator.ipynb

* Added field names and messages as constants

* Additional note in prompt

* Re-add the temperature to the prompty file

* Removed 'applicable' field and print statement

* Move excess/missing tool calls fields under additional details

* Typo fix and removal of redundant field in the prompt

* Modify per_tool_call_details field's name to details

* Revert "Modify per_tool_call_details field's name to details"

This reverts commit 2c3ce50.

* Revert 'Merge branch 'main' into selshafey/improve_tool_call_accuracy'

* Black reformat

* Reformat with black

* To re-trigger build pipelines

* Add notebook for bugbash

* modify bugbash notebook

* Add support for built-in tools for Tool Call Accuracy Evaluator

* Remove bugbash notebook

* Resolve issues with merge

* Fix id value

* Use existing built-in tool definitions

* Run black

* Prompt modifications

* Add test cases for built-in tools

* Handle converter format

* Add test cases for converter format

* Support only converter format

* Revert tool definitions to be required, run black

---------

Co-authored-by: Salma Elshafey <[email protected]>

Author: salma-elshafey

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_converters/_models.py

@@ -33,9 +33,12 @@
 # Constants to only be used internally in this file for the built-in tools.
 _CODE_INTERPRETER = "code_interpreter"
 _BING_GROUNDING = "bing_grounding"
+_BING_CUSTOM_SEARCH = "bing_custom_search"
 _FILE_SEARCH = "file_search"
 _AZURE_AI_SEARCH = "azure_ai_search"
+_SHAREPOINT_GROUNDING = "sharepoint_grounding"
 _FABRIC_DATAAGENT = "fabric_dataagent"
+_OPENAPI = "openapi"
 
 # Built-in tool descriptions and parameters are hidden, but we include basic descriptions
 # for evaluation purposes.
@@ -44,9 +47,12 @@
     + "generate code, and create graphs and charts using your data. Supports "
     + "up to 20 files.",
     _BING_GROUNDING: "Enhance model output with web data.",
-    _FILE_SEARCH: "Search for data across uploaded files.",
+    _BING_CUSTOM_SEARCH: "Enables agents to retrieve content from a curated subset of websites, enhancing relevance and reducing noise from public web searches.",
+    _FILE_SEARCH: "Search for data across uploaded files. A single call can return multiple results/files in the 'results' field.",
     _AZURE_AI_SEARCH: "Search an Azure AI Search index for relevant data.",
+    _SHAREPOINT_GROUNDING: "Allows agents to access and retrieve relevant content from Microsoft SharePoint document libraries, grounding responses in organizational knowledge.",
     _FABRIC_DATAAGENT: "Connect to Microsoft Fabric data agents to retrieve data across different data sources.",
+    _OPENAPI: "Connects agents to external RESTful APIs using OpenAPI 3.0 specifications, enabling seamless access to third-party services.",
 }
 
 # Built-in tool parameters are hidden, but we include basic parameters for evaluation purposes.
@@ -59,6 +65,15 @@
         "type": "object",
         "properties": {"requesturl": {"type": "string", "description": "URL used in Bing Search API."}},
     },
+    _BING_CUSTOM_SEARCH: {
+        "type": "object",
+        "properties": {
+            "requesturl": {
+                "type": "string",
+                "description": "Search queries, along with pre-configured site restrictions or domain filters.",
+            }
+        },
+    },
     _FILE_SEARCH: {
         "type": "object",
         "properties": {
@@ -76,10 +91,23 @@
         "type": "object",
         "properties": {"input": {"type": "string", "description": "Search terms to use."}},
     },
+    _SHAREPOINT_GROUNDING: {
+        "type": "object",
+        "properties": {
+            "input": {"type": "string", "description": "A natural language query to search SharePoint content."}
+        },
+    },
     _FABRIC_DATAAGENT: {
         "type": "object",
         "properties": {"input": {"type": "string", "description": "Search terms to use."}},
     },
+    _OPENAPI: {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string", "description": "The name of the function to call."},
+            "arguments": {"type": "string", "description": "JSON string of the arguments to pass to the function."},
+        },
+    },
 }
 
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py

@@ -5,7 +5,7 @@
 import os
 import logging
 import re
-from typing import Dict, List, Union, TypeVar, cast
+from typing import Dict, List, Union, TypeVar, Optional
 from typing_extensions import overload, override
 from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase
 from azure.ai.evaluation._exceptions import (
@@ -16,12 +16,46 @@
 )
 from ..._common.utils import check_score_is_valid
 from azure.ai.evaluation._common._experimental import experimental
+from ..._converters._models import (
+    _BUILT_IN_DESCRIPTIONS,
+    _BUILT_IN_PARAMS,
+)
 
 logger = logging.getLogger(__name__)
 
 T_EvalValue = TypeVar("T_EvalValue")
 
 
+def _get_built_in_definition(tool_name: str):
+    """Get the definition for the built-in tool."""
+    if tool_name in _BUILT_IN_DESCRIPTIONS:
+        return {
+            "type": tool_name,
+            "description": _BUILT_IN_DESCRIPTIONS[tool_name],
+            "name": tool_name,
+            "parameters": _BUILT_IN_PARAMS.get(tool_name, {}),
+        }
+    return None
+
+
+def _get_needed_built_in_definitions(tool_calls: List[Dict]) -> List[Dict]:
+    """Extract tool definitions needed for the given built-in tool calls."""
+    needed_definitions = []
+    for tool_call in tool_calls:
+        if isinstance(tool_call, dict):
+            tool_type = tool_call.get("type")
+
+            # Only support converter format: {type: "tool_call", name: "bing_custom_search", arguments: {...}}
+            if tool_type == "tool_call":
+                tool_name = tool_call.get("name")
+                if tool_name in _BUILT_IN_DESCRIPTIONS:
+                    built_in_def = _get_built_in_definition(tool_name)
+                    if built_in_def and built_in_def not in needed_definitions:
+                        needed_definitions.append(built_in_def)
+
+    return needed_definitions
+
+
 @experimental
 class ToolCallAccuracyEvaluator(PromptyEvaluatorBase[Union[str, float]]):
     """The Tool Call Accuracy evaluator assesses how accurately an AI uses tools by examining:
@@ -153,10 +187,9 @@ def _convert_kwargs_to_eval_input(self, **kwargs):
         # TODO add warning that only tool calls of type function are supported
         # Collect inputs
         tool_calls = kwargs.get("tool_calls")
-        tool_definitions = kwargs.get("tool_definitions")
+        tool_definitions = kwargs.get("tool_definitions", [])  # Default to empty list
         query = kwargs.get("query")
         response = kwargs.get("response")
-
         # TODO : Support classes that represents tool calls, messages etc once client side definitions are available
         if response:
             parsed_tool_calls = self._parse_tools_from_response(response)
@@ -165,20 +198,23 @@ def _convert_kwargs_to_eval_input(self, **kwargs):
 
         if not tool_calls:
             return {"error_message": self._NO_TOOL_CALLS_MESSAGE}
-        if not tool_definitions or len(tool_definitions) == 0:
-            return {"error_message": self._NO_TOOL_DEFINITIONS_MESSAGE}
 
         if not isinstance(tool_calls, list):
             tool_calls = [tool_calls]
         if not isinstance(tool_definitions, list):
-            tool_definitions = [tool_definitions]
+            tool_definitions = [tool_definitions] if tool_definitions else []
 
         try:
             needed_tool_definitions = self._extract_needed_tool_definitions(tool_calls, tool_definitions)
         except EvaluationException as e:
-            return {"error_message": self._TOOL_DEFINITIONS_MISSING_MESSAGE}
+            # Check if this is because no tool definitions were provided at all
+            if len(tool_definitions) == 0:
+                return {"error_message": self._NO_TOOL_DEFINITIONS_MESSAGE}
+            else:
+                return {"error_message": self._TOOL_DEFINITIONS_MISSING_MESSAGE}
+
         if len(needed_tool_definitions) == 0:
-            return {"error_message": self._TOOL_DEFINITIONS_MISSING_MESSAGE}
+            return {"error_message": self._NO_TOOL_DEFINITIONS_MESSAGE}
 
         return {
             "query": query,
@@ -269,32 +305,63 @@ def _not_applicable_result(self, error_message):
         }
 
     def _extract_needed_tool_definitions(self, tool_calls, tool_definitions):
-        """Extract the tool definitions that are needed for the provided tool calls.
-        :param tool_calls: List of tool calls to evaluate.
-        :type tool_calls: List[dict]
-        :param tool_definitions: List of tool definitions to use for evaluation.
-        :type tool_definitions: List[dict]
-        :return: List of tool definitions that are needed for the provided tool calls.
-        :rtype: List[dict]
-        """
+        """Extract the tool definitions that are needed for the provided tool calls."""
         needed_tool_definitions = []
+
+        # Add all user-provided tool definitions
+        needed_tool_definitions.extend(tool_definitions)
+
+        # Add the needed built-in tool definitions (if they are called)
+        built_in_definitions = _get_needed_built_in_definitions(tool_calls)
+        needed_tool_definitions.extend(built_in_definitions)
+
+        # Validate that all tool calls have corresponding definitions
         for tool_call in tool_calls:
-            if isinstance(tool_call, dict) and tool_call.get("type") == "tool_call":
-                tool_name = tool_call.get("name")
-                tool_definition = [
-                    tool
-                    for tool in tool_definitions
-                    if tool.get("name") == tool_name and tool.get("type", "function") == "function"
-                ]
-                if len(tool_definition) > 0:
-                    needed_tool_definitions.extend(tool_definition)
+            if isinstance(tool_call, dict):
+                tool_type = tool_call.get("type")
+
+                if tool_type == "tool_call":
+                    tool_name = tool_call.get("name")
+                    if tool_name and tool_name in _BUILT_IN_DESCRIPTIONS:
+                        # This is a built-in tool from converter, already handled above
+                        continue
+                    elif tool_name:
+                        # This is a regular function tool from converter
+                        tool_definition_exists = any(
+                            tool.get("name") == tool_name and tool.get("type", "function") == "function"
+                            for tool in tool_definitions
+                        )
+                        if not tool_definition_exists:
+                            raise EvaluationException(
+                                message=f"Tool definition for {tool_name} not found",
+                                blame=ErrorBlame.USER_ERROR,
+                                category=ErrorCategory.INVALID_VALUE,
+                                target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
+                            )
+                    else:
+                        raise EvaluationException(
+                            message=f"Tool call missing name: {tool_call}",
+                            blame=ErrorBlame.USER_ERROR,
+                            category=ErrorCategory.INVALID_VALUE,
+                            target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
+                        )
                 else:
+                    # Unsupported tool format - only converter format is supported
                     raise EvaluationException(
-                        message=f"Tool definition for {tool_name} not found",
+                        message=f"Unsupported tool call format. Only converter format is supported: {tool_call}",
                         blame=ErrorBlame.USER_ERROR,
                         category=ErrorCategory.INVALID_VALUE,
                         target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
                     )
+            else:
+                # Tool call is not a dictionary
+                raise EvaluationException(
+                    message=f"Tool call is not a dictionary: {tool_call}",
+                    blame=ErrorBlame.USER_ERROR,
+                    category=ErrorCategory.INVALID_VALUE,
+                    target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
+                )
+
         return needed_tool_definitions
 
     @override