Add support for Agent Browser Automation tool (#36462)

Author: dargilco

specification/ai/Azure.AI.Agents/tools/models.tsp

@@ -208,6 +208,31 @@ model AzureFunctionToolDefinition extends ToolDefinition {
   azureFunction: AzureFunctionDefinition;
 }
 
+@doc("The input definition information for a Browser Automation Tool, as used to configure an Agent.")
+@added(Versions.v2025_05_15_preview)
+model BrowserAutomationToolDefinition extends ToolDefinition {
+  @doc("The object type, which is always 'browser_automation'.")
+  type: "browser_automation";
+
+  @doc("The Browser Automation Tool parameters.")
+  @encodedName("application/json", "browser_automation")
+  browserAutomation: BrowserAutomationToolParameters;
+}
+
+@doc("Definition of input parameters for the Browser Automation Tool.")
+@added(Versions.v2025_05_15_preview)
+model BrowserAutomationToolParameters {
+  @doc("The connection parameters associated with the Browser Automation Tool.")
+  connection: BrowserAutomationToolConnectionParameters;
+}
+
+@doc("Definition of input parameters for the connection used by the Browser Automation Tool.")
+@added(Versions.v2025_05_15_preview)
+model BrowserAutomationToolConnectionParameters {
+  @doc("The ID of the connection to your Azure Playwright resource.")
+  id: string;
+}
+
 /**
  * Ranking options for file search.
  */
@@ -513,6 +538,52 @@ model RunStepAzureAISearchToolCall extends RunStepToolCall {
   azureAISearch: Record<string>;
 }
 
+@doc("""
+  A record of a call to a Browser Automation tool issued by the Agent.
+  """)
+@added(Versions.v2025_05_15_preview)
+model RunStepBrowserAutomationToolCall extends RunStepToolCall {
+  @doc("The object type, which is always 'browser_automation'.")
+  type: "browser_automation";
+
+  @doc("Details of the browser automation tool call.")
+  @encodedName("application/json", "browser_automation")
+  browserAutomation: BrowserAutomationToolCallDetails;
+}
+
+@doc("""
+  Details of a Browser Automation tool call.
+  """)
+@added(Versions.v2025_05_15_preview)
+model BrowserAutomationToolCallDetails {
+  @doc("The input provided to the Browser Automation tool.")
+  input: string;
+
+  @doc("The output returned by the Browser Automation tool.")
+  output: string;
+
+  @doc("The steps the Browser Automation tool executed.")
+  steps: BrowserAutomationToolCallStep[];
+}
+
+@doc("""
+  Describes a single step of a Browser Automation tool execution.
+  """)
+@added(Versions.v2025_05_15_preview)
+model BrowserAutomationToolCallStep {
+  @doc("The result of the last step executed with the Browser.")
+  @encodedName("application/json", "last_step_result")
+  lastStepResult: string;
+
+  @doc("The current state of execution with the Browser.")
+  @encodedName("application/json", "current_state")
+  currentState: string;
+
+  @doc("The next step to execute with the Browser.")
+  @encodedName("application/json", "next_step")
+  nextStep: string;
+}
+
 @doc("""
   A record of a call to a MCP tool, issued by the model in evaluation of a defined tool, that represents
   executed MCP actions.

specification/ai/data-plane/Azure.AI.Agents/preview/2025-05-15-preview/azure-ai-agents.json

@@ -3643,6 +3643,104 @@
       ],
       "x-ms-discriminator-value": "bing_grounding"
     },
+    "BrowserAutomationToolCallDetails": {
+      "type": "object",
+      "description": "Details of a Browser Automation tool call.",
+      "properties": {
+        "input": {
+          "type": "string",
+          "description": "The input provided to the Browser Automation tool."
+        },
+        "output": {
+          "type": "string",
+          "description": "The output returned by the Browser Automation tool."
+        },
+        "steps": {
+          "type": "array",
+          "description": "The steps the Browser Automation tool executed.",
+          "items": {
+            "$ref": "#/definitions/BrowserAutomationToolCallStep"
+          }
+        }
+      },
+      "required": [
+        "input",
+        "output",
+        "steps"
+      ]
+    },
+    "BrowserAutomationToolCallStep": {
+      "type": "object",
+      "description": "Describes a single step of a Browser Automation tool execution.",
+      "properties": {
+        "last_step_result": {
+          "type": "string",
+          "description": "The result of the last step executed with the Browser.",
+          "x-ms-client-name": "lastStepResult"
+        },
+        "current_state": {
+          "type": "string",
+          "description": "The current state of execution with the Browser.",
+          "x-ms-client-name": "currentState"
+        },
+        "next_step": {
+          "type": "string",
+          "description": "The next step to execute with the Browser.",
+          "x-ms-client-name": "nextStep"
+        }
+      },
+      "required": [
+        "last_step_result",
+        "current_state",
+        "next_step"
+      ]
+    },
+    "BrowserAutomationToolConnectionParameters": {
+      "type": "object",
+      "description": "Definition of input parameters for the connection used by the Browser Automation Tool.",
+      "properties": {
+        "id": {
+          "type": "string",
+          "description": "The ID of the connection to your Azure Playwright resource."
+        }
+      },
+      "required": [
+        "id"
+      ]
+    },
+    "BrowserAutomationToolDefinition": {
+      "type": "object",
+      "description": "The input definition information for a Browser Automation Tool, as used to configure an Agent.",
+      "properties": {
+        "browser_automation": {
+          "$ref": "#/definitions/BrowserAutomationToolParameters",
+          "description": "The Browser Automation Tool parameters.",
+          "x-ms-client-name": "browserAutomation"
+        }
+      },
+      "required": [
+        "browser_automation"
+      ],
+      "allOf": [
+        {
+          "$ref": "#/definitions/ToolDefinition"
+        }
+      ],
+      "x-ms-discriminator-value": "browser_automation"
+    },
+    "BrowserAutomationToolParameters": {
+      "type": "object",
+      "description": "Definition of input parameters for the Browser Automation Tool.",
+      "properties": {
+        "connection": {
+          "$ref": "#/definitions/BrowserAutomationToolConnectionParameters",
+          "description": "The connection parameters associated with the Browser Automation Tool."
+        }
+      },
+      "required": [
+        "connection"
+      ]
+    },
     "CodeInterpreterToolDefinition": {
       "type": "object",
       "description": "The input definition information for a code interpreter tool as used to configure an agent.",
@@ -6213,6 +6311,26 @@
       ],
       "x-ms-discriminator-value": "bing_grounding"
     },
+    "RunStepBrowserAutomationToolCall": {
+      "type": "object",
+      "description": "A record of a call to a Browser Automation tool issued by the Agent.",
+      "properties": {
+        "browser_automation": {
+          "$ref": "#/definitions/BrowserAutomationToolCallDetails",
+          "description": "Details of the browser automation tool call.",
+          "x-ms-client-name": "browserAutomation"
+        }
+      },
+      "required": [
+        "browser_automation"
+      ],
+      "allOf": [
+        {
+          "$ref": "#/definitions/RunStepToolCall"
+        }
+      ],
+      "x-ms-discriminator-value": "browser_automation"
+    },
     "RunStepCodeInterpreterImageOutput": {
       "type": "object",
       "description": "A representation of an image output emitted by a code interpreter tool in response to a tool call by the model.",