Skip to content

Commit 57b0c21

Browse files
authored
Add support for Browser Automation tool (#42392)
1 parent 5530ccb commit 57b0c21

17 files changed

+645
-11
lines changed

sdk/ai/azure-ai-agents/CHANGELOG.md

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,16 @@
11

2-
32
# Release History
43

4+
## 1.2.0b2 (Unreleased)
5+
6+
### Features Added
7+
8+
- Add support for Browser Automation tool.
9+
10+
### Sample updates
11+
12+
- Add new samples `sample_agents_browser_automation.py` and `sample_agents_browser_automation_async.py`.
13+
514
## 1.2.0b1 (2025-08-05)
615

716
**Beta Features Restored**: This release reintroduces all experimental features that were available in the 1.1.0b series but removed in the 1.1.0 stable release.

sdk/ai/azure-ai-agents/README.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ To report an issue with the client library, or request additional features, plea
3636
- [Function call](#create-agent-with-function-call)
3737
- [Azure Function Call](#create-agent-with-azure-function-call)
3838
- [OpenAPI](#create-agent-with-openapi)
39+
- [Browser Automation](#create-agent-with-browser-automation)
3940
- [Fabric data](#create-an-agent-with-fabric)
4041
- [Connected agents](#create-an-agent-using-another-agents)
4142
- [Deep Research](#create-agent-with-deep-research)
@@ -861,6 +862,41 @@ with project_client:
861862

862863
<!-- END SNIPPET -->
863864

865+
### Create Agent with Browser Automation
866+
867+
To enable your Agent to perform automated Browser navigation tasks, you will need the `BrowserAutomationTool`, along with a connection to
868+
a [Microsoft Playwright Workspace](https://azure.microsoft.com/products/playwright-testing) resource.
869+
870+
Here is an example:
871+
872+
<!-- SNIPPET:sample_agents_browser_automation.create_agent_with_browser_automation -->
873+
874+
```python
875+
connection_id = os.environ["AZURE_PLAYWRIGHT_CONNECTION_ID"]
876+
877+
# Initialize Browser Automation tool and add the connection id
878+
browser_automation = BrowserAutomationTool(connection_id=connection_id)
879+
880+
with project_client:
881+
882+
agents_client = project_client.agents
883+
884+
# Create a new Agent that has the Browser Automation tool attached.
885+
# Note: To add Browser Automation tool to an existing Agent with an `agent_id`, do the following:
886+
# agent = agents_client.update_agent(agent_id, tools=browser_automation.definitions)
887+
agent = agents_client.create_agent(
888+
model=os.environ["MODEL_DEPLOYMENT_NAME"],
889+
name="my-agent",
890+
instructions="""
891+
You are an Agent helping with browser automation tasks.
892+
You can answer questions, provide information, and assist with various tasks
893+
related to web browsing using the Browser Automation tool available to you.
894+
""",
895+
tools=browser_automation.definitions,
896+
)
897+
```
898+
899+
<!-- END SNIPPET -->
864900

865901
### Create an Agent with Fabric
866902

sdk/ai/azure-ai-agents/apiview-properties.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@
2222
"azure.ai.agents.models.BingGroundingSearchConfiguration": "Azure.AI.Agents.BingGroundingSearchConfiguration",
2323
"azure.ai.agents.models.BingGroundingSearchToolParameters": "Azure.AI.Agents.BingGroundingSearchToolParameters",
2424
"azure.ai.agents.models.BingGroundingToolDefinition": "Azure.AI.Agents.BingGroundingToolDefinition",
25+
"azure.ai.agents.models.BrowserAutomationToolCallDetails": "Azure.AI.Agents.BrowserAutomationToolCallDetails",
26+
"azure.ai.agents.models.BrowserAutomationToolCallStep": "Azure.AI.Agents.BrowserAutomationToolCallStep",
27+
"azure.ai.agents.models.BrowserAutomationToolConnectionParameters": "Azure.AI.Agents.BrowserAutomationToolConnectionParameters",
28+
"azure.ai.agents.models.BrowserAutomationToolDefinition": "Azure.AI.Agents.BrowserAutomationToolDefinition",
29+
"azure.ai.agents.models.BrowserAutomationToolParameters": "Azure.AI.Agents.BrowserAutomationToolParameters",
2530
"azure.ai.agents.models.CodeInterpreterToolDefinition": "Azure.AI.Agents.CodeInterpreterToolDefinition",
2631
"azure.ai.agents.models.CodeInterpreterToolResource": "Azure.AI.Agents.CodeInterpreterToolResource",
2732
"azure.ai.agents.models.ConnectedAgentDetails": "Azure.AI.Agents.ConnectedAgentDetails",
@@ -102,6 +107,7 @@
102107
"azure.ai.agents.models.RunStepAzureAISearchToolCall": "Azure.AI.Agents.RunStepAzureAISearchToolCall",
103108
"azure.ai.agents.models.RunStepBingCustomSearchToolCall": "Azure.AI.Agents.RunStepBingCustomSearchToolCall",
104109
"azure.ai.agents.models.RunStepBingGroundingToolCall": "Azure.AI.Agents.RunStepBingGroundingToolCall",
110+
"azure.ai.agents.models.RunStepBrowserAutomationToolCall": "Azure.AI.Agents.RunStepBrowserAutomationToolCall",
105111
"azure.ai.agents.models.RunStepCodeInterpreterToolCallOutput": "Azure.AI.Agents.RunStepCodeInterpreterToolCallOutput",
106112
"azure.ai.agents.models.RunStepCodeInterpreterImageOutput": "Azure.AI.Agents.RunStepCodeInterpreterImageOutput",
107113
"azure.ai.agents.models.RunStepCodeInterpreterImageReference": "Azure.AI.Agents.RunStepCodeInterpreterImageReference",

sdk/ai/azure-ai-agents/azure/ai/agents/_version.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@
66
# Changes may cause incorrect behavior and will be lost if the code is regenerated.
77
# --------------------------------------------------------------------------
88

9-
VERSION = "1.2.0b1"
9+
VERSION = "1.2.0b2"

sdk/ai/azure-ai-agents/azure/ai/agents/models/__init__.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,11 @@
3434
BingGroundingSearchConfiguration,
3535
BingGroundingSearchToolParameters,
3636
BingGroundingToolDefinition,
37+
BrowserAutomationToolCallDetails,
38+
BrowserAutomationToolCallStep,
39+
BrowserAutomationToolConnectionParameters,
40+
BrowserAutomationToolDefinition,
41+
BrowserAutomationToolParameters,
3742
CodeInterpreterToolDefinition,
3843
CodeInterpreterToolResource,
3944
ConnectedAgentDetails,
@@ -113,6 +118,7 @@
113118
RunStepAzureAISearchToolCall,
114119
RunStepBingCustomSearchToolCall,
115120
RunStepBingGroundingToolCall,
121+
RunStepBrowserAutomationToolCall,
116122
RunStepCodeInterpreterImageOutput,
117123
RunStepCodeInterpreterImageReference,
118124
RunStepCodeInterpreterLogOutput,
@@ -257,6 +263,11 @@
257263
"BingGroundingSearchConfiguration",
258264
"BingGroundingSearchToolParameters",
259265
"BingGroundingToolDefinition",
266+
"BrowserAutomationToolCallDetails",
267+
"BrowserAutomationToolCallStep",
268+
"BrowserAutomationToolConnectionParameters",
269+
"BrowserAutomationToolDefinition",
270+
"BrowserAutomationToolParameters",
260271
"CodeInterpreterToolDefinition",
261272
"CodeInterpreterToolResource",
262273
"ConnectedAgentDetails",
@@ -336,6 +347,7 @@
336347
"RunStepAzureAISearchToolCall",
337348
"RunStepBingCustomSearchToolCall",
338349
"RunStepBingGroundingToolCall",
350+
"RunStepBrowserAutomationToolCall",
339351
"RunStepCodeInterpreterImageOutput",
340352
"RunStepCodeInterpreterImageReference",
341353
"RunStepCodeInterpreterLogOutput",

sdk/ai/azure-ai-agents/azure/ai/agents/models/_models.py

Lines changed: 220 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -511,9 +511,10 @@ class ToolDefinition(_Model):
511511
512512
You probably want to use the sub-classes and not this class directly. Known sub-classes are:
513513
AzureAISearchToolDefinition, AzureFunctionToolDefinition, BingCustomSearchToolDefinition,
514-
BingGroundingToolDefinition, CodeInterpreterToolDefinition, ConnectedAgentToolDefinition,
515-
DeepResearchToolDefinition, MicrosoftFabricToolDefinition, FileSearchToolDefinition,
516-
FunctionToolDefinition, MCPToolDefinition, OpenApiToolDefinition, SharepointToolDefinition
514+
BingGroundingToolDefinition, BrowserAutomationToolDefinition, CodeInterpreterToolDefinition,
515+
ConnectedAgentToolDefinition, DeepResearchToolDefinition, MicrosoftFabricToolDefinition,
516+
FileSearchToolDefinition, FunctionToolDefinition, MCPToolDefinition, OpenApiToolDefinition,
517+
SharepointToolDefinition
517518
518519
:ivar type: The object type. Required. Default value is None.
519520
:vartype type: str
@@ -1005,6 +1006,179 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
10051006
super().__init__(*args, type="bing_grounding", **kwargs)
10061007

10071008

1009+
class BrowserAutomationToolCallDetails(_Model):
1010+
"""Details of a Browser Automation tool call.
1011+
1012+
:ivar input: The input provided to the Browser Automation tool. Required.
1013+
:vartype input: str
1014+
:ivar output: The output returned by the Browser Automation tool. Required.
1015+
:vartype output: str
1016+
:ivar steps: The steps the Browser Automation tool executed. Required.
1017+
:vartype steps: list[~azure.ai.agents.models.BrowserAutomationToolCallStep]
1018+
"""
1019+
1020+
input: str = rest_field(visibility=["read", "create", "update", "delete", "query"])
1021+
"""The input provided to the Browser Automation tool. Required."""
1022+
output: str = rest_field(visibility=["read", "create", "update", "delete", "query"])
1023+
"""The output returned by the Browser Automation tool. Required."""
1024+
steps: List["_models.BrowserAutomationToolCallStep"] = rest_field(
1025+
visibility=["read", "create", "update", "delete", "query"]
1026+
)
1027+
"""The steps the Browser Automation tool executed. Required."""
1028+
1029+
@overload
1030+
def __init__(
1031+
self,
1032+
*,
1033+
input: str,
1034+
output: str,
1035+
steps: List["_models.BrowserAutomationToolCallStep"],
1036+
) -> None: ...
1037+
1038+
@overload
1039+
def __init__(self, mapping: Mapping[str, Any]) -> None:
1040+
"""
1041+
:param mapping: raw JSON to initialize the model.
1042+
:type mapping: Mapping[str, Any]
1043+
"""
1044+
1045+
def __init__(self, *args: Any, **kwargs: Any) -> None:
1046+
super().__init__(*args, **kwargs)
1047+
1048+
1049+
class BrowserAutomationToolCallStep(_Model):
1050+
"""Describes a single step of a Browser Automation tool execution.
1051+
1052+
:ivar last_step_result: The result of the last step executed with the Browser. Required.
1053+
:vartype last_step_result: str
1054+
:ivar current_state: The current state of execution with the Browser. Required.
1055+
:vartype current_state: str
1056+
:ivar next_step: The next step to execute with the Browser. Required.
1057+
:vartype next_step: str
1058+
"""
1059+
1060+
last_step_result: str = rest_field(visibility=["read", "create", "update", "delete", "query"])
1061+
"""The result of the last step executed with the Browser. Required."""
1062+
current_state: str = rest_field(visibility=["read", "create", "update", "delete", "query"])
1063+
"""The current state of execution with the Browser. Required."""
1064+
next_step: str = rest_field(visibility=["read", "create", "update", "delete", "query"])
1065+
"""The next step to execute with the Browser. Required."""
1066+
1067+
@overload
1068+
def __init__(
1069+
self,
1070+
*,
1071+
last_step_result: str,
1072+
current_state: str,
1073+
next_step: str,
1074+
) -> None: ...
1075+
1076+
@overload
1077+
def __init__(self, mapping: Mapping[str, Any]) -> None:
1078+
"""
1079+
:param mapping: raw JSON to initialize the model.
1080+
:type mapping: Mapping[str, Any]
1081+
"""
1082+
1083+
def __init__(self, *args: Any, **kwargs: Any) -> None:
1084+
super().__init__(*args, **kwargs)
1085+
1086+
1087+
class BrowserAutomationToolConnectionParameters(_Model): # pylint: disable=name-too-long
1088+
"""Definition of input parameters for the connection used by the Browser Automation Tool.
1089+
1090+
:ivar id: The ID of the connection to your Azure Playwright resource. Required.
1091+
:vartype id: str
1092+
"""
1093+
1094+
id: str = rest_field(visibility=["read", "create", "update", "delete", "query"])
1095+
"""The ID of the connection to your Azure Playwright resource. Required."""
1096+
1097+
@overload
1098+
def __init__(
1099+
self,
1100+
*,
1101+
id: str, # pylint: disable=redefined-builtin
1102+
) -> None: ...
1103+
1104+
@overload
1105+
def __init__(self, mapping: Mapping[str, Any]) -> None:
1106+
"""
1107+
:param mapping: raw JSON to initialize the model.
1108+
:type mapping: Mapping[str, Any]
1109+
"""
1110+
1111+
def __init__(self, *args: Any, **kwargs: Any) -> None:
1112+
super().__init__(*args, **kwargs)
1113+
1114+
1115+
class BrowserAutomationToolDefinition(ToolDefinition, discriminator="browser_automation"):
1116+
"""The input definition information for a Browser Automation Tool, as used to configure an Agent.
1117+
1118+
:ivar type: The object type, which is always 'browser_automation'. Required. Default value is
1119+
"browser_automation".
1120+
:vartype type: str
1121+
:ivar browser_automation: The Browser Automation Tool parameters. Required.
1122+
:vartype browser_automation: ~azure.ai.agents.models.BrowserAutomationToolParameters
1123+
"""
1124+
1125+
type: Literal["browser_automation"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore
1126+
"""The object type, which is always 'browser_automation'. Required. Default value is
1127+
\"browser_automation\"."""
1128+
browser_automation: "_models.BrowserAutomationToolParameters" = rest_field(
1129+
visibility=["read", "create", "update", "delete", "query"]
1130+
)
1131+
"""The Browser Automation Tool parameters. Required."""
1132+
1133+
@overload
1134+
def __init__(
1135+
self,
1136+
*,
1137+
browser_automation: "_models.BrowserAutomationToolParameters",
1138+
) -> None: ...
1139+
1140+
@overload
1141+
def __init__(self, mapping: Mapping[str, Any]) -> None:
1142+
"""
1143+
:param mapping: raw JSON to initialize the model.
1144+
:type mapping: Mapping[str, Any]
1145+
"""
1146+
1147+
def __init__(self, *args: Any, **kwargs: Any) -> None:
1148+
super().__init__(*args, type="browser_automation", **kwargs)
1149+
1150+
1151+
class BrowserAutomationToolParameters(_Model):
1152+
"""Definition of input parameters for the Browser Automation Tool.
1153+
1154+
:ivar connection: The connection parameters associated with the Browser Automation Tool.
1155+
Required.
1156+
:vartype connection: ~azure.ai.agents.models.BrowserAutomationToolConnectionParameters
1157+
"""
1158+
1159+
connection: "_models.BrowserAutomationToolConnectionParameters" = rest_field(
1160+
visibility=["read", "create", "update", "delete", "query"]
1161+
)
1162+
"""The connection parameters associated with the Browser Automation Tool. Required."""
1163+
1164+
@overload
1165+
def __init__(
1166+
self,
1167+
*,
1168+
connection: "_models.BrowserAutomationToolConnectionParameters",
1169+
) -> None: ...
1170+
1171+
@overload
1172+
def __init__(self, mapping: Mapping[str, Any]) -> None:
1173+
"""
1174+
:param mapping: raw JSON to initialize the model.
1175+
:type mapping: Mapping[str, Any]
1176+
"""
1177+
1178+
def __init__(self, *args: Any, **kwargs: Any) -> None:
1179+
super().__init__(*args, **kwargs)
1180+
1181+
10081182
class CodeInterpreterToolDefinition(ToolDefinition, discriminator="code_interpreter"):
10091183
"""The input definition information for a code interpreter tool as used to configure an agent.
10101184
@@ -3975,9 +4149,9 @@ class RunStepToolCall(_Model):
39754149
39764150
You probably want to use the sub-classes and not this class directly. Known sub-classes are:
39774151
RunStepAzureAISearchToolCall, RunStepBingCustomSearchToolCall, RunStepBingGroundingToolCall,
3978-
RunStepCodeInterpreterToolCall, RunStepDeepResearchToolCall, RunStepMicrosoftFabricToolCall,
3979-
RunStepFileSearchToolCall, RunStepFunctionToolCall, RunStepMcpToolCall, RunStepOpenAPIToolCall,
3980-
RunStepSharepointToolCall
4152+
RunStepBrowserAutomationToolCall, RunStepCodeInterpreterToolCall, RunStepDeepResearchToolCall,
4153+
RunStepMicrosoftFabricToolCall, RunStepFileSearchToolCall, RunStepFunctionToolCall,
4154+
RunStepMcpToolCall, RunStepOpenAPIToolCall, RunStepSharepointToolCall
39814155
39824156
:ivar type: The object type. Required. Default value is None.
39834157
:vartype type: str
@@ -4131,6 +4305,46 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
41314305
super().__init__(*args, type="bing_grounding", **kwargs)
41324306

41334307

4308+
class RunStepBrowserAutomationToolCall(RunStepToolCall, discriminator="browser_automation"):
4309+
"""A record of a call to a Browser Automation tool issued by the Agent.
4310+
4311+
:ivar id: The ID of the tool call. This ID must be referenced when you submit tool outputs.
4312+
Required.
4313+
:vartype id: str
4314+
:ivar type: The object type, which is always 'browser_automation'. Required. Default value is
4315+
"browser_automation".
4316+
:vartype type: str
4317+
:ivar browser_automation: Details of the browser automation tool call. Required.
4318+
:vartype browser_automation: ~azure.ai.agents.models.BrowserAutomationToolCallDetails
4319+
"""
4320+
4321+
type: Literal["browser_automation"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore
4322+
"""The object type, which is always 'browser_automation'. Required. Default value is
4323+
\"browser_automation\"."""
4324+
browser_automation: "_models.BrowserAutomationToolCallDetails" = rest_field(
4325+
visibility=["read", "create", "update", "delete", "query"]
4326+
)
4327+
"""Details of the browser automation tool call. Required."""
4328+
4329+
@overload
4330+
def __init__(
4331+
self,
4332+
*,
4333+
id: str, # pylint: disable=redefined-builtin
4334+
browser_automation: "_models.BrowserAutomationToolCallDetails",
4335+
) -> None: ...
4336+
4337+
@overload
4338+
def __init__(self, mapping: Mapping[str, Any]) -> None:
4339+
"""
4340+
:param mapping: raw JSON to initialize the model.
4341+
:type mapping: Mapping[str, Any]
4342+
"""
4343+
4344+
def __init__(self, *args: Any, **kwargs: Any) -> None:
4345+
super().__init__(*args, type="browser_automation", **kwargs)
4346+
4347+
41344348
class RunStepCodeInterpreterToolCallOutput(_Model):
41354349
"""An abstract representation of an emitted output from a code interpreter tool.
41364350

0 commit comments

Comments
 (0)