[Agents] Computer Use Tool (#42483)

* initial code gen

* update sample

* start merge with main

* Update computer use sample, add fake screenshots

* run black

* remove double space

* sample comments

* changelog

* add note to sample on approval requirement

* fix mypy issue

* add computer use test

* fix mypy

Author: glharper

sdk/ai/azure-ai-agents/CHANGELOG.md

@@ -8,6 +8,7 @@
 
 ### Features Added
 
+- Added Computer Use Preview tool for use with the computer-use-preview model
 - Added static `merge_resources` method to `McpTool` with accompanying sample.
 
 ### Bugs Fixed
@@ -18,6 +19,7 @@
 
 ### Sample updates
 
+- Added Computer Use Preview tool sample demonstrating use with the computer-use-preview model
 - Added sample demonstrating multiple McpTool instance usage.
 
 ## 1.2.0b3 (2025-08-22)

sdk/ai/azure-ai-agents/apiview-properties.json

@@ -30,13 +30,23 @@
         "azure.ai.agents.models.BrowserAutomationToolConnectionParameters": "Azure.AI.Agents.BrowserAutomationToolConnectionParameters",
         "azure.ai.agents.models.BrowserAutomationToolDefinition": "Azure.AI.Agents.BrowserAutomationToolDefinition",
         "azure.ai.agents.models.BrowserAutomationToolParameters": "Azure.AI.Agents.BrowserAutomationToolParameters",
+        "azure.ai.agents.models.ComputerUseAction": "Azure.AI.Agents.ComputerUseAction",
+        "azure.ai.agents.models.ClickAction": "Azure.AI.Agents.ClickAction",
         "azure.ai.agents.models.CodeInterpreterToolDefinition": "Azure.AI.Agents.CodeInterpreterToolDefinition",
         "azure.ai.agents.models.CodeInterpreterToolResource": "Azure.AI.Agents.CodeInterpreterToolResource",
+        "azure.ai.agents.models.ComputerScreenshot": "Azure.AI.Agents.ComputerScreenshot",
+        "azure.ai.agents.models.ToolOutput": "Azure.AI.Agents.ToolOutput",
+        "azure.ai.agents.models.ComputerToolOutput": "Azure.AI.Agents.ComputerToolOutput",
+        "azure.ai.agents.models.ComputerUseToolDefinition": "Azure.AI.Agents.ComputerUseToolDefinition",
+        "azure.ai.agents.models.ComputerUseToolParameters": "Azure.AI.Agents.ComputerUseToolParameters",
         "azure.ai.agents.models.ConnectedAgentDetails": "Azure.AI.Agents.ConnectedAgentDetails",
         "azure.ai.agents.models.ConnectedAgentToolDefinition": "Azure.AI.Agents.ConnectedAgentToolDefinition",
+        "azure.ai.agents.models.CoordinatePoint": "Azure.AI.Agents.CoordinatePoint",
         "azure.ai.agents.models.DeepResearchBingGroundingConnection": "Azure.AI.Agents.DeepResearchBingGroundingConnection",
         "azure.ai.agents.models.DeepResearchDetails": "Azure.AI.Agents.DeepResearchDetails",
         "azure.ai.agents.models.DeepResearchToolDefinition": "Azure.AI.Agents.DeepResearchToolDefinition",
+        "azure.ai.agents.models.DoubleClickAction": "Azure.AI.Agents.DoubleClickAction",
+        "azure.ai.agents.models.DragAction": "Azure.AI.Agents.DragAction",
         "azure.ai.agents.models.FabricDataAgentToolParameters": "Azure.AI.Agents.FabricDataAgentToolParameters",
         "azure.ai.agents.models.FileInfo": "Azure.AI.Agents.FileInfo",
         "azure.ai.agents.models.FileListResponse": "Azure.AI.Agents.FileListResponse",
@@ -49,7 +59,9 @@
         "azure.ai.agents.models.FunctionDefinition": "Azure.AI.Agents.FunctionDefinition",
         "azure.ai.agents.models.FunctionName": "Azure.AI.Agents.FunctionName",
         "azure.ai.agents.models.FunctionToolDefinition": "Azure.AI.Agents.FunctionToolDefinition",
+        "azure.ai.agents.models.FunctionToolOutput": "Azure.AI.Agents.FunctionToolOutput",
         "azure.ai.agents.models.IncompleteRunDetails": "Azure.AI.Agents.IncompleteRunDetails",
+        "azure.ai.agents.models.KeyPressAction": "Azure.AI.Agents.KeyPressAction",
         "azure.ai.agents.models.MCPApprovalPerTool": "Azure.AI.Agents.MCPApprovalPerTool",
         "azure.ai.agents.models.MCPToolDefinition": "Azure.AI.Agents.MCPToolDefinition",
         "azure.ai.agents.models.MCPToolList": "Azure.AI.Agents.MCPToolList",
@@ -89,6 +101,7 @@
         "azure.ai.agents.models.MessageTextUrlCitationAnnotation": "Azure.AI.Agents.MessageTextUrlCitationAnnotation",
         "azure.ai.agents.models.MessageTextUrlCitationDetails": "Azure.AI.Agents.MessageTextUrlCitationDetails",
         "azure.ai.agents.models.MicrosoftFabricToolDefinition": "Azure.AI.Agents.MicrosoftFabricToolDefinition",
+        "azure.ai.agents.models.MoveAction": "Azure.AI.Agents.MoveAction",
         "azure.ai.agents.models.OpenApiAuthDetails": "Azure.AI.Agents.OpenApiAuthDetails",
         "azure.ai.agents.models.OpenApiAnonymousAuthDetails": "Azure.AI.Agents.OpenApiAnonymousAuthDetails",
         "azure.ai.agents.models.OpenApiConnectionAuthDetails": "Azure.AI.Agents.OpenApiConnectionAuthDetails",
@@ -99,6 +112,8 @@
         "azure.ai.agents.models.OpenApiToolDefinition": "Azure.AI.Agents.OpenApiToolDefinition",
         "azure.ai.agents.models.RequiredAction": "Azure.AI.Agents.RequiredAction",
         "azure.ai.agents.models.RequiredToolCall": "Azure.AI.Agents.RequiredToolCall",
+        "azure.ai.agents.models.RequiredComputerUseToolCall": "Azure.AI.Agents.RequiredComputerUseToolCall",
+        "azure.ai.agents.models.RequiredComputerUseToolCallDetails": "Azure.AI.Agents.RequiredComputerUseToolCallDetails",
         "azure.ai.agents.models.RequiredFunctionToolCall": "Azure.AI.Agents.RequiredFunctionToolCall",
         "azure.ai.agents.models.RequiredFunctionToolCallDetails": "Azure.AI.Agents.RequiredFunctionToolCallDetails",
         "azure.ai.agents.models.RequiredMcpToolCall": "Azure.AI.Agents.RequiredMcpToolCall",
@@ -122,6 +137,8 @@
         "azure.ai.agents.models.RunStepCodeInterpreterToolCall": "Azure.AI.Agents.RunStepCodeInterpreterToolCall",
         "azure.ai.agents.models.RunStepCodeInterpreterToolCallDetails": "Azure.AI.Agents.RunStepCodeInterpreterToolCallDetails",
         "azure.ai.agents.models.RunStepCompletionUsage": "Azure.AI.Agents.RunStepCompletionUsage",
+        "azure.ai.agents.models.RunStepComputerUseToolCall": "Azure.AI.Agents.RunStepComputerUseToolCall",
+        "azure.ai.agents.models.RunStepComputerUseToolCallDetails": "Azure.AI.Agents.RunStepComputerUseToolCallDetails",
         "azure.ai.agents.models.RunStepConnectedAgent": "Azure.AI.Agents.RunStepConnectedAgent",
         "azure.ai.agents.models.RunStepConnectedAgentToolCall": "Azure.AI.Agents.RunStepConnectedAgentToolCall",
         "azure.ai.agents.models.RunStepDeepResearchToolCall": "Azure.AI.Agents.RunStepDeepResearchToolCall",
@@ -168,6 +185,9 @@
         "azure.ai.agents.models.RunStepOpenAPIToolCall": "Azure.AI.Agents.RunStepOpenAPIToolCall",
         "azure.ai.agents.models.RunStepSharepointToolCall": "Azure.AI.Agents.RunStepSharepointToolCall",
         "azure.ai.agents.models.RunStepToolCallDetails": "Azure.AI.Agents.RunStepToolCallDetails",
+        "azure.ai.agents.models.SafetyCheck": "Azure.AI.Agents.SafetyCheck",
+        "azure.ai.agents.models.ScreenshotAction": "Azure.AI.Agents.ScreenshotAction",
+        "azure.ai.agents.models.ScrollAction": "Azure.AI.Agents.ScrollAction",
         "azure.ai.agents.models.SharepointGroundingToolParameters": "Azure.AI.Agents.SharepointGroundingToolParameters",
         "azure.ai.agents.models.SharepointToolDefinition": "Azure.AI.Agents.SharepointToolDefinition",
         "azure.ai.agents.models.SubmitToolApprovalAction": "Azure.AI.Agents.SubmitToolApprovalAction",
@@ -179,9 +199,9 @@
         "azure.ai.agents.models.ThreadRun": "Azure.AI.Agents.ThreadRun",
         "azure.ai.agents.models.ToolApproval": "Azure.AI.Agents.ToolApproval",
         "azure.ai.agents.models.ToolConnection": "Azure.AI.Agents.ToolConnection",
-        "azure.ai.agents.models.ToolOutput": "Azure.AI.Agents.ToolOutput",
         "azure.ai.agents.models.ToolResources": "Azure.AI.Agents.ToolResources",
         "azure.ai.agents.models.TruncationObject": "Azure.AI.Agents.TruncationObject",
+        "azure.ai.agents.models.TypeAction": "Azure.AI.Agents.TypeAction",
         "azure.ai.agents.models.VectorStore": "Azure.AI.Agents.VectorStore",
         "azure.ai.agents.models.VectorStoreChunkingStrategyRequest": "Azure.AI.Agents.VectorStoreChunkingStrategyRequest",
         "azure.ai.agents.models.VectorStoreAutoChunkingStrategyRequest": "Azure.AI.Agents.VectorStoreAutoChunkingStrategyRequest",
@@ -198,16 +218,19 @@
         "azure.ai.agents.models.VectorStoreStaticChunkingStrategyOptions": "Azure.AI.Agents.VectorStoreStaticChunkingStrategyOptions",
         "azure.ai.agents.models.VectorStoreStaticChunkingStrategyRequest": "Azure.AI.Agents.VectorStoreStaticChunkingStrategyRequest",
         "azure.ai.agents.models.VectorStoreStaticChunkingStrategyResponse": "Azure.AI.Agents.VectorStoreStaticChunkingStrategyResponse",
+        "azure.ai.agents.models.WaitAction": "Azure.AI.Agents.WaitAction",
         "azure.ai.agents.models.VectorStoreDataSourceAssetType": "Azure.AI.Agents.VectorStoreDataSourceAssetType",
         "azure.ai.agents.models.AzureAISearchQueryType": "Azure.AI.Agents.AzureAISearchQueryType",
         "azure.ai.agents.models.MessageRole": "Azure.AI.Agents.MessageRole",
         "azure.ai.agents.models.MessageBlockType": "Azure.AI.Agents.MessageBlockType",
         "azure.ai.agents.models.ImageDetailLevel": "Azure.AI.Agents.ImageDetailLevel",
+        "azure.ai.agents.models.ComputerUseEnvironment": "Azure.AI.Agents.ComputerUseEnvironment",
         "azure.ai.agents.models.OpenApiAuthType": "Azure.AI.Agents.OpenApiAuthType",
         "azure.ai.agents.models.ListSortOrder": "Azure.AI.Agents.ListSortOrder",
         "azure.ai.agents.models.MessageStatus": "Azure.AI.Agents.MessageStatus",
         "azure.ai.agents.models.MessageIncompleteDetailsReason": "Azure.AI.Agents.MessageIncompleteDetailsReason",
         "azure.ai.agents.models.RunStatus": "Azure.AI.Agents.RunStatus",
+        "azure.ai.agents.models.MouseButton": "Azure.AI.Agents.MouseButton",
         "azure.ai.agents.models.IncompleteDetailsReason": "Azure.AI.Agents.IncompleteDetailsReason",
         "azure.ai.agents.models.TruncationStrategy": "Azure.AI.Agents.TruncationStrategy",
         "azure.ai.agents.models.AgentsToolChoiceOptionMode": "Azure.AI.Agents.AgentsToolChoiceOptionMode",

sdk/ai/azure-ai-agents/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/ai/azure-ai-agents",
-  "Tag": "python/ai/azure-ai-agents_f5906767ec"
+  "Tag": "python/ai/azure-ai-agents_ce4eea88ee"
 }

sdk/ai/azure-ai-agents/azure/ai/agents/models/__init__.py

@@ -42,13 +42,22 @@
     BrowserAutomationToolConnectionParameters,
     BrowserAutomationToolDefinition,
     BrowserAutomationToolParameters,
+    ClickAction,
     CodeInterpreterToolDefinition,
     CodeInterpreterToolResource,
+    ComputerScreenshot,
+    ComputerToolOutput,
+    ComputerUseAction,
+    ComputerUseToolDefinition,
+    ComputerUseToolParameters,
     ConnectedAgentDetails,
     ConnectedAgentToolDefinition,
+    CoordinatePoint,
     DeepResearchBingGroundingConnection,
     DeepResearchDetails,
     DeepResearchToolDefinition,
+    DoubleClickAction,
+    DragAction,
     FabricDataAgentToolParameters,
     FileInfo,
     FileListResponse,
@@ -61,7 +70,9 @@
     FunctionDefinition,
     FunctionName,
     FunctionToolDefinition,
+    FunctionToolOutput,
     IncompleteRunDetails,
+    KeyPressAction,
     MCPApprovalPerTool,
     MCPToolDefinition,
     MCPToolList,
@@ -101,6 +112,7 @@
     MessageTextUrlCitationAnnotation,
     MessageTextUrlCitationDetails,
     MicrosoftFabricToolDefinition,
+    MoveAction,
     OpenApiAnonymousAuthDetails,
     OpenApiAuthDetails,
     OpenApiConnectionAuthDetails,
@@ -110,6 +122,8 @@
     OpenApiManagedSecurityScheme,
     OpenApiToolDefinition,
     RequiredAction,
+    RequiredComputerUseToolCall,
+    RequiredComputerUseToolCallDetails,
     RequiredFunctionToolCall,
     RequiredFunctionToolCallDetails,
     RequiredMcpToolCall,
@@ -132,6 +146,8 @@
     RunStepCodeInterpreterToolCallDetails,
     RunStepCodeInterpreterToolCallOutput,
     RunStepCompletionUsage,
+    RunStepComputerUseToolCall,
+    RunStepComputerUseToolCallDetails,
     RunStepConnectedAgent,
     RunStepConnectedAgentToolCall,
     RunStepDeepResearchToolCall,
@@ -180,6 +196,9 @@
     RunStepSharepointToolCall,
     RunStepToolCall,
     RunStepToolCallDetails,
+    SafetyCheck,
+    ScreenshotAction,
+    ScrollAction,
     SharepointGroundingToolParameters,
     SharepointToolDefinition,
     SubmitToolApprovalAction,
@@ -195,6 +214,7 @@
     ToolOutput,
     ToolResources,
     TruncationObject,
+    TypeAction,
     VectorStore,
     VectorStoreAutoChunkingStrategyRequest,
     VectorStoreAutoChunkingStrategyResponse,
@@ -211,6 +231,7 @@
     VectorStoreStaticChunkingStrategyOptions,
     VectorStoreStaticChunkingStrategyRequest,
     VectorStoreStaticChunkingStrategyResponse,
+    WaitAction,
 )
 
 from ._enums import (  # type: ignore
@@ -219,6 +240,7 @@
     AgentsResponseFormatMode,
     AgentsToolChoiceOptionMode,
     AzureAISearchQueryType,
+    ComputerUseEnvironment,
     DoneEvent,
     ErrorEvent,
     FilePurpose,
@@ -231,6 +253,7 @@
     MessageRole,
     MessageStatus,
     MessageStreamEvent,
+    MouseButton,
     OpenApiAuthType,
     ResponseFormat,
     RunAdditionalFieldList,
@@ -285,13 +308,22 @@
     "BrowserAutomationToolConnectionParameters",
     "BrowserAutomationToolDefinition",
     "BrowserAutomationToolParameters",
+    "ClickAction",
     "CodeInterpreterToolDefinition",
     "CodeInterpreterToolResource",
+    "ComputerScreenshot",
+    "ComputerToolOutput",
+    "ComputerUseAction",
+    "ComputerUseToolDefinition",
+    "ComputerUseToolParameters",
     "ConnectedAgentDetails",
     "ConnectedAgentToolDefinition",
+    "CoordinatePoint",
     "DeepResearchBingGroundingConnection",
     "DeepResearchDetails",
     "DeepResearchToolDefinition",
+    "DoubleClickAction",
+    "DragAction",
     "FabricDataAgentToolParameters",
     "FileInfo",
     "FileListResponse",
@@ -304,7 +336,9 @@
     "FunctionDefinition",
     "FunctionName",
     "FunctionToolDefinition",
+    "FunctionToolOutput",
     "IncompleteRunDetails",
+    "KeyPressAction",
     "MCPApprovalPerTool",
     "MCPToolDefinition",
     "MCPToolList",
@@ -344,6 +378,7 @@
     "MessageTextUrlCitationAnnotation",
     "MessageTextUrlCitationDetails",
     "MicrosoftFabricToolDefinition",
+    "MoveAction",
     "OpenApiAnonymousAuthDetails",
     "OpenApiAuthDetails",
     "OpenApiConnectionAuthDetails",
@@ -353,6 +388,8 @@
     "OpenApiManagedSecurityScheme",
     "OpenApiToolDefinition",
     "RequiredAction",
+    "RequiredComputerUseToolCall",
+    "RequiredComputerUseToolCallDetails",
     "RequiredFunctionToolCall",
     "RequiredFunctionToolCallDetails",
     "RequiredMcpToolCall",
@@ -375,6 +412,8 @@
     "RunStepCodeInterpreterToolCallDetails",
     "RunStepCodeInterpreterToolCallOutput",
     "RunStepCompletionUsage",
+    "RunStepComputerUseToolCall",
+    "RunStepComputerUseToolCallDetails",
     "RunStepConnectedAgent",
     "RunStepConnectedAgentToolCall",
     "RunStepDeepResearchToolCall",
@@ -423,6 +462,9 @@
     "RunStepSharepointToolCall",
     "RunStepToolCall",
     "RunStepToolCallDetails",
+    "SafetyCheck",
+    "ScreenshotAction",
+    "ScrollAction",
     "SharepointGroundingToolParameters",
     "SharepointToolDefinition",
     "SubmitToolApprovalAction",
@@ -438,6 +480,7 @@
     "ToolOutput",
     "ToolResources",
     "TruncationObject",
+    "TypeAction",
     "VectorStore",
     "VectorStoreAutoChunkingStrategyRequest",
     "VectorStoreAutoChunkingStrategyResponse",
@@ -454,11 +497,13 @@
     "VectorStoreStaticChunkingStrategyOptions",
     "VectorStoreStaticChunkingStrategyRequest",
     "VectorStoreStaticChunkingStrategyResponse",
+    "WaitAction",
     "AgentStreamEvent",
     "AgentsNamedToolChoiceType",
     "AgentsResponseFormatMode",
     "AgentsToolChoiceOptionMode",
     "AzureAISearchQueryType",
+    "ComputerUseEnvironment",
     "DoneEvent",
     "ErrorEvent",
     "FilePurpose",
@@ -471,6 +516,7 @@
     "MessageRole",
     "MessageStatus",
     "MessageStreamEvent",
+    "MouseButton",
     "OpenApiAuthType",
     "ResponseFormat",
     "RunAdditionalFieldList",

sdk/ai/azure-ai-agents/azure/ai/agents/models/_enums.py

@@ -35,6 +35,8 @@ class AgentsNamedToolChoiceType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Tool type ``deep_research``"""
     MCP = "mcp"
     """Tool type ``mcp``"""
+    COMPUTER_USE_PREVIEW = "computer_use_preview"
+    """Tool type ``computer_use_preview``"""
 
 
 class AgentsResponseFormatMode(str, Enum, metaclass=CaseInsensitiveEnumMeta):
@@ -150,6 +152,19 @@ class AzureAISearchQueryType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Query type ``vector_semantic_hybrid``"""
 
 
+class ComputerUseEnvironment(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """The environment types supported by the computer use tool."""
+
+    WINDOWS = "windows"
+    """Windows environment"""
+    MAC = "mac"
+    """Mac environment"""
+    LINUX = "linux"
+    """Linux environment"""
+    BROWSER = "browser"
+    """Browser environment"""
+
+
 class DoneEvent(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Terminal event indicating the successful end of a stream."""
 
@@ -299,6 +314,21 @@ class MessageStreamEvent(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Event sent before a message is completed. The data of this event is of type ThreadMessage"""
 
 
+class MouseButton(str, Enum, metaclass=CaseInsensitiveEnumMeta):
+    """The mouse button types supported by click actions."""
+
+    LEFT = "left"
+    """Left mouse button"""
+    RIGHT = "right"
+    """Right mouse button"""
+    WHEEL = "wheel"
+    """Mouse wheel button"""
+    BACK = "back"
+    """Back mouse button"""
+    FORWARD = "forward"
+    """Forward mouse button"""
+
+
 class OpenApiAuthType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """Authentication type for OpenApi endpoint. Allowed types are: