diff --git a/sdk/ai/azure-ai-agents/apiview-properties.json b/sdk/ai/azure-ai-agents/apiview-properties.json index 91d949c6d0c4..87c62dc6d260 100644 --- a/sdk/ai/azure-ai-agents/apiview-properties.json +++ b/sdk/ai/azure-ai-agents/apiview-properties.json @@ -27,13 +27,23 @@ "azure.ai.agents.models.BrowserAutomationToolConnectionParameters": "Azure.AI.Agents.BrowserAutomationToolConnectionParameters", "azure.ai.agents.models.BrowserAutomationToolDefinition": "Azure.AI.Agents.BrowserAutomationToolDefinition", "azure.ai.agents.models.BrowserAutomationToolParameters": "Azure.AI.Agents.BrowserAutomationToolParameters", + "azure.ai.agents.models.ComputerUseAction": "Azure.AI.Agents.ComputerUseAction", + "azure.ai.agents.models.ClickAction": "Azure.AI.Agents.ClickAction", "azure.ai.agents.models.CodeInterpreterToolDefinition": "Azure.AI.Agents.CodeInterpreterToolDefinition", "azure.ai.agents.models.CodeInterpreterToolResource": "Azure.AI.Agents.CodeInterpreterToolResource", + "azure.ai.agents.models.ComputerScreenshot": "Azure.AI.Agents.ComputerScreenshot", + "azure.ai.agents.models.ToolOutput": "Azure.AI.Agents.ToolOutput", + "azure.ai.agents.models.ComputerToolOutput": "Azure.AI.Agents.ComputerToolOutput", + "azure.ai.agents.models.ComputerUseToolDefinition": "Azure.AI.Agents.ComputerUseToolDefinition", + "azure.ai.agents.models.ComputerUseToolParameters": "Azure.AI.Agents.ComputerUseToolParameters", "azure.ai.agents.models.ConnectedAgentDetails": "Azure.AI.Agents.ConnectedAgentDetails", "azure.ai.agents.models.ConnectedAgentToolDefinition": "Azure.AI.Agents.ConnectedAgentToolDefinition", + "azure.ai.agents.models.CoordinatePoint": "Azure.AI.Agents.CoordinatePoint", "azure.ai.agents.models.DeepResearchBingGroundingConnection": "Azure.AI.Agents.DeepResearchBingGroundingConnection", "azure.ai.agents.models.DeepResearchDetails": "Azure.AI.Agents.DeepResearchDetails", "azure.ai.agents.models.DeepResearchToolDefinition": "Azure.AI.Agents.DeepResearchToolDefinition", + "azure.ai.agents.models.DoubleClickAction": "Azure.AI.Agents.DoubleClickAction", + "azure.ai.agents.models.DragAction": "Azure.AI.Agents.DragAction", "azure.ai.agents.models.FabricDataAgentToolParameters": "Azure.AI.Agents.FabricDataAgentToolParameters", "azure.ai.agents.models.FileInfo": "Azure.AI.Agents.FileInfo", "azure.ai.agents.models.FileListResponse": "Azure.AI.Agents.FileListResponse", @@ -45,7 +55,9 @@ "azure.ai.agents.models.FunctionDefinition": "Azure.AI.Agents.FunctionDefinition", "azure.ai.agents.models.FunctionName": "Azure.AI.Agents.FunctionName", "azure.ai.agents.models.FunctionToolDefinition": "Azure.AI.Agents.FunctionToolDefinition", + "azure.ai.agents.models.FunctionToolOutput": "Azure.AI.Agents.FunctionToolOutput", "azure.ai.agents.models.IncompleteRunDetails": "Azure.AI.Agents.IncompleteRunDetails", + "azure.ai.agents.models.KeyPressAction": "Azure.AI.Agents.KeyPressAction", "azure.ai.agents.models.MCPApprovalPerTool": "Azure.AI.Agents.MCPApprovalPerTool", "azure.ai.agents.models.MCPToolDefinition": "Azure.AI.Agents.MCPToolDefinition", "azure.ai.agents.models.MCPToolList": "Azure.AI.Agents.MCPToolList", @@ -85,6 +97,7 @@ "azure.ai.agents.models.MessageTextUrlCitationAnnotation": "Azure.AI.Agents.MessageTextUrlCitationAnnotation", "azure.ai.agents.models.MessageTextUrlCitationDetails": "Azure.AI.Agents.MessageTextUrlCitationDetails", "azure.ai.agents.models.MicrosoftFabricToolDefinition": "Azure.AI.Agents.MicrosoftFabricToolDefinition", + "azure.ai.agents.models.MoveAction": "Azure.AI.Agents.MoveAction", "azure.ai.agents.models.OpenApiAuthDetails": "Azure.AI.Agents.OpenApiAuthDetails", "azure.ai.agents.models.OpenApiAnonymousAuthDetails": "Azure.AI.Agents.OpenApiAnonymousAuthDetails", "azure.ai.agents.models.OpenApiConnectionAuthDetails": "Azure.AI.Agents.OpenApiConnectionAuthDetails", @@ -95,6 +108,8 @@ "azure.ai.agents.models.OpenApiToolDefinition": "Azure.AI.Agents.OpenApiToolDefinition", "azure.ai.agents.models.RequiredAction": "Azure.AI.Agents.RequiredAction", "azure.ai.agents.models.RequiredToolCall": "Azure.AI.Agents.RequiredToolCall", + "azure.ai.agents.models.RequiredComputerUseToolCall": "Azure.AI.Agents.RequiredComputerUseToolCall", + "azure.ai.agents.models.RequiredComputerUseToolCallDetails": "Azure.AI.Agents.RequiredComputerUseToolCallDetails", "azure.ai.agents.models.RequiredFunctionToolCall": "Azure.AI.Agents.RequiredFunctionToolCall", "azure.ai.agents.models.RequiredFunctionToolCallDetails": "Azure.AI.Agents.RequiredFunctionToolCallDetails", "azure.ai.agents.models.RequiredMcpToolCall": "Azure.AI.Agents.RequiredMcpToolCall", @@ -115,6 +130,8 @@ "azure.ai.agents.models.RunStepCodeInterpreterToolCall": "Azure.AI.Agents.RunStepCodeInterpreterToolCall", "azure.ai.agents.models.RunStepCodeInterpreterToolCallDetails": "Azure.AI.Agents.RunStepCodeInterpreterToolCallDetails", "azure.ai.agents.models.RunStepCompletionUsage": "Azure.AI.Agents.RunStepCompletionUsage", + "azure.ai.agents.models.RunStepComputerUseToolCall": "Azure.AI.Agents.RunStepComputerUseToolCall", + "azure.ai.agents.models.RunStepComputerUseToolCallDetails": "Azure.AI.Agents.RunStepComputerUseToolCallDetails", "azure.ai.agents.models.RunStepConnectedAgent": "Azure.AI.Agents.RunStepConnectedAgent", "azure.ai.agents.models.RunStepConnectedAgentToolCall": "Azure.AI.Agents.RunStepConnectedAgentToolCall", "azure.ai.agents.models.RunStepDeepResearchToolCall": "Azure.AI.Agents.RunStepDeepResearchToolCall", @@ -157,6 +174,9 @@ "azure.ai.agents.models.RunStepOpenAPIToolCall": "Azure.AI.Agents.RunStepOpenAPIToolCall", "azure.ai.agents.models.RunStepSharepointToolCall": "Azure.AI.Agents.RunStepSharepointToolCall", "azure.ai.agents.models.RunStepToolCallDetails": "Azure.AI.Agents.RunStepToolCallDetails", + "azure.ai.agents.models.SafetyCheck": "Azure.AI.Agents.SafetyCheck", + "azure.ai.agents.models.ScreenshotAction": "Azure.AI.Agents.ScreenshotAction", + "azure.ai.agents.models.ScrollAction": "Azure.AI.Agents.ScrollAction", "azure.ai.agents.models.SharepointGroundingToolParameters": "Azure.AI.Agents.SharepointGroundingToolParameters", "azure.ai.agents.models.SharepointToolDefinition": "Azure.AI.Agents.SharepointToolDefinition", "azure.ai.agents.models.SubmitToolApprovalAction": "Azure.AI.Agents.SubmitToolApprovalAction", @@ -168,9 +188,9 @@ "azure.ai.agents.models.ThreadRun": "Azure.AI.Agents.ThreadRun", "azure.ai.agents.models.ToolApproval": "Azure.AI.Agents.ToolApproval", "azure.ai.agents.models.ToolConnection": "Azure.AI.Agents.ToolConnection", - "azure.ai.agents.models.ToolOutput": "Azure.AI.Agents.ToolOutput", "azure.ai.agents.models.ToolResources": "Azure.AI.Agents.ToolResources", "azure.ai.agents.models.TruncationObject": "Azure.AI.Agents.TruncationObject", + "azure.ai.agents.models.TypeAction": "Azure.AI.Agents.TypeAction", "azure.ai.agents.models.VectorStore": "Azure.AI.Agents.VectorStore", "azure.ai.agents.models.VectorStoreChunkingStrategyRequest": "Azure.AI.Agents.VectorStoreChunkingStrategyRequest", "azure.ai.agents.models.VectorStoreAutoChunkingStrategyRequest": "Azure.AI.Agents.VectorStoreAutoChunkingStrategyRequest", @@ -187,16 +207,19 @@ "azure.ai.agents.models.VectorStoreStaticChunkingStrategyOptions": "Azure.AI.Agents.VectorStoreStaticChunkingStrategyOptions", "azure.ai.agents.models.VectorStoreStaticChunkingStrategyRequest": "Azure.AI.Agents.VectorStoreStaticChunkingStrategyRequest", "azure.ai.agents.models.VectorStoreStaticChunkingStrategyResponse": "Azure.AI.Agents.VectorStoreStaticChunkingStrategyResponse", + "azure.ai.agents.models.WaitAction": "Azure.AI.Agents.WaitAction", "azure.ai.agents.models.VectorStoreDataSourceAssetType": "Azure.AI.Agents.VectorStoreDataSourceAssetType", "azure.ai.agents.models.AzureAISearchQueryType": "Azure.AI.Agents.AzureAISearchQueryType", "azure.ai.agents.models.MessageRole": "Azure.AI.Agents.MessageRole", "azure.ai.agents.models.MessageBlockType": "Azure.AI.Agents.MessageBlockType", "azure.ai.agents.models.ImageDetailLevel": "Azure.AI.Agents.ImageDetailLevel", + "azure.ai.agents.models.ComputerUseEnvironment": "Azure.AI.Agents.ComputerUseEnvironment", "azure.ai.agents.models.OpenApiAuthType": "Azure.AI.Agents.OpenApiAuthType", "azure.ai.agents.models.ListSortOrder": "Azure.AI.Agents.ListSortOrder", "azure.ai.agents.models.MessageStatus": "Azure.AI.Agents.MessageStatus", "azure.ai.agents.models.MessageIncompleteDetailsReason": "Azure.AI.Agents.MessageIncompleteDetailsReason", "azure.ai.agents.models.RunStatus": "Azure.AI.Agents.RunStatus", + "azure.ai.agents.models.MouseButton": "Azure.AI.Agents.MouseButton", "azure.ai.agents.models.IncompleteDetailsReason": "Azure.AI.Agents.IncompleteDetailsReason", "azure.ai.agents.models.TruncationStrategy": "Azure.AI.Agents.TruncationStrategy", "azure.ai.agents.models.AgentsToolChoiceOptionMode": "Azure.AI.Agents.AgentsToolChoiceOptionMode", diff --git a/sdk/ai/azure-ai-agents/azure/ai/agents/_utils/model_base.py b/sdk/ai/azure-ai-agents/azure/ai/agents/_utils/model_base.py index aaa6692b2346..e9ac32aaf9d4 100644 --- a/sdk/ai/azure-ai-agents/azure/ai/agents/_utils/model_base.py +++ b/sdk/ai/azure-ai-agents/azure/ai/agents/_utils/model_base.py @@ -29,6 +29,7 @@ from azure.core import CaseInsensitiveEnumMeta from azure.core.pipeline import PipelineResponse from azure.core.serialization import _Null +from azure.core.rest import HttpResponse _LOGGER = logging.getLogger(__name__) @@ -940,13 +941,13 @@ def _deserialize( def _failsafe_deserialize( deserializer: typing.Any, - value: typing.Any, + response: HttpResponse, module: typing.Optional[str] = None, rf: typing.Optional["_RestField"] = None, format: typing.Optional[str] = None, ) -> typing.Any: try: - return _deserialize(deserializer, value, module, rf, format) + return _deserialize(deserializer, response.json(), module, rf, format) except DeserializationError: _LOGGER.warning( "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True @@ -956,10 +957,10 @@ def _failsafe_deserialize( def _failsafe_deserialize_xml( deserializer: typing.Any, - value: typing.Any, + response: HttpResponse, ) -> typing.Any: try: - return _deserialize_xml(deserializer, value) + return _deserialize_xml(deserializer, response.text()) except DeserializationError: _LOGGER.warning( "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True diff --git a/sdk/ai/azure-ai-agents/azure/ai/agents/aio/operations/_operations.py b/sdk/ai/azure-ai-agents/azure/ai/agents/aio/operations/_operations.py index cbf041efa9a5..ff1adf0233de 100644 --- a/sdk/ai/azure-ai-agents/azure/ai/agents/aio/operations/_operations.py +++ b/sdk/ai/azure-ai-agents/azure/ai/agents/aio/operations/_operations.py @@ -250,7 +250,7 @@ async def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -337,7 +337,7 @@ async def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -392,7 +392,7 @@ async def get(self, thread_id: str, **kwargs: Any) -> _models.AgentThread: except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -556,7 +556,7 @@ async def update( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -617,7 +617,7 @@ async def _delete_thread(self, thread_id: str, **kwargs: Any) -> _models._models except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -825,7 +825,7 @@ async def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -977,7 +977,7 @@ async def get(self, thread_id: str, message_id: str, **kwargs: Any) -> _models.T except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -1138,7 +1138,7 @@ async def update( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -1531,7 +1531,7 @@ async def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -1622,7 +1622,7 @@ async def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -1680,7 +1680,7 @@ async def get(self, thread_id: str, run_id: str, **kwargs: Any) -> _models.Threa except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -1841,7 +1841,7 @@ async def update( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2014,7 +2014,7 @@ async def submit_tool_outputs( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2078,7 +2078,7 @@ async def cancel(self, thread_id: str, run_id: str, **kwargs: Any) -> _models.Th except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2177,7 +2177,7 @@ async def get( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2279,7 +2279,7 @@ async def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -2355,7 +2355,7 @@ async def list( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2429,7 +2429,7 @@ async def _upload_file( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2490,7 +2490,7 @@ async def _delete_file(self, file_id: str, **kwargs: Any) -> _models._models.Fil except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2553,7 +2553,7 @@ async def get(self, file_id: str, **kwargs: Any) -> _models.FileInfo: except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2614,7 +2614,7 @@ async def _get_file_content(self, file_id: str, **kwargs: Any) -> AsyncIterator[ except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) deserialized = response.iter_bytes() @@ -2716,7 +2716,7 @@ async def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -2890,7 +2890,7 @@ async def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2951,7 +2951,7 @@ async def get(self, vector_store_id: str, **kwargs: Any) -> _models.VectorStore: except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3113,7 +3113,7 @@ async def modify( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3177,7 +3177,7 @@ async def _delete_vector_store( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3293,7 +3293,7 @@ async def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -3445,7 +3445,7 @@ async def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3509,7 +3509,7 @@ async def get(self, vector_store_id: str, file_id: str, **kwargs: Any) -> _model except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3577,7 +3577,7 @@ async def _delete_vector_store_file( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3755,7 +3755,7 @@ async def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3819,7 +3819,7 @@ async def get(self, vector_store_id: str, batch_id: str, **kwargs: Any) -> _mode except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3884,7 +3884,7 @@ async def cancel(self, vector_store_id: str, batch_id: str, **kwargs: Any) -> _m except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3984,7 +3984,7 @@ async def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -4215,7 +4215,7 @@ async def create_agent( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4302,7 +4302,7 @@ async def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -4357,7 +4357,7 @@ async def get_agent(self, agent_id: str, **kwargs: Any) -> _models.Agent: except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4604,7 +4604,7 @@ async def update_agent( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4665,7 +4665,7 @@ async def _delete_agent(self, agent_id: str, **kwargs: Any) -> _models._models.A except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4985,7 +4985,7 @@ async def create_thread_and_run( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: diff --git a/sdk/ai/azure-ai-agents/azure/ai/agents/models/__init__.py b/sdk/ai/azure-ai-agents/azure/ai/agents/models/__init__.py index 580ed0213375..0ab1219c7e4a 100644 --- a/sdk/ai/azure-ai-agents/azure/ai/agents/models/__init__.py +++ b/sdk/ai/azure-ai-agents/azure/ai/agents/models/__init__.py @@ -39,13 +39,22 @@ BrowserAutomationToolConnectionParameters, BrowserAutomationToolDefinition, BrowserAutomationToolParameters, + ClickAction, CodeInterpreterToolDefinition, CodeInterpreterToolResource, + ComputerScreenshot, + ComputerToolOutput, + ComputerUseAction, + ComputerUseToolDefinition, + ComputerUseToolParameters, ConnectedAgentDetails, ConnectedAgentToolDefinition, + CoordinatePoint, DeepResearchBingGroundingConnection, DeepResearchDetails, DeepResearchToolDefinition, + DoubleClickAction, + DragAction, FabricDataAgentToolParameters, FileInfo, FileListResponse, @@ -57,7 +66,9 @@ FunctionDefinition, FunctionName, FunctionToolDefinition, + FunctionToolOutput, IncompleteRunDetails, + KeyPressAction, MCPApprovalPerTool, MCPToolDefinition, MCPToolList, @@ -97,6 +108,7 @@ MessageTextUrlCitationAnnotation, MessageTextUrlCitationDetails, MicrosoftFabricToolDefinition, + MoveAction, OpenApiAnonymousAuthDetails, OpenApiAuthDetails, OpenApiConnectionAuthDetails, @@ -106,6 +118,8 @@ OpenApiManagedSecurityScheme, OpenApiToolDefinition, RequiredAction, + RequiredComputerUseToolCall, + RequiredComputerUseToolCallDetails, RequiredFunctionToolCall, RequiredFunctionToolCallDetails, RequiredMcpToolCall, @@ -126,6 +140,8 @@ RunStepCodeInterpreterToolCallDetails, RunStepCodeInterpreterToolCallOutput, RunStepCompletionUsage, + RunStepComputerUseToolCall, + RunStepComputerUseToolCallDetails, RunStepConnectedAgent, RunStepConnectedAgentToolCall, RunStepDeepResearchToolCall, @@ -169,6 +185,9 @@ RunStepSharepointToolCall, RunStepToolCall, RunStepToolCallDetails, + SafetyCheck, + ScreenshotAction, + ScrollAction, SharepointGroundingToolParameters, SharepointToolDefinition, SubmitToolApprovalAction, @@ -184,6 +203,7 @@ ToolOutput, ToolResources, TruncationObject, + TypeAction, VectorStore, VectorStoreAutoChunkingStrategyRequest, VectorStoreAutoChunkingStrategyResponse, @@ -200,6 +220,7 @@ VectorStoreStaticChunkingStrategyOptions, VectorStoreStaticChunkingStrategyRequest, VectorStoreStaticChunkingStrategyResponse, + WaitAction, ) from ._enums import ( # type: ignore @@ -208,6 +229,7 @@ AgentsResponseFormatMode, AgentsToolChoiceOptionMode, AzureAISearchQueryType, + ComputerUseEnvironment, DoneEvent, ErrorEvent, FilePurpose, @@ -220,6 +242,7 @@ MessageRole, MessageStatus, MessageStreamEvent, + MouseButton, OpenApiAuthType, ResponseFormat, RunAdditionalFieldList, @@ -271,13 +294,22 @@ "BrowserAutomationToolConnectionParameters", "BrowserAutomationToolDefinition", "BrowserAutomationToolParameters", + "ClickAction", "CodeInterpreterToolDefinition", "CodeInterpreterToolResource", + "ComputerScreenshot", + "ComputerToolOutput", + "ComputerUseAction", + "ComputerUseToolDefinition", + "ComputerUseToolParameters", "ConnectedAgentDetails", "ConnectedAgentToolDefinition", + "CoordinatePoint", "DeepResearchBingGroundingConnection", "DeepResearchDetails", "DeepResearchToolDefinition", + "DoubleClickAction", + "DragAction", "FabricDataAgentToolParameters", "FileInfo", "FileListResponse", @@ -289,7 +321,9 @@ "FunctionDefinition", "FunctionName", "FunctionToolDefinition", + "FunctionToolOutput", "IncompleteRunDetails", + "KeyPressAction", "MCPApprovalPerTool", "MCPToolDefinition", "MCPToolList", @@ -329,6 +363,7 @@ "MessageTextUrlCitationAnnotation", "MessageTextUrlCitationDetails", "MicrosoftFabricToolDefinition", + "MoveAction", "OpenApiAnonymousAuthDetails", "OpenApiAuthDetails", "OpenApiConnectionAuthDetails", @@ -338,6 +373,8 @@ "OpenApiManagedSecurityScheme", "OpenApiToolDefinition", "RequiredAction", + "RequiredComputerUseToolCall", + "RequiredComputerUseToolCallDetails", "RequiredFunctionToolCall", "RequiredFunctionToolCallDetails", "RequiredMcpToolCall", @@ -358,6 +395,8 @@ "RunStepCodeInterpreterToolCallDetails", "RunStepCodeInterpreterToolCallOutput", "RunStepCompletionUsage", + "RunStepComputerUseToolCall", + "RunStepComputerUseToolCallDetails", "RunStepConnectedAgent", "RunStepConnectedAgentToolCall", "RunStepDeepResearchToolCall", @@ -401,6 +440,9 @@ "RunStepSharepointToolCall", "RunStepToolCall", "RunStepToolCallDetails", + "SafetyCheck", + "ScreenshotAction", + "ScrollAction", "SharepointGroundingToolParameters", "SharepointToolDefinition", "SubmitToolApprovalAction", @@ -416,6 +458,7 @@ "ToolOutput", "ToolResources", "TruncationObject", + "TypeAction", "VectorStore", "VectorStoreAutoChunkingStrategyRequest", "VectorStoreAutoChunkingStrategyResponse", @@ -432,11 +475,13 @@ "VectorStoreStaticChunkingStrategyOptions", "VectorStoreStaticChunkingStrategyRequest", "VectorStoreStaticChunkingStrategyResponse", + "WaitAction", "AgentStreamEvent", "AgentsNamedToolChoiceType", "AgentsResponseFormatMode", "AgentsToolChoiceOptionMode", "AzureAISearchQueryType", + "ComputerUseEnvironment", "DoneEvent", "ErrorEvent", "FilePurpose", @@ -449,6 +494,7 @@ "MessageRole", "MessageStatus", "MessageStreamEvent", + "MouseButton", "OpenApiAuthType", "ResponseFormat", "RunAdditionalFieldList", diff --git a/sdk/ai/azure-ai-agents/azure/ai/agents/models/_enums.py b/sdk/ai/azure-ai-agents/azure/ai/agents/models/_enums.py index d62db671633e..00e0cd03be58 100644 --- a/sdk/ai/azure-ai-agents/azure/ai/agents/models/_enums.py +++ b/sdk/ai/azure-ai-agents/azure/ai/agents/models/_enums.py @@ -35,6 +35,8 @@ class AgentsNamedToolChoiceType(str, Enum, metaclass=CaseInsensitiveEnumMeta): """Tool type ``deep_research``""" MCP = "mcp" """Tool type ``mcp``""" + COMPUTER_USE_PREVIEW = "computer_use_preview" + """Tool type ``computer_use_preview``""" class AgentsResponseFormatMode(str, Enum, metaclass=CaseInsensitiveEnumMeta): @@ -150,6 +152,19 @@ class AzureAISearchQueryType(str, Enum, metaclass=CaseInsensitiveEnumMeta): """Query type ``vector_semantic_hybrid``""" +class ComputerUseEnvironment(str, Enum, metaclass=CaseInsensitiveEnumMeta): + """The environment types supported by the computer use tool.""" + + WINDOWS = "windows" + """Windows environment""" + MAC = "mac" + """Mac environment""" + LINUX = "linux" + """Linux environment""" + BROWSER = "browser" + """Browser environment""" + + class DoneEvent(str, Enum, metaclass=CaseInsensitiveEnumMeta): """Terminal event indicating the successful end of a stream.""" @@ -299,6 +314,21 @@ class MessageStreamEvent(str, Enum, metaclass=CaseInsensitiveEnumMeta): """Event sent before a message is completed. The data of this event is of type ThreadMessage""" +class MouseButton(str, Enum, metaclass=CaseInsensitiveEnumMeta): + """The mouse button types supported by click actions.""" + + LEFT = "left" + """Left mouse button""" + RIGHT = "right" + """Right mouse button""" + WHEEL = "wheel" + """Mouse wheel button""" + BACK = "back" + """Back mouse button""" + FORWARD = "forward" + """Forward mouse button""" + + class OpenApiAuthType(str, Enum, metaclass=CaseInsensitiveEnumMeta): """Authentication type for OpenApi endpoint. Allowed types are: diff --git a/sdk/ai/azure-ai-agents/azure/ai/agents/models/_models.py b/sdk/ai/azure-ai-agents/azure/ai/agents/models/_models.py index a71811f6d4c4..63a4342e48f2 100644 --- a/sdk/ai/azure-ai-agents/azure/ai/agents/models/_models.py +++ b/sdk/ai/azure-ai-agents/azure/ai/agents/models/_models.py @@ -236,7 +236,7 @@ class AgentsNamedToolChoice(_Model): :ivar type: the type of tool. If type is ``function``, the function name must be set. Required. Known values are: "function", "code_interpreter", "file_search", "bing_grounding", "fabric_dataagent", "sharepoint_grounding", "azure_ai_search", "bing_custom_search", - "connected_agent", "deep_research", and "mcp". + "connected_agent", "deep_research", "mcp", and "computer_use_preview". :vartype type: str or ~azure.ai.agents.models.AgentsNamedToolChoiceType :ivar function: The name of the function to call. :vartype function: ~azure.ai.agents.models.FunctionName @@ -248,7 +248,7 @@ class AgentsNamedToolChoice(_Model): """the type of tool. If type is ``function``, the function name must be set. Required. Known values are: \"function\", \"code_interpreter\", \"file_search\", \"bing_grounding\", \"fabric_dataagent\", \"sharepoint_grounding\", \"azure_ai_search\", \"bing_custom_search\", - \"connected_agent\", \"deep_research\", and \"mcp\".""" + \"connected_agent\", \"deep_research\", \"mcp\", and \"computer_use_preview\".""" function: Optional["_models.FunctionName"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) """The name of the function to call.""" @@ -512,9 +512,9 @@ class ToolDefinition(_Model): You probably want to use the sub-classes and not this class directly. Known sub-classes are: AzureAISearchToolDefinition, AzureFunctionToolDefinition, BingCustomSearchToolDefinition, BingGroundingToolDefinition, BrowserAutomationToolDefinition, CodeInterpreterToolDefinition, - ConnectedAgentToolDefinition, DeepResearchToolDefinition, MicrosoftFabricToolDefinition, - FileSearchToolDefinition, FunctionToolDefinition, MCPToolDefinition, OpenApiToolDefinition, - SharepointToolDefinition + ComputerUseToolDefinition, ConnectedAgentToolDefinition, DeepResearchToolDefinition, + MicrosoftFabricToolDefinition, FileSearchToolDefinition, FunctionToolDefinition, + MCPToolDefinition, OpenApiToolDefinition, SharepointToolDefinition :ivar type: The object type. Required. Default value is None. :vartype type: str @@ -1179,6 +1179,85 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) +class ComputerUseAction(_Model): + """An abstract representation of a computer use action. + + You probably want to use the sub-classes and not this class directly. Known sub-classes are: + ClickAction, DoubleClickAction, DragAction, KeyPressAction, MoveAction, ScreenshotAction, + ScrollAction, TypeAction, WaitAction + + :ivar type: The type of computer use action. Required. Default value is None. + :vartype type: str + """ + + __mapping__: Dict[str, _Model] = {} + type: str = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) + """The type of computer use action. Required. Default value is None.""" + + @overload + def __init__( + self, + *, + type: str, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + + +class ClickAction(ComputerUseAction, discriminator="click"): + """A click action. + + :ivar type: Specifies the event type. For a click action, this property is always set to click. + Required. Default value is "click". + :vartype type: str + :ivar x: The x-coordinate where the click occurred. Required. + :vartype x: int + :ivar y: The y-coordinate where the click occurred. Required. + :vartype y: int + :ivar button: Indicates which mouse button was pressed during the click. Required. Known values + are: "left", "right", "wheel", "back", and "forward". + :vartype button: str or ~azure.ai.agents.models.MouseButton + """ + + type: Literal["click"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """Specifies the event type. For a click action, this property is always set to click. Required. + Default value is \"click\".""" + x: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The x-coordinate where the click occurred. Required.""" + y: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The y-coordinate where the click occurred. Required.""" + button: Union[str, "_models.MouseButton"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """Indicates which mouse button was pressed during the click. Required. Known values are: + \"left\", \"right\", \"wheel\", \"back\", and \"forward\".""" + + @overload + def __init__( + self, + *, + x: int, + y: int, + button: Union[str, "_models.MouseButton"], + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="click", **kwargs) + + class CodeInterpreterToolDefinition(ToolDefinition, discriminator="code_interpreter"): """The input definition information for a code interpreter tool as used to configure an agent. @@ -1247,6 +1326,209 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) +class ComputerScreenshot(_Model): + """The output from a computer use tool representing a screenshot. + + :ivar type: Specifies the event type. For a computer screenshot, this property is always set to + computer_screenshot. Required. Default value is "computer_screenshot". + :vartype type: str + :ivar file_id: The identifier of an uploaded file that contains the screenshot. + :vartype file_id: str + :ivar image_url: The URL of the screenshot image. + :vartype image_url: str + """ + + type: Literal["computer_screenshot"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """Specifies the event type. For a computer screenshot, this property is always set to + computer_screenshot. Required. Default value is \"computer_screenshot\".""" + file_id: Optional[str] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The identifier of an uploaded file that contains the screenshot.""" + image_url: Optional[str] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The URL of the screenshot image.""" + + @overload + def __init__( + self, + *, + file_id: Optional[str] = None, + image_url: Optional[str] = None, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.type: Literal["computer_screenshot"] = "computer_screenshot" + + +class ToolOutput(_Model): + """The data provided during a tool outputs submission to resolve pending tool calls and allow the + model to continue. + + You probably want to use the sub-classes and not this class directly. Known sub-classes are: + ComputerToolOutput, FunctionToolOutput + + :ivar type: The object type for the tool output. Required. Default value is None. + :vartype type: str + :ivar tool_call_id: The ID of the tool call being resolved, as provided in the tool calls of a + required action from a run. Required. + :vartype tool_call_id: str + """ + + __mapping__: Dict[str, _Model] = {} + type: str = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) + """The object type for the tool output. Required. Default value is None.""" + tool_call_id: str = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The ID of the tool call being resolved, as provided in the tool calls of a required action from + a run. Required.""" + + @overload + def __init__( + self, + *, + type: str, + tool_call_id: str, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + + +class ComputerToolOutput(ToolOutput, discriminator="computer_call_output"): + """The output from a computer use tool to be submitted. + + :ivar tool_call_id: The ID of the tool call being resolved, as provided in the tool calls of a + required action from a run. Required. + :vartype tool_call_id: str + :ivar type: The object type, which is always 'computer_call_output'. Required. Default value is + "computer_call_output". + :vartype type: str + :ivar output: The output from the computer use tool. Required. + :vartype output: ~azure.ai.agents.models.ComputerScreenshot + :ivar acknowledged_safety_checks: Safety checks that have been acknowledged by the developer. + :vartype acknowledged_safety_checks: list[~azure.ai.agents.models.SafetyCheck] + """ + + type: Literal["computer_call_output"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """The object type, which is always 'computer_call_output'. Required. Default value is + \"computer_call_output\".""" + output: "_models.ComputerScreenshot" = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The output from the computer use tool. Required.""" + acknowledged_safety_checks: Optional[List["_models.SafetyCheck"]] = rest_field( + visibility=["read", "create", "update", "delete", "query"] + ) + """Safety checks that have been acknowledged by the developer.""" + + @overload + def __init__( + self, + *, + tool_call_id: str, + output: "_models.ComputerScreenshot", + acknowledged_safety_checks: Optional[List["_models.SafetyCheck"]] = None, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="computer_call_output", **kwargs) + + +class ComputerUseToolDefinition(ToolDefinition, discriminator="computer_use_preview"): + """The input definition information for a Computer Use tool as used to configure an agent. + + :ivar type: The object type, which is always 'computer_use_preview'. Required. Default value is + "computer_use_preview". + :vartype type: str + :ivar computer_use_preview: The computer use tool parameters. Required. + :vartype computer_use_preview: ~azure.ai.agents.models.ComputerUseToolParameters + """ + + type: Literal["computer_use_preview"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """The object type, which is always 'computer_use_preview'. Required. Default value is + \"computer_use_preview\".""" + computer_use_preview: "_models.ComputerUseToolParameters" = rest_field( + visibility=["read", "create", "update", "delete", "query"] + ) + """The computer use tool parameters. Required.""" + + @overload + def __init__( + self, + *, + computer_use_preview: "_models.ComputerUseToolParameters", + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="computer_use_preview", **kwargs) + + +class ComputerUseToolParameters(_Model): + """The computer use tool parameters. + + :ivar display_width: The display width for the computer use tool. Required. + :vartype display_width: int + :ivar display_height: The display height for the computer use tool. Required. + :vartype display_height: int + :ivar environment: The environment for the computer use tool. Required. Known values are: + "windows", "mac", "linux", and "browser". + :vartype environment: str or ~azure.ai.agents.models.ComputerUseEnvironment + """ + + display_width: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The display width for the computer use tool. Required.""" + display_height: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The display height for the computer use tool. Required.""" + environment: Union[str, "_models.ComputerUseEnvironment"] = rest_field( + visibility=["read", "create", "update", "delete", "query"] + ) + """The environment for the computer use tool. Required. Known values are: \"windows\", \"mac\", + \"linux\", and \"browser\".""" + + @overload + def __init__( + self, + *, + display_width: int, + display_height: int, + environment: Union[str, "_models.ComputerUseEnvironment"], + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + + class ConnectedAgentDetails(_Model): """Information for connecting one agent to another as a tool. @@ -1324,6 +1606,39 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, type="connected_agent", **kwargs) +class CoordinatePoint(_Model): + """A coordinate point with x and y values. + + :ivar x: The x-coordinate. Required. + :vartype x: int + :ivar y: The y-coordinate. Required. + :vartype y: int + """ + + x: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The x-coordinate. Required.""" + y: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The y-coordinate. Required.""" + + @overload + def __init__( + self, + *, + x: int, + y: int, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + + class DeepResearchBingGroundingConnection(_Model): """The connection resource ID for the Bing grounding resource . @@ -1425,6 +1740,79 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, type="deep_research", **kwargs) +class DoubleClickAction(ComputerUseAction, discriminator="double_click"): + """A double click action. + + :ivar type: Specifies the event type. For a double click action, this property is always set to + double_click. Required. Default value is "double_click". + :vartype type: str + :ivar x: The x-coordinate where the double click occurred. Required. + :vartype x: int + :ivar y: The y-coordinate where the double click occurred. Required. + :vartype y: int + """ + + type: Literal["double_click"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """Specifies the event type. For a double click action, this property is always set to + double_click. Required. Default value is \"double_click\".""" + x: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The x-coordinate where the double click occurred. Required.""" + y: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The y-coordinate where the double click occurred. Required.""" + + @overload + def __init__( + self, + *, + x: int, + y: int, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="double_click", **kwargs) + + +class DragAction(ComputerUseAction, discriminator="drag"): + """A drag action. + + :ivar type: Specifies the event type. For a drag action, this property is always set to drag. + Required. Default value is "drag". + :vartype type: str + :ivar path: An array of coordinates representing the path of the drag action. Required. + :vartype path: list[~azure.ai.agents.models.CoordinatePoint] + """ + + type: Literal["drag"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """Specifies the event type. For a drag action, this property is always set to drag. Required. + Default value is \"drag\".""" + path: List["_models.CoordinatePoint"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """An array of coordinates representing the path of the drag action. Required.""" + + @overload + def __init__( + self, + *, + path: List["_models.CoordinatePoint"], + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="drag", **kwargs) + + class FabricDataAgentToolParameters(_Model): """The fabric data agent tool parameters. @@ -1893,6 +2281,44 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, type="function", **kwargs) +class FunctionToolOutput(ToolOutput, discriminator="function_call_output"): + """The output from a function tool to be submitted. + + :ivar tool_call_id: The ID of the tool call being resolved, as provided in the tool calls of a + required action from a run. Required. + :vartype tool_call_id: str + :ivar type: The object type, which is always 'function_call_output'. Required. Default value is + "function_call_output". + :vartype type: str + :ivar output: The output from the function tool to be submitted. Required. + :vartype output: str + """ + + type: Literal["function_call_output"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """The object type, which is always 'function_call_output'. Required. Default value is + \"function_call_output\".""" + output: str = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The output from the function tool to be submitted. Required.""" + + @overload + def __init__( + self, + *, + tool_call_id: str, + output: str, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="function_call_output", **kwargs) + + class IncompleteRunDetails(_Model): """Details on why the run is incomplete. Will be ``null`` if the run is not incomplete. @@ -1927,6 +2353,42 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) +class KeyPressAction(ComputerUseAction, discriminator="keypress"): + """A collection of keypresses the model would like to perform. + + :ivar type: Specifies the event type. For a keypress action, this property is always set to + keypress. Required. Default value is "keypress". + :vartype type: str + :ivar keys_property: The combination of keys the model is requesting to be pressed. This is an + array of strings, each representing a key. Required. + :vartype keys_property: list[str] + """ + + type: Literal["keypress"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """Specifies the event type. For a keypress action, this property is always set to keypress. + Required. Default value is \"keypress\".""" + keys_property: List[str] = rest_field(name="keys", visibility=["read", "create", "update", "delete", "query"]) + """The combination of keys the model is requesting to be pressed. This is an array of strings, + each representing a key. Required.""" + + @overload + def __init__( + self, + *, + keys_property: List[str], + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="keypress", **kwargs) + + class MCPApprovalPerTool(_Model): """Customized MCP approval object, listing tools requiring and not requiring approvals. @@ -3387,7 +3849,46 @@ def __init__(self, mapping: Mapping[str, Any]) -> None: """ def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, type="fabric_dataagent", **kwargs) + super().__init__(*args, type="fabric_dataagent", **kwargs) + + +class MoveAction(ComputerUseAction, discriminator="move"): + """A mouse move action. + + :ivar type: Specifies the event type. For a move action, this property is always set to move. + Required. Default value is "move". + :vartype type: str + :ivar x: The x-coordinate to move to. Required. + :vartype x: int + :ivar y: The y-coordinate to move to. Required. + :vartype y: int + """ + + type: Literal["move"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """Specifies the event type. For a move action, this property is always set to move. Required. + Default value is \"move\".""" + x: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The x-coordinate to move to. Required.""" + y: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The y-coordinate to move to. Required.""" + + @overload + def __init__( + self, + *, + x: int, + y: int, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="move", **kwargs) class OpenApiAuthDetails(_Model): @@ -3698,7 +4199,7 @@ class RequiredToolCall(_Model): """An abstract representation of a tool invocation needed by the model to continue a run. You probably want to use the sub-classes and not this class directly. Known sub-classes are: - RequiredFunctionToolCall, RequiredMcpToolCall + RequiredComputerUseToolCall, RequiredFunctionToolCall, RequiredMcpToolCall :ivar type: The object type for the required tool call. Required. Default value is None. :vartype type: str @@ -3732,6 +4233,91 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) +class RequiredComputerUseToolCall(RequiredToolCall, discriminator="computer_use_preview"): + """A representation of a requested call to a Computer Use tool, needed by the model to continue + evaluation of a run. + + :ivar id: The ID of the tool call. This ID must be referenced when submitting tool outputs. + Required. + :vartype id: str + :ivar type: The object type of the required tool call. Always 'computer_use_preview' for + Computer Use tools. Required. Default value is "computer_use_preview". + :vartype type: str + :ivar computer_use_preview: Detailed information about the computer use action to be executed. + Required. + :vartype computer_use_preview: ~azure.ai.agents.models.RequiredComputerUseToolCallDetails + """ + + type: Literal["computer_use_preview"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """The object type of the required tool call. Always 'computer_use_preview' for Computer Use + tools. Required. Default value is \"computer_use_preview\".""" + computer_use_preview: "_models.RequiredComputerUseToolCallDetails" = rest_field( + visibility=["read", "create", "update", "delete", "query"] + ) + """Detailed information about the computer use action to be executed. Required.""" + + @overload + def __init__( + self, + *, + id: str, # pylint: disable=redefined-builtin + computer_use_preview: "_models.RequiredComputerUseToolCallDetails", + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="computer_use_preview", **kwargs) + + +class RequiredComputerUseToolCallDetails(_Model): + """The detailed information for a computer use tool invocation. + + :ivar action: The action to be performed by the computer use tool. Required. + :vartype action: ~azure.ai.agents.models.ComputerUseAction + :ivar pending_safety_checks: Safety checks that are pending acknowledgment by the developer. + Required. + :vartype pending_safety_checks: list[~azure.ai.agents.models.SafetyCheck] + :ivar type: The type of the computer call. Always 'computer_call'. Required. Default value is + "computer_call". + :vartype type: str + """ + + action: "_models.ComputerUseAction" = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The action to be performed by the computer use tool. Required.""" + pending_safety_checks: List["_models.SafetyCheck"] = rest_field( + visibility=["read", "create", "update", "delete", "query"] + ) + """Safety checks that are pending acknowledgment by the developer. Required.""" + type: Literal["computer_call"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The type of the computer call. Always 'computer_call'. Required. Default value is + \"computer_call\".""" + + @overload + def __init__( + self, + *, + action: "_models.ComputerUseAction", + pending_safety_checks: List["_models.SafetyCheck"], + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.type: Literal["computer_call"] = "computer_call" + + class RequiredFunctionToolCall(RequiredToolCall, discriminator="function"): """A representation of a requested call to a function tool, needed by the model to continue evaluation of a run. @@ -4149,7 +4735,7 @@ class RunStepToolCall(_Model): You probably want to use the sub-classes and not this class directly. Known sub-classes are: RunStepAzureAISearchToolCall, RunStepBingCustomSearchToolCall, RunStepBingGroundingToolCall, - RunStepBrowserAutomationToolCall, RunStepCodeInterpreterToolCall, + RunStepBrowserAutomationToolCall, RunStepCodeInterpreterToolCall, RunStepComputerUseToolCall, RunStepConnectedAgentToolCall, RunStepDeepResearchToolCall, RunStepMicrosoftFabricToolCall, RunStepFileSearchToolCall, RunStepFunctionToolCall, RunStepMcpToolCall, RunStepOpenAPIToolCall, RunStepSharepointToolCall @@ -4592,6 +5178,104 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) +class RunStepComputerUseToolCall(RunStepToolCall, discriminator="computer_use_preview"): + """A record of a call to a Computer Use tool, issued by the model in evaluation of a defined tool, + that represents + executed computer automation actions. + + :ivar id: The ID of the tool call. This ID must be referenced when you submit tool outputs. + Required. + :vartype id: str + :ivar type: The object type, which is always 'computer_use_preview'. Required. Default value is + "computer_use_preview". + :vartype type: str + :ivar computer_use_preview: The detailed information about the computer use tool call. + Required. + :vartype computer_use_preview: ~azure.ai.agents.models.RunStepComputerUseToolCallDetails + :ivar output: The output from the computer use tool. Required. + :vartype output: ~azure.ai.agents.models.ComputerScreenshot + :ivar acknowledged_safety_checks: Safety checks that have been acknowledged by the developer. + :vartype acknowledged_safety_checks: list[~azure.ai.agents.models.SafetyCheck] + """ + + type: Literal["computer_use_preview"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """The object type, which is always 'computer_use_preview'. Required. Default value is + \"computer_use_preview\".""" + computer_use_preview: "_models.RunStepComputerUseToolCallDetails" = rest_field( + visibility=["read", "create", "update", "delete", "query"] + ) + """The detailed information about the computer use tool call. Required.""" + output: "_models.ComputerScreenshot" = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The output from the computer use tool. Required.""" + acknowledged_safety_checks: Optional[List["_models.SafetyCheck"]] = rest_field( + visibility=["read", "create", "update", "delete", "query"] + ) + """Safety checks that have been acknowledged by the developer.""" + + @overload + def __init__( + self, + *, + id: str, # pylint: disable=redefined-builtin + computer_use_preview: "_models.RunStepComputerUseToolCallDetails", + output: "_models.ComputerScreenshot", + acknowledged_safety_checks: Optional[List["_models.SafetyCheck"]] = None, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="computer_use_preview", **kwargs) + + +class RunStepComputerUseToolCallDetails(_Model): + """The detailed information about a computer use tool call. + + :ivar action: The action to be performed by the computer use tool. Required. + :vartype action: ~azure.ai.agents.models.ComputerUseAction + :ivar pending_safety_checks: Safety checks that are pending acknowledgment by the developer. + Required. + :vartype pending_safety_checks: list[~azure.ai.agents.models.SafetyCheck] + :ivar type: The type of the computer call. Always 'computer_call'. Required. Default value is + "computer_call". + :vartype type: str + """ + + action: "_models.ComputerUseAction" = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The action to be performed by the computer use tool. Required.""" + pending_safety_checks: List["_models.SafetyCheck"] = rest_field( + visibility=["read", "create", "update", "delete", "query"] + ) + """Safety checks that are pending acknowledgment by the developer. Required.""" + type: Literal["computer_call"] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The type of the computer call. Always 'computer_call'. Required. Default value is + \"computer_call\".""" + + @overload + def __init__( + self, + *, + action: "_models.ComputerUseAction", + pending_safety_checks: List["_models.SafetyCheck"], + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.type: Literal["computer_call"] = "computer_call" + + class RunStepConnectedAgent(_Model): """The detailed information about connected agent tool call. @@ -6213,6 +6897,121 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, type=RunStepType.TOOL_CALLS, **kwargs) +class SafetyCheck(_Model): + """Safety check that has been acknowledged by the developer. + + :ivar id: The ID of the pending safety check. Required. + :vartype id: str + :ivar code: The type of the pending safety check. + :vartype code: str + :ivar message: Details about the pending safety check. + :vartype message: str + """ + + id: str = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The ID of the pending safety check. Required.""" + code: Optional[str] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The type of the pending safety check.""" + message: Optional[str] = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """Details about the pending safety check.""" + + @overload + def __init__( + self, + *, + id: str, # pylint: disable=redefined-builtin + code: Optional[str] = None, + message: Optional[str] = None, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + + +class ScreenshotAction(ComputerUseAction, discriminator="screenshot"): + """A screenshot action. + + :ivar type: Specifies the event type. For a screenshot action, this property is always set to + screenshot. Required. Default value is "screenshot". + :vartype type: str + """ + + type: Literal["screenshot"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """Specifies the event type. For a screenshot action, this property is always set to screenshot. + Required. Default value is \"screenshot\".""" + + @overload + def __init__( + self, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="screenshot", **kwargs) + + +class ScrollAction(ComputerUseAction, discriminator="scroll"): + """A scroll action. + + :ivar type: Specifies the event type. For a scroll action, this property is always set to + scroll. Required. Default value is "scroll". + :vartype type: str + :ivar x: The x-coordinate where the scroll occurred. Required. + :vartype x: int + :ivar y: The y-coordinate where the scroll occurred. Required. + :vartype y: int + :ivar scroll_x: The horizontal scroll distance. Required. + :vartype scroll_x: int + :ivar scroll_y: The vertical scroll distance. Required. + :vartype scroll_y: int + """ + + type: Literal["scroll"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """Specifies the event type. For a scroll action, this property is always set to scroll. Required. + Default value is \"scroll\".""" + x: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The x-coordinate where the scroll occurred. Required.""" + y: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The y-coordinate where the scroll occurred. Required.""" + scroll_x: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The horizontal scroll distance. Required.""" + scroll_y: int = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The vertical scroll distance. Required.""" + + @overload + def __init__( + self, + *, + x: int, + y: int, + scroll_x: int, + scroll_y: int, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="scroll", **kwargs) + + class SharepointGroundingToolParameters(_Model): """The sharepoint grounding tool parameters. @@ -6934,42 +7733,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) -class ToolOutput(_Model): - """The data provided during a tool outputs submission to resolve pending tool calls and allow the - model to continue. - - :ivar tool_call_id: The ID of the tool call being resolved, as provided in the tool calls of a - required action from a run. - :vartype tool_call_id: str - :ivar output: The output from the tool to be submitted. - :vartype output: str - """ - - tool_call_id: Optional[str] = rest_field(visibility=["read", "create", "update", "delete", "query"]) - """The ID of the tool call being resolved, as provided in the tool calls of a required action from - a run.""" - output: Optional[str] = rest_field(visibility=["read", "create", "update", "delete", "query"]) - """The output from the tool to be submitted.""" - - @overload - def __init__( - self, - *, - tool_call_id: Optional[str] = None, - output: Optional[str] = None, - ) -> None: ... - - @overload - def __init__(self, mapping: Mapping[str, Any]) -> None: - """ - :param mapping: raw JSON to initialize the model. - :type mapping: Mapping[str, Any] - """ - - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - - class ToolResources(_Model): """A set of resources that are used by the agent's tools. The resources are specific to the type of @@ -7075,6 +7838,40 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) +class TypeAction(ComputerUseAction, discriminator="type"): + """An action to type in text. + + :ivar type: Specifies the event type. For a type action, this property is always set to type. + Required. Default value is "type". + :vartype type: str + :ivar text: The text to type. Required. + :vartype text: str + """ + + type: Literal["type"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """Specifies the event type. For a type action, this property is always set to type. Required. + Default value is \"type\".""" + text: str = rest_field(visibility=["read", "create", "update", "delete", "query"]) + """The text to type. Required.""" + + @overload + def __init__( + self, + *, + text: str, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="type", **kwargs) + + class UploadFileRequest(_Model): """UploadFileRequest. @@ -7900,3 +8697,31 @@ def __init__(self, mapping: Mapping[str, Any]) -> None: def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, type=VectorStoreChunkingStrategyResponseType.STATIC, **kwargs) + + +class WaitAction(ComputerUseAction, discriminator="wait"): + """A wait action. + + :ivar type: Specifies the event type. For a wait action, this property is always set to wait. + Required. Default value is "wait". + :vartype type: str + """ + + type: Literal["wait"] = rest_discriminator(name="type", visibility=["read", "create", "update", "delete", "query"]) # type: ignore + """Specifies the event type. For a wait action, this property is always set to wait. Required. + Default value is \"wait\".""" + + @overload + def __init__( + self, + ) -> None: ... + + @overload + def __init__(self, mapping: Mapping[str, Any]) -> None: + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, type="wait", **kwargs) diff --git a/sdk/ai/azure-ai-agents/azure/ai/agents/models/_patch.py b/sdk/ai/azure-ai-agents/azure/ai/agents/models/_patch.py index da0646632140..044d819e8ad1 100644 --- a/sdk/ai/azure-ai-agents/azure/ai/agents/models/_patch.py +++ b/sdk/ai/azure-ai-agents/azure/ai/agents/models/_patch.py @@ -53,6 +53,8 @@ CodeInterpreterToolResource, ConnectedAgentToolDefinition, ConnectedAgentDetails, + ComputerUseToolDefinition, + ComputerUseToolParameters, FileSearchToolDefinition, FileSearchToolResource, FunctionDefinition, @@ -1164,6 +1166,45 @@ def execute(self, tool_call: Any) -> Any: pass +class ComputerUseTool(Tool[ComputerUseToolDefinition]): + """ + A tool that enables the agent to perform computer use actions (preview). + + :param display_width: The display width for the computer use tool. + :type display_width: int + :param display_height: The display height for the computer use tool. + :type display_height: int + :param environment: The target environment for computer use, e.g. "browser", "windows", "mac", or "linux". + :type environment: str + """ + + def __init__(self, display_width: int, display_height: int, environment: str): + self._params = ComputerUseToolParameters( + display_width=display_width, display_height=display_height, environment=environment + ) + + @property + def definitions(self) -> List[ComputerUseToolDefinition]: + """ + Get the Computer Use tool definitions. + + :rtype: List[ToolDefinition] + """ + return [ComputerUseToolDefinition(computer_use_preview=self._params)] + + @property + def resources(self) -> ToolResources: + """ + Get the tool resources. + + :rtype: ToolResources + """ + return ToolResources() + + def execute(self, tool_call: Any) -> Any: # noqa: D401 - client-side execution not applicable + pass + + class BingGroundingTool(Tool[BingGroundingToolDefinition]): """ A tool that searches for information using Bing. @@ -2155,6 +2196,7 @@ def _is_valid_connection_id(connection_id: str) -> bool: "BaseAsyncAgentEventHandler", "BaseAgentEventHandler", "BrowserAutomationTool", + "ComputerUseTool", "CodeInterpreterTool", "ConnectedAgentTool", "DeepResearchTool", diff --git a/sdk/ai/azure-ai-agents/azure/ai/agents/operations/_operations.py b/sdk/ai/azure-ai-agents/azure/ai/agents/operations/_operations.py index 5b2f46f1423f..b70f9c497553 100644 --- a/sdk/ai/azure-ai-agents/azure/ai/agents/operations/_operations.py +++ b/sdk/ai/azure-ai-agents/azure/ai/agents/operations/_operations.py @@ -1368,7 +1368,7 @@ def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -1455,7 +1455,7 @@ def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -1510,7 +1510,7 @@ def get(self, thread_id: str, **kwargs: Any) -> _models.AgentThread: except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -1674,7 +1674,7 @@ def update( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -1735,7 +1735,7 @@ def _delete_thread(self, thread_id: str, **kwargs: Any) -> _models._models.Threa except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -1943,7 +1943,7 @@ def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2095,7 +2095,7 @@ def get(self, thread_id: str, message_id: str, **kwargs: Any) -> _models.ThreadM except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2256,7 +2256,7 @@ def update( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2649,7 +2649,7 @@ def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2740,7 +2740,7 @@ def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -2798,7 +2798,7 @@ def get(self, thread_id: str, run_id: str, **kwargs: Any) -> _models.ThreadRun: except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -2959,7 +2959,7 @@ def update( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3132,7 +3132,7 @@ def submit_tool_outputs( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3196,7 +3196,7 @@ def cancel(self, thread_id: str, run_id: str, **kwargs: Any) -> _models.ThreadRu except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3295,7 +3295,7 @@ def get( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3397,7 +3397,7 @@ def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -3473,7 +3473,7 @@ def list( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3545,7 +3545,7 @@ def _upload_file(self, body: Union[_models._models.UploadFileRequest, JSON], **k except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3606,7 +3606,7 @@ def _delete_file(self, file_id: str, **kwargs: Any) -> _models._models.FileDelet except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3669,7 +3669,7 @@ def get(self, file_id: str, **kwargs: Any) -> _models.FileInfo: except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -3730,7 +3730,7 @@ def _get_file_content(self, file_id: str, **kwargs: Any) -> Iterator[bytes]: except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) deserialized = response.iter_bytes() @@ -3832,7 +3832,7 @@ def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -4004,7 +4004,7 @@ def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4065,7 +4065,7 @@ def get(self, vector_store_id: str, **kwargs: Any) -> _models.VectorStore: except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4227,7 +4227,7 @@ def modify( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4289,7 +4289,7 @@ def _delete_vector_store(self, vector_store_id: str, **kwargs: Any) -> _models._ except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4405,7 +4405,7 @@ def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -4557,7 +4557,7 @@ def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4621,7 +4621,7 @@ def get(self, vector_store_id: str, file_id: str, **kwargs: Any) -> _models.Vect except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4689,7 +4689,7 @@ def _delete_vector_store_file( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4867,7 +4867,7 @@ def create( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4931,7 +4931,7 @@ def get(self, vector_store_id: str, batch_id: str, **kwargs: Any) -> _models.Vec except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -4996,7 +4996,7 @@ def cancel(self, vector_store_id: str, batch_id: str, **kwargs: Any) -> _models. except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -5096,7 +5096,7 @@ def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -5325,7 +5325,7 @@ def create_agent( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -5412,7 +5412,7 @@ def get_next(_continuation_token=None): if response.status_code not in [200]: map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) return pipeline_response @@ -5467,7 +5467,7 @@ def get_agent(self, agent_id: str, **kwargs: Any) -> _models.Agent: except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -5714,7 +5714,7 @@ def update_agent( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -5775,7 +5775,7 @@ def _delete_agent(self, agent_id: str, **kwargs: Any) -> _models._models.AgentDe except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: @@ -6095,7 +6095,7 @@ def create_thread_and_run( except (StreamConsumedError, StreamClosedError): pass map_error(status_code=response.status_code, response=response, error_map=error_map) - error = _failsafe_deserialize(_models.AgentV1Error, response.json()) + error = _failsafe_deserialize(_models.AgentV1Error, response) raise HttpResponseError(response=response, model=error) if _stream: diff --git a/sdk/ai/azure-ai-agents/samples/agents_tools/sample_agents_computer_use.py b/sdk/ai/azure-ai-agents/samples/agents_tools/sample_agents_computer_use.py new file mode 100644 index 000000000000..b68bae3acac3 --- /dev/null +++ b/sdk/ai/azure-ai-agents/samples/agents_tools/sample_agents_computer_use.py @@ -0,0 +1,116 @@ +# pylint: disable=line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +DESCRIPTION: + This sample demonstrates how to use agent operations with the Computer Use tool (preview) + using a synchronous client. + +USAGE: + python sample_agents_computer_use.py + + Before running the sample: + + pip install azure-ai-agents --pre + pip install azure-ai-projects azure-identity + + Set these environment variables with your own values: + 1) PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview + page of your Azure AI Foundry portal. + 2) MODEL_DEPLOYMENT_NAME - The deployment name of the AI model, as found under the "Name" column in + the "Models + endpoints" tab in your Azure AI Foundry project. + + Optional: + - To target a specific environment, set COMPUTER_USE_ENVIRONMENT to one of: windows, mac, linux, browser + Otherwise defaults to 'browser'. +""" + +import os +from azure.ai.projects import AIProjectClient +from azure.ai.agents.models import ( + MessageRole, + RunStepToolCallDetails, + RunStepComputerUseToolCall, + ComputerUseTool, +) +from azure.identity import DefaultAzureCredential + +project_client = AIProjectClient(endpoint=os.environ["PROJECT_ENDPOINT"], credential=DefaultAzureCredential()) + +# [START create_agent_with_computer_use] +# Initialize Computer Use tool with a browser-sized viewport +environment = os.environ.get("COMPUTER_USE_ENVIRONMENT", "browser") +computer_use = ComputerUseTool(display_width=1280, display_height=800, environment=environment) + +with project_client: + + agents_client = project_client.agents + + # Create a new Agent that has the Computer Use tool attached. + agent = agents_client.create_agent( + model=os.environ["MODEL_DEPLOYMENT_NAME"], + name="my-agent-computer-use", + instructions=( + "You are an Agent helping with computer use tasks. " + "You can perform actions and browse as needed using the Computer Use tool available to you." + ), + tools=computer_use.definitions, + ) + + # [END create_agent_with_computer_use] + + print(f"Created agent, ID: {agent.id}") + + # Create thread for communication + thread = agents_client.threads.create() + print(f"Created thread, ID: {thread.id}") + + # Create message to thread + message = agents_client.messages.create( + thread_id=thread.id, + role=MessageRole.USER, + content=( + "Open the Microsoft homepage and take a screenshot of the landing page." + ), + ) + print(f"Created message, ID: {message.id}") + + # Create and process agent run in thread with tools + print("Waiting for Agent run to complete. Please wait...") + run = agents_client.runs.create_and_process(thread_id=thread.id, agent_id=agent.id) + print(f"Run finished with status: {run.status}") + + if run.status == "failed": + print(f"Run failed: {run.last_error}") + + # Fetch run steps to get the details of the agent run + run_steps = agents_client.run_steps.list(thread_id=thread.id, run_id=run.id) + for step in run_steps: + print(f"Step {step.id} status: {step.status}") + + if isinstance(step.step_details, RunStepToolCallDetails): + print(" Tool calls:") + tool_calls = step.step_details.tool_calls + + for call in tool_calls: + print(f" Tool call ID: {call.id}") + print(f" Tool call type: {call.type}") + + if isinstance(call, RunStepComputerUseToolCall): + details = call.computer_use_preview + print(f" Computer use action type: {details.action.type}") + # The output is a ComputerScreenshot, which may include a file_id or image_url + print(f" Screenshot file_id: {call.output.file_id}") + print(f" Screenshot image_url: {call.output.image_url}") + + print() # extra newline between tool calls + + print() # extra newline between run steps + + # Optional: Delete the agent once the run is finished. + agents_client.delete_agent(agent.id) + print("Deleted agent") + diff --git a/sdk/ai/azure-ai-agents/tsp-location.yaml b/sdk/ai/azure-ai-agents/tsp-location.yaml index b88118a192cc..7a33477c4d53 100644 --- a/sdk/ai/azure-ai-agents/tsp-location.yaml +++ b/sdk/ai/azure-ai-agents/tsp-location.yaml @@ -1,4 +1,4 @@ directory: specification/ai/Azure.AI.Agents -commit: 511fb938a3f30e21347f0dd27e63f78e3ce42e5e +commit: 0f43fffbc24ebc32c3b90c31ea4c624d80b6a36c repo: Azure/azure-rest-api-specs additionalDirectories: