Skip to content

Commit 37933a0

Browse files
dargilcodargilco
authored
Add OpenAPI tool test with recordings (#44400)
1 parent 6f47585 commit 37933a0

File tree

4 files changed

+233
-3
lines changed

4 files changed

+233
-3
lines changed

‎sdk/ai/azure-ai-projects/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "python",
44
"TagPrefix": "python/ai/azure-ai-projects",
5-
"Tag": "python/ai/azure-ai-projects_93d1dc0fe7"
5+
"Tag": "python/ai/azure-ai-projects_56eacfd554"
66
}

‎sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_openapi_with_project_connection.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,6 @@
3636
PromptAgentDefinition,
3737
OpenApiAgentTool,
3838
OpenApiFunctionDefinition,
39-
OpenApiAnonymousAuthDetails,
40-
OpenApiManagedAuthDetails,
4139
OpenApiProjectConnectionAuthDetails,
4240
OpenApiProjectConnectionSecurityScheme,
4341
)

‎sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_openapi.py‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
# pylint: disable=too-many-lines,line-too-long,useless-suppression
2+
# ------------------------------------
3+
# Copyright (c) Microsoft Corporation.
4+
# Licensed under the MIT License.
5+
# ------------------------------------
6+
# cSpell:disable
7+
8+
import os
9+
10+
import pytest
11+
import jsonref
12+
from test_base import TestBase, servicePreparer
13+
from devtools_testutils import recorded_by_proxy, RecordedTransport
14+
from azure.ai.projects.models import (
15+
PromptAgentDefinition,
16+
OpenApiAgentTool,
17+
OpenApiFunctionDefinition,
18+
OpenApiAnonymousAuthDetails,
19+
)
20+
21+
22+
class TestAgentOpenApi(TestBase):
23+
24+
# To run this test:
25+
# pytest tests/agents/tools/test_agent_openapi.py::TestAgentOpenApi::test_agent_openapi -s
26+
@servicePreparer()
27+
@recorded_by_proxy(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX)
28+
def test_agent_openapi(self, **kwargs):
29+
"""
30+
Test agent with OpenAPI tool capabilities.
31+
32+
This test verifies that an agent can:
33+
1. Use OpenApiAgentTool to call external APIs defined by OpenAPI specifications
34+
2. Load and parse OpenAPI spec from JSON file
35+
3. Make API calls and incorporate results into responses
36+
37+
Routes used in this test:
38+
39+
Action REST API Route Client Method
40+
------+---------------------------------------------+-----------------------------------
41+
# Setup:
42+
POST /agents/{agent_name}/versions project_client.agents.create_version()
43+
44+
# Test focus:
45+
POST /openai/responses openai_client.responses.create() (with OpenApiAgentTool)
46+
47+
# Teardown:
48+
DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version()
49+
"""
50+
51+
model = self.test_agents_params["model_deployment_name"]
52+
53+
with (
54+
self.create_client(operation_group="agents", **kwargs) as project_client,
55+
project_client.get_openai_client() as openai_client,
56+
):
57+
# Load OpenAPI spec from assets folder
58+
weather_asset_file_path = os.path.abspath(
59+
os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/weather_openapi.json")
60+
)
61+
62+
assert os.path.exists(weather_asset_file_path), f"OpenAPI spec file not found at: {weather_asset_file_path}"
63+
print(f"Using OpenAPI spec file: {weather_asset_file_path}")
64+
65+
with open(weather_asset_file_path, "r") as f:
66+
openapi_weather = jsonref.loads(f.read())
67+
68+
# Create OpenAPI tool
69+
tool = OpenApiAgentTool(
70+
openapi=OpenApiFunctionDefinition(
71+
name="get_weather",
72+
spec=openapi_weather,
73+
description="Retrieve weather information for a location.",
74+
auth=OpenApiAnonymousAuthDetails(),
75+
)
76+
)
77+
78+
agent_name = "openapi-agent"
79+
80+
# Create agent with OpenAPI tool
81+
agent = project_client.agents.create_version(
82+
agent_name=agent_name,
83+
definition=PromptAgentDefinition(
84+
model=model,
85+
instructions="You are a helpful assistant.",
86+
tools=[tool],
87+
),
88+
description="Agent for testing OpenAPI tool capabilities.",
89+
)
90+
self._validate_agent_version(agent, expected_name=agent_name)
91+
92+
# Ask the agent to use the OpenAPI tool
93+
print("\nAsking agent to use OpenAPI tool to get weather...")
94+
95+
response = openai_client.responses.create(
96+
input="Use the OpenAPI tool to print out, what is the weather in Seattle, WA today.",
97+
extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},
98+
)
99+
self.validate_response(response)
100+
101+
# Get the response text
102+
response_text = response.output_text
103+
print(f"\nAgent's response: {response_text[:300]}...")
104+
105+
# Verify we got a meaningful response
106+
assert len(response_text) > 30, f"Expected a substantial response from the agent. Got '{response_text}'"
107+
108+
# The response should mention weather or Seattle (indicating the OpenAPI tool was used)
109+
response_lower = response_text.lower()
110+
assert any(
111+
keyword in response_lower for keyword in ["weather", "seattle", "temperature", "forecast"]
112+
), f"Expected response to contain weather information, but got: {response_text[:200]}"
113+
114+
print("\n✓ Agent successfully used OpenAPI tool to call external API")
115+
116+
# Teardown
117+
project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)
118+
print("Agent deleted")
119+
120+
# To run this test:
121+
# pytest tests/agents/tools/test_agent_openapi.py::TestAgentOpenApi::test_agent_openapi_with_auth -s
122+
@servicePreparer()
123+
@pytest.mark.skip(reason="Add test here once we have a Foundry Project with a connection with auth credentials")
124+
@recorded_by_proxy(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX)
125+
def test_agent_openapi_with_auth(self, **kwargs):
126+
pass

‎sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_openapi_async.py‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# pylint: disable=too-many-lines,line-too-long,useless-suppression
2+
# ------------------------------------
3+
# Copyright (c) Microsoft Corporation.
4+
# Licensed under the MIT License.
5+
# ------------------------------------
6+
# cSpell:disable
7+
8+
import os
9+
10+
import pytest
11+
import jsonref
12+
from test_base import TestBase, servicePreparer
13+
from devtools_testutils.aio import recorded_by_proxy_async
14+
from devtools_testutils import RecordedTransport
15+
from azure.ai.projects.models import (
16+
PromptAgentDefinition,
17+
OpenApiAgentTool,
18+
OpenApiFunctionDefinition,
19+
OpenApiAnonymousAuthDetails,
20+
)
21+
22+
23+
class TestAgentOpenApiAsync(TestBase):
24+
25+
# To run this test:
26+
# pytest tests/agents/tools/test_agent_openapi_async.py::TestAgentOpenApiAsync::test_agent_openapi_async -s
27+
@servicePreparer()
28+
@recorded_by_proxy_async(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX)
29+
async def test_agent_openapi_async(self, **kwargs):
30+
31+
model = self.test_agents_params["model_deployment_name"]
32+
33+
async with (
34+
self.create_async_client(operation_group="agents", **kwargs) as project_client,
35+
project_client.get_openai_client() as openai_client,
36+
):
37+
# Load OpenAPI spec from assets folder
38+
weather_asset_file_path = os.path.abspath(
39+
os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/weather_openapi.json")
40+
)
41+
42+
assert os.path.exists(weather_asset_file_path), f"OpenAPI spec file not found at: {weather_asset_file_path}"
43+
print(f"Using OpenAPI spec file: {weather_asset_file_path}")
44+
45+
with open(weather_asset_file_path, "r") as f:
46+
openapi_weather = jsonref.loads(f.read())
47+
48+
# Create OpenAPI tool
49+
tool = OpenApiAgentTool(
50+
openapi=OpenApiFunctionDefinition(
51+
name="get_weather",
52+
spec=openapi_weather,
53+
description="Retrieve weather information for a location.",
54+
auth=OpenApiAnonymousAuthDetails(),
55+
)
56+
)
57+
58+
agent_name = "openapi-agent"
59+
60+
# Create agent with OpenAPI tool
61+
agent = await project_client.agents.create_version(
62+
agent_name=agent_name,
63+
definition=PromptAgentDefinition(
64+
model=model,
65+
instructions="You are a helpful assistant.",
66+
tools=[tool],
67+
),
68+
description="Agent for testing OpenAPI tool capabilities.",
69+
)
70+
self._validate_agent_version(agent, expected_name=agent_name)
71+
72+
# Ask the agent to use the OpenAPI tool
73+
print("\nAsking agent to use OpenAPI tool to get weather...")
74+
75+
response = await openai_client.responses.create(
76+
input="Use the OpenAPI tool to print out, what is the weather in Seattle, WA today.",
77+
extra_body={"agent": {"name": agent.name, "type": "agent_reference"}},
78+
)
79+
self.validate_response(response)
80+
81+
# Get the response text
82+
response_text = response.output_text
83+
print(f"\nAgent's response: {response_text[:300]}...")
84+
85+
# Verify we got a meaningful response
86+
assert len(response_text) > 30, f"Expected a substantial response from the agent. Got '{response_text}'"
87+
88+
# The response should mention weather or Seattle (indicating the OpenAPI tool was used)
89+
response_lower = response_text.lower()
90+
assert any(
91+
keyword in response_lower for keyword in ["weather", "seattle", "temperature", "forecast"]
92+
), f"Expected response to contain weather information, but got: {response_text[:200]}"
93+
94+
print("\n✓ Agent successfully used OpenAPI tool to call external API")
95+
96+
# Teardown
97+
await project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version)
98+
print("Agent deleted")
99+
100+
# To run this test:
101+
# pytest tests/agents/tools/test_agent_openapi_async.py::TestAgentOpenApiAsync::test_agent_openapi_with_auth_async -s
102+
@servicePreparer()
103+
@pytest.mark.skip(reason="Add test here once we have a Foundry Project with a connection with auth credentials")
104+
@recorded_by_proxy_async(RecordedTransport.AZURE_CORE, RecordedTransport.HTTPX)
105+
async def test_agent_openapi_with_auth_async(self, **kwargs):
106+
pass

0 commit comments

Comments
 (0)