From 6c01edad94fb57f1933a0e98ea10f397c4807212 Mon Sep 17 00:00:00 2001 From: Paul Batum Date: Thu, 20 Nov 2025 17:46:02 -0800 Subject: [PATCH 1/8] Add comprehensive agent tools tests for azure-ai-projects This commit introduces extensive test coverage for agent tools functionality, validating various tool types and their combinations across different scenarios. New test coverage: - Individual tool tests: file search, code interpreter, function tools, AI search, web search, bing grounding, MCP, and image generation - Multi-tool integration tests: combinations of file search, code interpreter, and functions - Conversation-based tool tests: multi-turn interactions with various tools - Model verification tests: basic validation across different models - Async test support: parallel execution testing for AI search Test organization: - tests/agents/tools/test_agent_*.py: Individual tool validation - tests/agents/tools/multitool/*: Multi-tool integration scenarios - tests/agents/test_model_verification.py: Model compatibility checks Infrastructure updates: - Enhanced servicePreparer with connection IDs for bing, AI search, and MCP - Sample file improvements (agent naming, typo fixes) - Comprehensive README documentation for agent tools tests Note: One test (code interpreter file download) currently fails due to a known service limitation where the container file download API does not support token authentication. This will be resolved once the service adds support. --- .../tools/sample_agent_bing_grounding.py | 2 +- ...ample_agent_mcp_with_project_connection.py | 2 +- .../tests/agents/test_model_verification.py | 93 ++++ .../tests/agents/tools/README.md | 307 +++++++++++ .../tests/agents/tools/__init__.py | 4 + .../tests/agents/tools/multitool/__init__.py | 4 + ...est_agent_code_interpreter_and_function.py | 144 +++++ ..._agent_file_search_and_code_interpreter.py | 156 ++++++ .../test_agent_file_search_and_function.py | 519 ++++++++++++++++++ ...t_file_search_code_interpreter_function.py | 182 ++++++ .../test_multitool_with_conversations.py | 199 +++++++ .../agents/tools/test_agent_ai_search.py | 208 +++++++ .../tools/test_agent_ai_search_async.py | 233 ++++++++ .../agents/tools/test_agent_bing_grounding.py | 224 ++++++++ .../tools/test_agent_code_interpreter.py | 233 ++++++++ .../agents/tools/test_agent_file_search.py | 327 +++++++++++ .../tools/test_agent_file_search_stream.py | 137 +++++ .../agents/tools/test_agent_function_tool.py | 504 +++++++++++++++++ .../tools/test_agent_image_generation.py | 121 ++++ .../tests/agents/tools/test_agent_mcp.py | 303 ++++++++++ .../test_agent_tools_with_conversations.py | 394 +++++++++++++ .../agents/tools/test_agent_web_search.py | 99 ++++ sdk/ai/azure-ai-projects/tests/test_base.py | 4 + 23 files changed, 4397 insertions(+), 2 deletions(-) create mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_model_verification.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/README.md create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/__init__.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/multitool/__init__.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py create mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_bing_grounding.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_bing_grounding.py index c6110d4d3c79..28bb5d47a165 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_bing_grounding.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_bing_grounding.py @@ -64,7 +64,7 @@ # [END tool_declaration] agent = project_client.agents.create_version( - agent_name="MyAgent", + agent_name="bing-grounding-agent", definition=PromptAgentDefinition( model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], instructions="You are a helpful assistant.", diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_with_project_connection.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_with_project_connection.py index c5a62a3213c1..3534a00ae3eb 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_with_project_connection.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_with_project_connection.py @@ -22,7 +22,7 @@ 2) AZURE_AI_MODEL_DEPLOYMENT_NAME - The deployment name of the AI model, as found under the "Name" column in the "Models + endpoints" tab in your Microsoft Foundry project. 3) MCP_PROJECT_CONNECTION_ID - The connection resource ID in Custom keys - with key equals to "Authorization" and value to be "Bear ". + with key equals to "Authorization" and value to be "Bearer ". Token can be created in https://github.com/settings/personal-access-tokens/new """ diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_model_verification.py b/sdk/ai/azure-ai-projects/tests/agents/test_model_verification.py new file mode 100644 index 000000000000..1d22c6cbf491 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/test_model_verification.py @@ -0,0 +1,93 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +""" +Test to verify model substitution is working correctly. +Creates an agent and asks it to identify its model. +""" + +import uuid +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition + + +class TestModelVerification(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_model_identity(self, **kwargs): + """ + Simple test to verify which model is actually being used. + Creates an agent and asks it to identify its model. + + DOES NOT CLEAN UP - agents are left in place for verification. + """ + + # Get model from test_agents_params (which now reads from environment if available) + model = self.test_agents_params["model_deployment_name"] + print(f"\n{'='*80}") + print(f"๐Ÿ“‹ Model: {model}") + print(f"{'='*80}\n") + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create unique agent name with model and random ID to avoid conflicts + random_id = str(uuid.uuid4())[:8] + agent_name = f"model-verify-{model}-{random_id}" + + # Create agent + agent = project_client.agents.create_version( + agent_name=agent_name, + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant. When asked what model you are, respond with your exact model name/identifier.", + ), + description=f"Model verification test for {model}", + ) + + print(f"โœ… Agent created:") + print(f" - ID: {agent.id}") + print(f" - Name: {agent.name}") + print(f" - Version: {agent.version}") + print(f" - Model parameter passed: {model}") + + assert agent.id is not None + assert agent.name == agent_name + + # Ask the agent what model it is + print(f"\nโ“ Asking agent: What model are you?") + + response = openai_client.responses.create( + input="What model are you? Please tell me your exact model name or identifier.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"โœ… Response completed (id: {response.id})") + + response_text = response.output_text + print(f"\n๐Ÿค– Agent's response:") + print(f"{'='*80}") + print(response_text) + print(f"{'='*80}") + + # Basic assertions + assert response.id is not None + assert len(response_text) > 0, "Expected a response from the agent" + + # Print summary + print(f"\n๐Ÿ“Š SUMMARY:") + print(f" - Expected model: {model}") + print(f" - Agent name: {agent.name}") + print(f" - Agent response: {response_text[:100]}...") + + # NOTE: NOT cleaning up - agent stays for manual verification + print(f"\nโš ๏ธ Agent NOT deleted (left for verification)") + print(f" Agent: {agent.name}:{agent.version}") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/README.md b/sdk/ai/azure-ai-projects/tests/agents/tools/README.md new file mode 100644 index 000000000000..3a5bc8c62066 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/README.md @@ -0,0 +1,307 @@ +# Azure AI Agent Tools Tests + +This directory contains comprehensive tests for Azure AI Agents with various tool capabilities. These tests demonstrate how agents can be enhanced with different tools to perform specialized tasks like searching documents, executing code, calling custom functions, and more. + +## ๐Ÿ“ Directory Structure + +``` +tests/agents/tools/ +โ”œโ”€โ”€ README.md # This file +โ”œโ”€โ”€ test_agent_*.py # Single-tool tests +โ”œโ”€โ”€ test_agent_tools_with_conversations.py # Single tools + conversations +โ””โ”€โ”€ multitool/ # Multi-tool combinations + โ”œโ”€โ”€ test_agent_*_and_*.py # Dual-tool tests + โ”œโ”€โ”€ test_agent_*_*_*.py # Three-tool tests + โ””โ”€โ”€ test_multitool_with_conversations.py # Multi-tools + conversations +``` + +## ๐Ÿ”ง Tool Types & Architecture + +### Server-Side Tools (Automatic Execution) +These tools are executed entirely on the server. No client-side dispatch loop required. + +| Tool | Test File | What It Does | +|------|-----------|--------------| +| **FileSearchTool** | `test_agent_file_search.py` | Searches uploaded documents using vector stores | +| **CodeInterpreterTool** | `test_agent_code_interpreter.py` | Executes Python code in sandboxed environment | +| **AzureAISearchAgentTool** | `test_agent_ai_search.py` | Queries Azure AI Search indexes | +| **BingGroundingTool** | `test_agent_bing_grounding.py` | Searches the web using Bing API | +| **WebSearchTool** | `test_agent_web_search.py` | Performs web searches | +| **MCPTool** | `test_agent_mcp.py` | Model Context Protocol integrations | + +**Usage Pattern:** +```python +# Server-side tools - just create and go! +response = openai_client.responses.create( + input="Search for information about...", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}} +) +# Server handles everything - no loop needed +print(response.output_text) +``` + +### Client-Side Tools (Manual Dispatch Loop) +These tools require client-side execution logic. You must implement a dispatch loop. + +| Tool | Test File | What It Does | +|------|-----------|--------------| +| **FunctionTool** | `test_agent_function_tool.py` | Calls custom Python functions defined in your code | + +**Usage Pattern:** +```python +# Client-side tools - requires dispatch loop +response = openai_client.responses.create( + input="Calculate 15 plus 27", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}} +) + +# Check for function calls +for item in response.output: + if item.type == "function_call": + # Execute function locally + result = my_function(json.loads(item.arguments)) + + # Send result back + response = openai_client.responses.create( + input=[FunctionCallOutput( + call_id=item.call_id, + output=json.dumps(result) + )], + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}} + ) +``` + +## ๐Ÿ“ Test Categories + +### 1. Single-Tool Tests +Each test focuses on one tool type: + +- **`test_agent_file_search.py`** - Vector store document search + - Upload files to vector store + - Search across multiple documents + - Citation handling + - Stream vs non-stream responses + +- **`test_agent_code_interpreter.py`** - Python code execution + - Execute Python calculations + - Generate data files (CSV, images) + - File upload and download + - Error handling + +- **`test_agent_function_tool.py`** - Custom function calling + - Define function schemas + - Client-side execution loop + - Multi-turn function calls + - JSON parameter handling + +- **`test_agent_ai_search.py`** - Azure AI Search integration + - Connect to existing indexes + - Query with citations + - Multiple index support + +- **`test_agent_bing_grounding.py`** - Bing web search + - Real-time web queries + - URL citations + - Grounding with web sources + +- **`test_agent_mcp.py`** - Model Context Protocol + - GitHub integration + - Custom MCP servers + - Tool discovery + +- **`test_agent_web_search.py`** - Web search capabilities +- **`test_agent_image_generation.py`** - DALL-E image generation + +### 2. Multi-Tool Tests (`multitool/`) +Tests combining multiple tools in a single agent: + +- **`test_agent_file_search_and_function.py`** + - Search documents, then save results via function + - 4 comprehensive tests demonstrating different workflows + +- **`test_agent_code_interpreter_and_function.py`** + - Generate data with code, save via function + - Calculate and persist results + +- **`test_agent_file_search_and_code_interpreter.py`** + - Search docs, analyze with Python code + - Data extraction and processing + +- **`test_agent_file_search_code_interpreter_function.py`** + - All three tools working together + - Complete analysis workflows + +### 3. Conversation State Management +Tests demonstrating multi-turn interactions with state preservation: + +#### Single-Tool Conversations (`test_agent_tools_with_conversations.py`) +- **`test_function_tool_with_conversation`** + - Multiple function calls in one conversation + - Context preservation (agent remembers previous results) + - Conversation state verification + +- **`test_file_search_with_conversation`** + - Multiple searches in one conversation + - Follow-up questions with context + +- **`test_code_interpreter_with_conversation`** + - Sequential code executions + - Variable/state management across turns + +#### Multi-Tool Conversations (`multitool/test_multitool_with_conversations.py`) +- **`test_file_search_and_function_with_conversation`** + - Mix server-side (FileSearch) and client-side (Function) tools + - Complex workflow: Search โ†’ Follow-up โ†’ Save report + - Verifies both tool types tracked in conversation + +## ๐Ÿ”„ Conversation Patterns + +Tests demonstrate three patterns for multi-turn interactions: + +### Pattern 1: Manual History Management +```python +history = [{"role": "user", "content": "first message"}] +response = client.responses.create(input=history) +history += [{"role": el.role, "content": el.content} for el in response.output] +history.append({"role": "user", "content": "follow-up"}) +response = client.responses.create(input=history) +``` + +### Pattern 2: Using `previous_response_id` +```python +response_1 = client.responses.create(input="first message") +response_2 = client.responses.create( + input="follow-up", + previous_response_id=response_1.id +) +``` + +### Pattern 3: Using Conversations API (Recommended) +```python +conversation = client.conversations.create() +response_1 = client.responses.create( + input="first message", + conversation=conversation.id +) +response_2 = client.responses.create( + input="follow-up", + conversation=conversation.id +) +``` + +โš ๏ธ **Important:** When using `conversation` parameter, do NOT use `previous_response_id` - they are mutually exclusive. + +### Conversation State Verification +Tests verify conversation state by reading back all items: +```python +all_items = list(client.conversations.items.list(conversation.id)) +# Verify all user messages, assistant messages, tool calls preserved +``` + +## โš™๏ธ Environment Setup + +### Required Environment Variables + +Tool tests require additional environment variables beyond the basic SDK tests: + +```bash +# Core settings (already in base .env) +AZURE_AI_PROJECTS_TESTS_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com/api/projects/your-project +AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-4o + +# Tool-specific connections (needed for tool tests) +AZURE_AI_PROJECTS_TESTS_BING_CONNECTION_ID=/subscriptions/.../connections/your-bing-connection +AZURE_AI_PROJECTS_TESTS_AI_SEARCH_CONNECTION_ID=/subscriptions/.../connections/your-search-connection +AZURE_AI_PROJECTS_TESTS_AI_SEARCH_INDEX_NAME=your-index-name +AZURE_AI_PROJECTS_TESTS_MCP_PROJECT_CONNECTION_ID=/subscriptions/.../connections/your-mcp-connection +``` + +### ๐Ÿš€ Quick Setup with Auto-Discovery + +Use the `generate_env_file.py` script to automatically discover your project's configuration: + +```bash +# Navigate to the project directory +cd sdk/ai/azure-ai-projects + +# Run the generator with your project endpoint +uv run python scripts/generate_env_file.py \ + "https://your-project.services.ai.azure.com/api/projects/your-project" \ + myproject + +# This creates: .env.generated.myproject +``` + +The script will automatically discover: +- โœ… Available model deployments +- โœ… Bing connection (if configured) +- โœ… AI Search connection (if configured) +- โœ… GitHub/MCP connections (if configured) + +**Then copy the discovered values to your main `.env` file:** + +```bash +# Review the generated file +cat .env.generated.myproject + +# Copy relevant values to your .env +# You may need to manually add the AI Search index name +``` + +### Manual Setup + +If you prefer manual setup or need specific connections: + +1. **Go to Azure AI Foundry**: https://ai.azure.com +2. **Open your project** +3. **Navigate to "Connections" tab** +4. **Copy connection IDs** (full resource paths starting with `/subscriptions/...`) +5. **Add to your `.env` file** + +For AI Search index name: +1. Go to your AI Search service +2. Navigate to "Indexes" tab +3. Copy the index name + +## ๐Ÿงช Running Tests + +### Run All Tool Tests +```bash +pytest tests/agents/tools/ -v +``` + +### Run Single-Tool Tests Only +```bash +pytest tests/agents/tools/test_agent_*.py -v +``` + +### Run Multi-Tool Tests Only +```bash +pytest tests/agents/tools/multitool/ -v +``` + +### Run Conversation Tests +```bash +# Single-tool conversations +pytest tests/agents/tools/test_agent_tools_with_conversations.py -v + +# Multi-tool conversations +pytest tests/agents/tools/multitool/test_multitool_with_conversations.py -v +``` + +### Run Specific Tool Tests +```bash +# Function tool +pytest tests/agents/tools/test_agent_function_tool.py -v + +# File search +pytest tests/agents/tools/test_agent_file_search.py -v + +# Code interpreter +pytest tests/agents/tools/test_agent_code_interpreter.py -v +``` + +### Run Single Test +```bash +pytest tests/agents/tools/test_agent_function_tool.py::TestAgentFunctionTool::test_agent_function_tool -v -s +``` \ No newline at end of file diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/__init__.py b/sdk/ai/azure-ai-projects/tests/agents/tools/__init__.py new file mode 100644 index 000000000000..b74cfa3b899c --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/__init__.py @@ -0,0 +1,4 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/__init__.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/__init__.py new file mode 100644 index 000000000000..b74cfa3b899c --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/__init__.py @@ -0,0 +1,4 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py new file mode 100644 index 000000000000..d9d5d6dc5576 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py @@ -0,0 +1,144 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Multi-Tool Tests: Code Interpreter + Function Tool + +Tests various scenarios using an agent with Code Interpreter and Function Tool. +All tests use the same tool combination but different inputs and workflows. +""" + +import os +import json +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, CodeInterpreterTool, CodeInterpreterToolAuto, FunctionTool +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestAgentCodeInterpreterAndFunction(TestBase): + """Tests for agents using Code Interpreter + Function Tool combination.""" + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_calculate_and_save(self, **kwargs): + """ + Test calculation with Code Interpreter and saving with Function Tool. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Define function tool + func_tool = FunctionTool( + name="save_result", + description="Save analysis result", + parameters={ + "type": "object", + "properties": { + "result": {"type": "string", "description": "The result"}, + }, + "required": ["result"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="code-func-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Run calculations and save results.", + tools=[ + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + func_tool, + ], + ), + description="Agent with Code Interpreter and Function Tool.", + ) + print(f"Agent created (id: {agent.id})") + + # Use the agent + response = openai_client.responses.create( + input="Calculate 5 + 3 and save the result.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response received (id: {response.id})") + + assert response.id is not None + print("โœ“ Code Interpreter + Function Tool works!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_generate_data_and_report(self, **kwargs): + """ + Test generating data with Code Interpreter and reporting with Function. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Define function tool + report_function = FunctionTool( + name="generate_report", + description="Generate a report with the provided data", + parameters={ + "type": "object", + "properties": { + "title": {"type": "string", "description": "Report title"}, + "summary": {"type": "string", "description": "Report summary"}, + }, + "required": ["title", "summary"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="code-func-report-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Generate data using code and create reports with the generate_report function.", + tools=[ + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + report_function, + ], + ), + description="Agent for data generation and reporting.", + ) + print(f"Agent created (id: {agent.id})") + + # Request data generation and report + response = openai_client.responses.create( + input="Generate a list of 10 random numbers between 1 and 100, calculate their average, and create a report.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response received (id: {response.id})") + assert response.id is not None + print("โœ“ Data generation and reporting works!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py new file mode 100644 index 000000000000..594ad985d9b7 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py @@ -0,0 +1,156 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Multi-Tool Tests: File Search + Code Interpreter + +Tests various scenarios using an agent with File Search and Code Interpreter. +All tests use the same tool combination but different inputs and workflows. +""" + +import os +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool, CodeInterpreterTool, CodeInterpreterToolAuto + + +class TestAgentFileSearchAndCodeInterpreter(TestBase): + """Tests for agents using File Search + Code Interpreter combination.""" + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_find_and_analyze_data(self, **kwargs): + """ + Test finding data with File Search and analyzing with Code Interpreter. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create data file + txt_content = "Sample data: 10, 20, 30, 40, 50" + vector_store = openai_client.vector_stores.create(name="DataStore") + + from io import BytesIO + txt_file = BytesIO(txt_content.encode('utf-8')) + txt_file.name = "data.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=txt_file, + ) + print(f"File uploaded (id: {file.id})") + + # Create agent + agent = project_client.agents.create_version( + agent_name="file-search-code-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Find data and analyze it.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + ], + ), + description="Agent with File Search and Code Interpreter.", + ) + print(f"Agent created (id: {agent.id})") + + # Use the agent + response = openai_client.responses.create( + input="Find the data file and calculate the average.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response received (id: {response.id})") + + assert response.id is not None + assert len(response.output_text) > 20 + print("โœ“ File Search + Code Interpreter works!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_analyze_code_file(self, **kwargs): + """ + Test finding code file and analyzing it. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create Python code file + python_code = """def fibonacci(n): + if n <= 1: + return n + return fibonacci(n-1) + fibonacci(n-2) + +result = fibonacci(10) +print(f"Fibonacci(10) = {result}") +""" + + vector_store = openai_client.vector_stores.create(name="CodeAnalysisStore") + + from io import BytesIO + code_file = BytesIO(python_code.encode('utf-8')) + code_file.name = "fibonacci.py" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=code_file, + ) + print(f"Code file uploaded (id: {file.id})") + + # Create agent + agent = project_client.agents.create_version( + agent_name="file-search-code-analysis-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Find code files and analyze them. You can run code to test it.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + ], + ), + description="Agent for code analysis.", + ) + print(f"Agent created (id: {agent.id})") + + # Request analysis + response = openai_client.responses.create( + input="Find the fibonacci code and explain what it does. What is the computational complexity?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_text = response.output_text + print(f"Response: {response_text[:300]}...") + + assert len(response_text) > 50 + response_lower = response_lower = response_text.lower() + assert any(keyword in response_lower for keyword in ["fibonacci", "recursive", "complexity", "exponential"]), \ + "Expected analysis of fibonacci algorithm" + + print("โœ“ Code file analysis completed") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py new file mode 100644 index 000000000000..3911289b21b0 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py @@ -0,0 +1,519 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Multi-Tool Tests: File Search + Function Tool + +Tests various scenarios using an agent with File Search and Function Tool. +All tests use the same tool combination but different inputs and workflows. +""" + +import os +import json +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool, FunctionTool +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestAgentFileSearchAndFunction(TestBase): + """Tests for agents using File Search + Function Tool combination.""" + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_data_analysis_workflow(self, **kwargs): + """ + Test data analysis workflow: upload data, search, save results. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create sample data file + txt_content = """Sales Data Q1-Q3: + +Product: Widget A +Q1: $15,000 +Q2: $18,000 +Q3: $21,000 +Total: $54,000 + +Product: Widget B +Q1: $22,000 +Q2: $25,000 +Q3: $28,000 +Total: $75,000 + +Overall Total Revenue: $129,000 +""" + + # Create vector store and upload + vector_store = openai_client.vector_stores.create(name="SalesDataStore") + print(f"Vector store created (id: {vector_store.id})") + + from io import BytesIO + txt_file = BytesIO(txt_content.encode('utf-8')) + txt_file.name = "sales_data.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=txt_file, + ) + print(f"File uploaded (id: {file.id})") + assert file.status == "completed" + + # Define function tool + save_results_function = FunctionTool( + name="save_analysis_results", + description="Save the analysis results", + parameters={ + "type": "object", + "properties": { + "summary": { + "type": "string", + "description": "Summary of the analysis", + }, + }, + "required": ["summary"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="file-search-function-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a data analyst. Use file search to find data and save_analysis_results function to save your findings.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + save_results_function, + ], + ), + description="Agent with File Search and Function Tool.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + + # Request analysis + print("\nAsking agent to analyze the sales data...") + + response = openai_client.responses.create( + input="Analyze the sales data and calculate the total revenue for each product. Then save the results.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + + # Check if function was called + function_calls_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "function_call": + function_calls_found += 1 + print(f"Function call detected (id: {item.call_id}, name: {item.name})") + + assert item.name == "save_analysis_results" + + arguments = json.loads(item.arguments) + print(f"Function arguments: {arguments}") + + assert "summary" in arguments + assert len(arguments["summary"]) > 20 + + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"status": "success", "saved": True}), + ) + ) + + assert function_calls_found > 0, "Expected save_analysis_results function to be called" + + # Send function results back + if input_list: + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Final response: {response.output_text[:200]}...") + + print("\nโœ“ Workflow completed successfully") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_empty_vector_store_handling(self, **kwargs): + """ + Test how agent handles empty vector store (no files uploaded). + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create EMPTY vector store + vector_store = openai_client.vector_stores.create(name="EmptyStore") + print(f"Empty vector store created (id: {vector_store.id})") + + # Define function tool + error_function = FunctionTool( + name="report_error", + description="Report when data is not available", + parameters={ + "type": "object", + "properties": { + "error_message": { + "type": "string", + "description": "Description of the error", + }, + }, + "required": ["error_message"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="file-search-function-error-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Search for data. If you can't find it, use report_error function to explain what happened.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + error_function, + ], + ), + description="Agent for testing error handling.", + ) + print(f"Agent created (id: {agent.id})") + + # Request analysis of non-existent file + print("\nAsking agent to find non-existent data...") + + response = openai_client.responses.create( + input="Find and analyze the quarterly sales report.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_text = response.output_text + print(f"Response: '{response_text[:200] if response_text else '(empty)'}...'") + + # Verify agent didn't crash + assert response.id is not None, "Agent should return a valid response" + assert len(response.output) >= 0, "Agent should return output items" + + # If there's text, it should be meaningful + if response_text: + assert len(response_text) > 10, "Non-empty response should be meaningful" + + print("\nโœ“ Agent handled missing data gracefully") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_python_code_file_search(self, **kwargs): + """ + Test searching for Python code files. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create sample Python code + python_code = """# Sample Python code for analysis +def calculate_sum(numbers): + return sum(numbers) + +# Test data +data = [1, 2, 3, 4, 5] +result = calculate_sum(data) +print(f"Sum: {result}") +""" + + # Create vector store and upload + vector_store = openai_client.vector_stores.create(name="CodeStore") + + from io import BytesIO + code_file = BytesIO(python_code.encode('utf-8')) + code_file.name = "sample_code.py" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=code_file, + ) + print(f"Python file uploaded (id: {file.id})") + + # Define function tool + save_function = FunctionTool( + name="save_code_review", + description="Save code review findings", + parameters={ + "type": "object", + "properties": { + "findings": { + "type": "string", + "description": "Code review findings", + }, + }, + "required": ["findings"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="file-search-function-code-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You can search for code files and describe what they do. Save your findings.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + save_function, + ], + ), + description="Agent for testing file search with code.", + ) + print(f"Agent created (id: {agent.id})") + + # Request code analysis + print("\nAsking agent to find and analyze the Python code...") + + response = openai_client.responses.create( + input="Find the Python code file and tell me what the calculate_sum function does.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_text = response.output_text + print(f"Response: {response_text[:300]}...") + + # Verify agent found and analyzed the code + assert len(response_text) > 50, "Expected detailed analysis" + + response_lower = response_text.lower() + assert any(keyword in response_lower for keyword in ["sum", "calculate", "function", "numbers", "code", "python"]), \ + "Expected response to discuss the code" + + print("\nโœ“ Agent successfully found code file using File Search") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_multi_turn_search_and_save_workflow(self, **kwargs): + """ + Test multi-turn workflow: search documents, ask follow-ups, save findings. + + This tests: + - File Search across multiple turns + - Function calls interspersed with searches + - Context retention across searches and function calls + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create research documents + doc1_content = """Research Paper: Machine Learning in Healthcare + +Abstract: +Machine learning algorithms are revolutionizing healthcare diagnostics. +Recent studies show 95% accuracy in disease detection using neural networks. +Key applications include medical imaging, patient risk prediction, and drug discovery. + +Conclusion: +ML techniques offer promising solutions for early disease detection and personalized treatment. +""" + + doc2_content = """Research Paper: AI Ethics and Governance + +Abstract: +As AI systems become more prevalent, ethical considerations are paramount. +Issues include bias in algorithms, data privacy, and accountability. +Regulatory frameworks are being developed globally to address these concerns. + +Conclusion: +Responsible AI development requires multistakeholder collaboration and transparent governance. +""" + + # Create vector store and upload documents + vector_store = openai_client.vector_stores.create(name="ResearchStore") + print(f"Vector store created: {vector_store.id}") + + from io import BytesIO + file1 = BytesIO(doc1_content.encode('utf-8')) + file1.name = "ml_healthcare.txt" + file2 = BytesIO(doc2_content.encode('utf-8')) + file2.name = "ai_ethics.txt" + + uploaded1 = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=file1, + ) + uploaded2 = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=file2, + ) + print(f"Documents uploaded: {uploaded1.id}, {uploaded2.id}") + + # Define save function + save_finding = FunctionTool( + name="save_finding", + description="Save research finding", + parameters={ + "type": "object", + "properties": { + "topic": {"type": "string", "description": "Research topic"}, + "finding": {"type": "string", "description": "Key finding"}, + }, + "required": ["topic", "finding"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="research-assistant-multi-turn", + definition=PromptAgentDefinition( + model=model, + instructions="You are a research assistant. Search documents and save important findings.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + save_finding, + ], + ), + description="Research assistant for multi-turn testing.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Search for ML in healthcare + print("\n--- Turn 1: Initial search query ---") + response_1 = openai_client.responses.create( + input="What does the research say about machine learning in healthcare?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "95" in response_1_text or "accuracy" in response_1_text.lower() + + # Turn 2: Follow-up for specifics + print("\n--- Turn 2: Follow-up for details ---") + response_2 = openai_client.responses.create( + input="What specific applications are mentioned?", + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + response_2_lower = response_2_text.lower() + assert any(keyword in response_2_lower for keyword in ["imaging", "drug", "risk", "prediction"]) + + # Turn 3: Save the finding + print("\n--- Turn 3: Save finding ---") + response_3 = openai_client.responses.create( + input="Please save that finding about ML accuracy.", + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + function_called = False + for item in response_3.output: + if item.type == "function_call": + function_called = True + print(f"Function called: {item.name} with args: {item.arguments}") + assert item.name == "save_finding" + + args = json.loads(item.arguments) + assert "topic" in args and "finding" in args + print(f" Topic: {args['topic']}") + print(f" Finding: {args['finding'][:100]}...") + + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"status": "saved", "id": "finding_001"}), + ) + ) + + assert function_called, "Expected save_finding to be called" + + # Send function result + response_3 = openai_client.responses.create( + input=input_list, + previous_response_id=response_3.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response 3: {response_3.output_text[:150]}...") + + # Turn 4: Switch to different topic (AI ethics) + print("\n--- Turn 4: New search topic ---") + response_4 = openai_client.responses.create( + input="Now tell me about AI ethics concerns mentioned in the research.", + previous_response_id=response_3.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_4_text = response_4.output_text + print(f"Response 4: {response_4_text[:200]}...") + response_4_lower = response_4_text.lower() + assert any(keyword in response_4_lower for keyword in ["bias", "privacy", "ethics", "accountability"]) + + print("\nโœ“ Multi-turn File Search + Function workflow successful!") + print(" - Multiple searches across different documents") + print(" - Function called after context-building searches") + print(" - Topic switching works correctly") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py new file mode 100644 index 000000000000..1648b121b34a --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py @@ -0,0 +1,182 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Multi-Tool Tests: File Search + Code Interpreter + Function Tool + +Tests various scenarios using an agent with all three tools together. +All tests use the same 3-tool combination but different inputs and workflows. +""" + +import os +import json +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool, CodeInterpreterTool, CodeInterpreterToolAuto, FunctionTool +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestAgentFileSearchCodeInterpreterFunction(TestBase): + """Tests for agents using File Search + Code Interpreter + Function Tool.""" + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_complete_analysis_workflow(self, **kwargs): + """ + Test complete workflow: find data, analyze it, save results. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create data file + txt_content = "Sample data for analysis" + vector_store = openai_client.vector_stores.create(name="ThreeToolStore") + + from io import BytesIO + txt_file = BytesIO(txt_content.encode('utf-8')) + txt_file.name = "data.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=txt_file, + ) + print(f"File uploaded (id: {file.id})") + + # Define function tool + func_tool = FunctionTool( + name="save_result", + description="Save analysis result", + parameters={ + "type": "object", + "properties": { + "result": {"type": "string", "description": "The result"}, + }, + "required": ["result"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent with all three tools + agent = project_client.agents.create_version( + agent_name="three-tool-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Use file search to find data, code interpreter to analyze it, and save_result to save findings.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + func_tool, + ], + ), + description="Agent using File Search, Code Interpreter, and Function Tool.", + ) + print(f"Agent created (id: {agent.id})") + + # Use the agent + response = openai_client.responses.create( + input="Find the data file, analyze it, and save the results.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response received (id: {response.id})") + + assert response.id is not None + print("โœ“ Three-tool combination works!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_four_tools_combination(self, **kwargs): + """ + Test with 4 tools: File Search + Code Interpreter + 2 Functions. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create vector store + txt_content = "Test data" + vector_store = openai_client.vector_stores.create(name="FourToolStore") + + from io import BytesIO + txt_file = BytesIO(txt_content.encode('utf-8')) + txt_file.name = "data.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=txt_file, + ) + + # Define two function tools + func_tool_1 = FunctionTool( + name="save_result", + description="Save result", + parameters={ + "type": "object", + "properties": { + "result": {"type": "string", "description": "The result"}, + }, + "required": ["result"], + "additionalProperties": False, + }, + strict=True, + ) + + func_tool_2 = FunctionTool( + name="log_action", + description="Log an action", + parameters={ + "type": "object", + "properties": { + "action": {"type": "string", "description": "Action taken"}, + }, + "required": ["action"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent with 4 tools + agent = project_client.agents.create_version( + agent_name="four-tool-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Use all available tools.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + func_tool_1, + func_tool_2, + ], + ), + description="Agent with 4 tools.", + ) + print(f"Agent with 4 tools created (id: {agent.id})") + + assert agent.id is not None + print("โœ“ 4 tools works!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py new file mode 100644 index 000000000000..d4d1fc68faa9 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py @@ -0,0 +1,199 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +Test agents using multiple tools within conversations. + +This test file demonstrates how to use multiple agent tools (both server-side and client-side) +within the context of conversations, testing conversation state management with multi-tool interactions. +""" + +import json +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + FunctionTool, + FileSearchTool, + PromptAgentDefinition, +) +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestMultiToolWithConversations(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_file_search_and_function_with_conversation(self, **kwargs): + """ + Test using multiple tools (FileSearch + Function) within one conversation. + + This tests: + - Mixing FileSearch (server-side) and Function (client-side) tools in same conversation + - Complex multi-turn workflow with different tool types + - Conversation managing state across different tool executions + - Verifying conversation state preserves all tool interactions + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create document + doc_content = """Sales Data Q1 2024 + +Product A: $45,000 +Product B: $67,000 +Product C: $32,000 + +Total Revenue: $144,000 +""" + + vector_store = openai_client.vector_stores.create(name="SalesDataStore") + from io import BytesIO + file = BytesIO(doc_content.encode('utf-8')) + file.name = "sales.txt" + openai_client.vector_stores.files.upload_and_poll(vector_store_id=vector_store.id, file=file) + print(f"Vector store created: {vector_store.id}") + + # Define save function + save_report = FunctionTool( + name="save_report", + description="Save a report summary. Use this when explicitly asked to perform a save operation.", + parameters={ + "type": "object", + "properties": { + "title": {"type": "string"}, + "summary": {"type": "string"}, + }, + "required": ["title", "summary"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent with both tools + agent = project_client.agents.create_version( + agent_name="mixed-tools-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are an analyst. Search data to answer questions and save reports when instructed", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + save_report, + ], + ), + description="Mixed tools agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Search (server-side tool) + print("\n--- Turn 1: File Search ---") + response_1 = openai_client.responses.create( + input="What was the total revenue in Q1 2024?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response 1: {response_1.output_text[:150]}...") + assert "144,000" in response_1.output_text or "144000" in response_1.output_text + + # Turn 2: Follow-up search + print("\n--- Turn 2: Follow-up search ---") + response_2 = openai_client.responses.create( + input="Which product had the highest sales?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response 2: {response_2.output_text[:150]}...") + assert "Product B" in response_2.output_text or "67,000" in response_2.output_text + + # Turn 3: Save report (client-side tool) + print("\n--- Turn 3: Save report using function ---") + response_3 = openai_client.responses.create( + input="Save a summary report of these Q1 results", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_3.output: + if item.type == "function_call": + print(f"Function called: {item.name}") + args = json.loads(item.arguments) + print(f" Title: {args['title']}") + print(f" Summary: {args['summary'][:100]}...") + + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"status": "saved", "report_id": "Q1_2024"}), + ) + ) + + response_3 = openai_client.responses.create( + input=input_list, + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response 3: {response_3.output_text[:150]}...") + + print("\nโœ“ Mixed tools with conversation successful!") + print(" - File search (server-side) worked") + print(" - Function call (client-side) worked") + print(" - Both tools used in same conversation") + + # Verify conversation state with multiple tool types + print("\n--- Verifying multi-tool conversation state ---") + all_items = list(openai_client.conversations.items.list(conversation.id)) + print(f"Total conversation items: {len(all_items)}") + + # Count different item types + user_messages = sum(1 for item in all_items if item.type == "message" and item.role == "user") + assistant_messages = sum(1 for item in all_items if item.type == "message" and item.role == "assistant") + function_calls = sum(1 for item in all_items if item.type == "function_call") + function_outputs = sum(1 for item in all_items if item.type == "function_call_output") + + print(f" User messages: {user_messages}") + print(f" Assistant messages: {assistant_messages}") + print(f" Function calls: {function_calls}") + print(f" Function outputs: {function_outputs}") + + # Print item sequence to show tool interleaving + print("\n Conversation item sequence:") + for i, item in enumerate(all_items, 1): + if item.type == "message": + content_preview = str(item.content[0] if item.content else "")[:50] + print(f" {i}. {item.type} ({item.role}): {content_preview}...") + else: + print(f" {i}. {item.type}") + + # Verify we have items from all three turns + assert user_messages >= 3, "Expected at least 3 user messages (three turns)" + assert assistant_messages >= 3, "Expected assistant responses for each turn" + assert function_calls >= 1, "Expected at least 1 function call (save_report)" + assert function_outputs >= 1, "Expected at least 1 function output" + + print("\nโœ“ Multi-tool conversation state verified") + print(" - Both server-side (FileSearch) and client-side (Function) tools tracked") + print(" - All 3 turns preserved in conversation") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py new file mode 100644 index 000000000000..c44a6ccb54cd --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py @@ -0,0 +1,208 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + PromptAgentDefinition, + AzureAISearchAgentTool, + AzureAISearchToolResource, + AISearchIndexResource, + AzureAISearchQueryType, +) + + +class TestAgentAISearch(TestBase): + + # Test questions with expected answers + TEST_QUESTIONS = [ + { + "question": "Agent Lightning's unified data interface and MDP formulation are designed to separate task-specific agent design from learning-based policy optimization.", + "answer": True, + }, + { + "question": "LightningRL optimizes multi-call agent trajectories mainly by masking out non-LLM tokens in long trajectories, without decomposing them into transitions.", + "answer": False, + }, + { + "question": "The Trainingโ€“Agent Disaggregation architecture uses an Agent Lightning Server (with an OpenAI-like API) and a Client so that agents can run their own tool/code logic without being co-located with the GPU training framework.", + "answer": True, + }, + { + "question": "In the text-to-SQL experiment, the authors used LangChain to build a 3-agent workflow on the Spider dataset, but only trained 2 of those agents (the SQL-writing and rewriting agents).", + "answer": True, + }, + { + "question": "The math QA task in the experiments was implemented with LangChain and used a SQL executor as its external tool.", + "answer": False, + }, + ] + + @servicePreparer() + @pytest.mark.skip(reason="Slow sequential sync test - covered by faster parallel async test") + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_ai_search_question_answering(self, **kwargs): + """ + Test agent with Azure AI Search capabilities for question answering. + + NOTE: This test is skipped in favor of the parallel async version which is + significantly faster (~3x) and provides the same coverage. + See test_agent_ai_search_async.py::test_agent_ai_search_question_answering_async_parallel + + This test verifies that an agent can be created with AzureAISearchAgentTool, + use it to search indexed content, and provide accurate answers to questions + based on the search results. + + The test asks 5 true/false questions and validates that at least 4 are + answered correctly by the agent using the search index. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with AI Search) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Get AI Search connection and index from environment + ai_search_connection_id = kwargs.get("azure_ai_projects_tests_ai_search_connection_id") + ai_search_index_name = kwargs.get("azure_ai_projects_tests_ai_search_index_name") + + if not ai_search_connection_id: + pytest.skip("AI_SEARCH_PROJECT_CONNECTION_ID environment variable not set") + + if not ai_search_index_name: + pytest.skip("AI_SEARCH_INDEX_NAME environment variable not set") + + assert isinstance(ai_search_connection_id, str), "ai_search_connection_id must be a string" + assert isinstance(ai_search_index_name, str), "ai_search_index_name must be a string" + + # Create agent with Azure AI Search tool + agent = project_client.agents.create_version( + agent_name="ai-search-qa-agent", + definition=PromptAgentDefinition( + model=model, + instructions="""You are a helpful assistant that answers true/false questions based on the provided search results. + Always use the Azure AI Search tool to find relevant information before answering. + Respond with only 'True' or 'False' based on what you find in the search results. + If you cannot find clear evidence in the search results, answer 'False'.""", + tools=[ + AzureAISearchAgentTool( + azure_ai_search=AzureAISearchToolResource( + indexes=[ + AISearchIndexResource( + project_connection_id=ai_search_connection_id, + index_name=ai_search_index_name, + query_type=AzureAISearchQueryType.SIMPLE, + ), + ] + ) + ) + ], + ), + description="Agent for testing AI Search question answering.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "ai-search-qa-agent" + assert agent.version is not None + + # Test each question + correct_answers = 0 + total_questions = len(self.TEST_QUESTIONS) + + for i, qa_pair in enumerate(self.TEST_QUESTIONS, 1): + question = qa_pair["question"] + expected_answer = qa_pair["answer"] + + print(f"\n{'='*80}") + print(f"Question {i}/{total_questions}:") + print(f"Q: {question}") + print(f"Expected: {expected_answer}") + + output_text = "" + + stream_response = openai_client.responses.create( + stream=True, + tool_choice="required", + input=f"Answer this question with only 'True' or 'False': {question}", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + for event in stream_response: + if event.type == "response.created": + print(f"Response created with ID: {event.response.id}") + elif event.type == "response.output_text.delta": + pass # Don't print deltas to reduce output + elif event.type == "response.completed": + output_text = event.response.output_text + print(f"Agent's answer: {output_text}") + + # Parse the answer from the output + # Look for "True" or "False" in the response + output_lower = output_text.lower() + agent_answer = None + + # Try to extract boolean answer + if "true" in output_lower and "false" not in output_lower: + agent_answer = True + elif "false" in output_lower and "true" not in output_lower: + agent_answer = False + elif output_lower.strip() in ["true", "false"]: + agent_answer = output_lower.strip() == "true" + else: + # Try to determine based on more complex responses + # Count occurrences + true_count = output_lower.count("true") + false_count = output_lower.count("false") + if true_count > false_count: + agent_answer = True + elif false_count > true_count: + agent_answer = False + + if agent_answer is not None: + is_correct = agent_answer == expected_answer + if is_correct: + correct_answers += 1 + print(f"โœ“ CORRECT (Agent: {agent_answer}, Expected: {expected_answer})") + else: + print(f"โœ— INCORRECT (Agent: {agent_answer}, Expected: {expected_answer})") + else: + print(f"โœ— UNABLE TO PARSE ANSWER from: {output_text}") + + # Print summary + print(f"\n{'='*80}") + print(f"SUMMARY: {correct_answers}/{total_questions} questions answered correctly") + print(f"{'='*80}") + + # Verify that at least 4 out of 5 questions were answered correctly + assert correct_answers >= 4, ( + f"Expected at least 4 correct answers out of {total_questions}, " + f"but got {correct_answers}. The agent needs to answer at least 80% correctly." + ) + + print(f"\nโœ“ Test passed! Agent answered {correct_answers}/{total_questions} questions correctly (>= 4 required)") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py new file mode 100644 index 000000000000..1f4f498d30fe --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py @@ -0,0 +1,233 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import asyncio +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + PromptAgentDefinition, + AzureAISearchAgentTool, + AzureAISearchToolResource, + AISearchIndexResource, + AzureAISearchQueryType, +) + + +class TestAgentAISearchAsync(TestBase): + + # Test questions with expected answers + TEST_QUESTIONS = [ + { + "title": "Unified Data Interface", + "question": "Agent Lightning's unified data interface and MDP formulation are designed to separate task-specific agent design from learning-based policy optimization.", + "answer": True, + }, + { + "title": "LightningRL Optimization", + "question": "LightningRL optimizes multi-call agent trajectories mainly by masking out non-LLM tokens in long trajectories, without decomposing them into transitions.", + "answer": False, + }, + { + "title": "Training-Agent Disaggregation", + "question": "The Trainingโ€“Agent Disaggregation architecture uses an Agent Lightning Server (with an OpenAI-like API) and a Client so that agents can run their own tool/code logic without being co-located with the GPU training framework.", + "answer": True, + }, + { + "title": "Text-to-SQL Experiment", + "question": "In the text-to-SQL experiment, the authors used LangChain to build a 3-agent workflow on the Spider dataset, but only trained 2 of those agents (the SQL-writing and rewriting agents).", + "answer": True, + }, + { + "title": "Math QA Implementation", + "question": "The math QA task in the experiments was implemented with LangChain and used a SQL executor as its external tool.", + "answer": False, + }, + ] + + async def _ask_question_async(self, openai_client, agent_name: str, title: str, question: str, expected_answer: bool, question_num: int, total_questions: int): + """Helper method to ask a single question asynchronously.""" + print(f"\n{'='*80}") + print(f"Q{question_num}/{total_questions}: {title}") + print(f"{question}") + print(f"Expected: {expected_answer}") + + output_text = "" + + stream_response = await openai_client.responses.create( + stream=True, + tool_choice="required", + input=f"Answer this question with only 'True' or 'False': {question}", + extra_body={"agent": {"name": agent_name, "type": "agent_reference"}}, + ) + + async for event in stream_response: + if event.type == "response.created": + print(f"Response ID: {event.response.id}") + elif event.type == "response.completed": + output_text = event.response.output_text + + # Parse the answer from the output + output_lower = output_text.lower() + agent_answer = None + + # Try to extract boolean answer + if "true" in output_lower and "false" not in output_lower: + agent_answer = True + elif "false" in output_lower and "true" not in output_lower: + agent_answer = False + elif output_lower.strip() in ["true", "false"]: + agent_answer = output_lower.strip() == "true" + else: + # Try to determine based on more complex responses + true_count = output_lower.count("true") + false_count = output_lower.count("false") + if true_count > false_count: + agent_answer = True + elif false_count > true_count: + agent_answer = False + + is_correct = False + if agent_answer is not None: + is_correct = agent_answer == expected_answer + if is_correct: + print(f"โœ“ Q{question_num} ({title}): CORRECT") + else: + print(f"โœ— Q{question_num} ({title}): INCORRECT (Agent: {agent_answer}, Expected: {expected_answer})") + else: + print(f"โœ— Q{question_num} ({title}): UNABLE TO PARSE ANSWER from: {output_text}") + + return is_correct + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + @pytest.mark.asyncio + async def test_agent_ai_search_question_answering_async_parallel(self, **kwargs): + """ + Test agent with Azure AI Search capabilities for question answering using async (parallel). + + This test verifies that an agent can be created with AzureAISearchAgentTool, + and handle multiple concurrent requests to search indexed content and provide + accurate answers to questions based on the search results. + + The test asks 5 true/false questions IN PARALLEL using asyncio.gather() and + validates that at least 4 are answered correctly by the agent using the search index. + + This should be significantly faster than the sequential version. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses (concurrent) openai_client.responses.create() (with AI Search, parallel) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_async_client(operation_group="agents", **kwargs) + + async with project_client: + openai_client = await project_client.get_openai_client() + + # Get AI Search connection and index from environment + ai_search_connection_id = kwargs.get("azure_ai_projects_tests_ai_search_connection_id") + ai_search_index_name = kwargs.get("azure_ai_projects_tests_ai_search_index_name") + + if not ai_search_connection_id: + pytest.skip("AI_SEARCH_PROJECT_CONNECTION_ID environment variable not set") + + if not ai_search_index_name: + pytest.skip("AI_SEARCH_INDEX_NAME environment variable not set") + + assert isinstance(ai_search_connection_id, str), "ai_search_connection_id must be a string" + assert isinstance(ai_search_index_name, str), "ai_search_index_name must be a string" + + # Create agent with Azure AI Search tool + agent = await project_client.agents.create_version( + agent_name="ai-search-qa-agent-async-parallel", + definition=PromptAgentDefinition( + model=model, + instructions="""You are a helpful assistant that answers true/false questions based on the provided search results. + Always use the Azure AI Search tool to find relevant information before answering. + Respond with only 'True' or 'False' based on what you find in the search results. + If you cannot find clear evidence in the search results, answer 'False'.""", + tools=[ + AzureAISearchAgentTool( + azure_ai_search=AzureAISearchToolResource( + indexes=[ + AISearchIndexResource( + project_connection_id=ai_search_connection_id, + index_name=ai_search_index_name, + query_type=AzureAISearchQueryType.SIMPLE, + ), + ] + ) + ) + ], + ), + description="Agent for testing AI Search question answering (async parallel).", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "ai-search-qa-agent-async-parallel" + assert agent.version is not None + + # Test all questions IN PARALLEL using asyncio.gather() + total_questions = len(self.TEST_QUESTIONS) + print(f"\nRunning {total_questions} questions in parallel...") + + # Create tasks for all questions + tasks = [] + for i, qa_pair in enumerate(self.TEST_QUESTIONS, 1): + title = qa_pair["title"] + question = qa_pair["question"] + expected_answer = qa_pair["answer"] + + task = self._ask_question_async( + openai_client, + agent.name, + title, + question, + expected_answer, + i, + total_questions + ) + tasks.append(task) + + # Run all tasks in parallel and collect results + results = await asyncio.gather(*tasks) + + # Count correct answers + correct_answers = sum(1 for is_correct in results if is_correct) + + # Print summary + print(f"\n{'='*80}") + print(f"SUMMARY: {correct_answers}/{total_questions} questions answered correctly") + print(f"{'='*80}") + + # Verify that at least 4 out of 5 questions were answered correctly + assert correct_answers >= 4, ( + f"Expected at least 4 correct answers out of {total_questions}, " + f"but got {correct_answers}. The agent needs to answer at least 80% correctly." + ) + + print(f"\nโœ“ Test passed! Agent answered {correct_answers}/{total_questions} questions correctly (>= 4 required)") + + # Teardown + await project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py new file mode 100644 index 000000000000..109a0aac716a --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py @@ -0,0 +1,224 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + PromptAgentDefinition, + BingGroundingAgentTool, + BingGroundingSearchToolParameters, + BingGroundingSearchConfiguration, +) + + +class TestAgentBingGrounding(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_bing_grounding(self, **kwargs): + """ + Test agent with Bing grounding capabilities. + + This test verifies that an agent can be created with BingGroundingAgentTool, + use it to search the web for current information, and provide responses with + URL citations. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with Bing grounding) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Note: This test requires BING_PROJECT_CONNECTION_ID environment variable + # to be set with a valid Bing connection ID from the project + bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_connection_id") + + if not bing_connection_id: + pytest.skip("BING_PROJECT_CONNECTION_ID environment variable not set") + + assert isinstance(bing_connection_id, str), "bing_connection_id must be a string" + + # Create agent with Bing grounding tool + agent = project_client.agents.create_version( + agent_name="bing-grounding-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant.", + tools=[ + BingGroundingAgentTool( + bing_grounding=BingGroundingSearchToolParameters( + search_configurations=[ + BingGroundingSearchConfiguration( + project_connection_id=bing_connection_id + ) + ] + ) + ) + ], + ), + description="You are a helpful agent.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "bing-grounding-agent" + assert agent.version is not None + + # Test agent with a query that requires current web information + output_text = "" + url_citations = [] + + stream_response = openai_client.responses.create( + stream=True, + tool_choice="required", + input="What is the current weather in Seattle?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + for event in stream_response: + if event.type == "response.created": + print(f"Follow-up response created with ID: {event.response.id}") + assert event.response.id is not None + elif event.type == "response.output_text.delta": + print(f"Delta: {event.delta}") + elif event.type == "response.text.done": + print(f"Follow-up response done!") + elif event.type == "response.output_item.done": + if event.item.type == "message": + item = event.item + if item.content and len(item.content) > 0: + if item.content[-1].type == "output_text": + text_content = item.content[-1] + for annotation in text_content.annotations: + if annotation.type == "url_citation": + print(f"URL Citation: {annotation.url}") + url_citations.append(annotation.url) + elif event.type == "response.completed": + print(f"Follow-up completed!") + print(f"Full response: {event.response.output_text}") + output_text = event.response.output_text + + # Verify that we got a response + assert len(output_text) > 0, "Expected non-empty response text" + + # Verify that we got URL citations (Bing grounding should provide sources) + assert len(url_citations) > 0, "Expected URL citations from Bing grounding" + + # Verify that citations are valid URLs + for url in url_citations: + assert url.startswith("http://") or url.startswith("https://"), f"Invalid URL citation: {url}" + + print(f"Test completed successfully with {len(url_citations)} URL citations") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_bing_grounding_multiple_queries(self, **kwargs): + """ + Test agent with Bing grounding for multiple queries. + + This test verifies that an agent can handle multiple queries using + Bing grounding and provide accurate responses with citations. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_connection_id") + + if not bing_connection_id: + pytest.skip("BING_PROJECT_CONNECTION_ID environment variable not set") + + assert isinstance(bing_connection_id, str), "bing_connection_id must be a string" + + # Create agent with Bing grounding tool + agent = project_client.agents.create_version( + agent_name="bing-grounding-multi-query-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that provides current information.", + tools=[ + BingGroundingAgentTool( + bing_grounding=BingGroundingSearchToolParameters( + search_configurations=[ + BingGroundingSearchConfiguration( + project_connection_id=bing_connection_id + ) + ] + ) + ) + ], + ), + description="Agent for testing multiple Bing grounding queries.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + + # Test with multiple different queries + queries = [ + "What is today's date?", + "What are the latest news about AI?", + ] + + for query in queries: + print(f"\nTesting query: {query}") + output_text = "" + url_citations = [] + + stream_response = openai_client.responses.create( + stream=True, + tool_choice="required", + input=query, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + for event in stream_response: + if event.type == "response.output_item.done": + if event.item.type == "message": + item = event.item + if item.content and len(item.content) > 0: + if item.content[-1].type == "output_text": + text_content = item.content[-1] + for annotation in text_content.annotations: + if annotation.type == "url_citation": + url_citations.append(annotation.url) + elif event.type == "response.completed": + output_text = event.response.output_text + + # Verify that we got a response for each query + assert len(output_text) > 0, f"Expected non-empty response text for query: {query}" + print(f"Response length: {len(output_text)} characters") + print(f"URL citations found: {len(url_citations)}") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py new file mode 100644 index 000000000000..17b864dc3820 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py @@ -0,0 +1,233 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import os +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + PromptAgentDefinition, + CodeInterpreterTool, + CodeInterpreterToolAuto, +) + + +class TestAgentCodeInterpreter(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_code_interpreter_simple_math(self, **kwargs): + """ + Test agent with Code Interpreter for simple Python code execution. + + This test verifies that an agent can execute simple Python code + without any file uploads or downloads - just pure code execution. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with Code Interpreter) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create agent with code interpreter tool (no files) + agent = project_client.agents.create_version( + agent_name="code-interpreter-simple-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can execute Python code.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[]))], + ), + description="Simple code interpreter agent for basic Python execution.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "code-interpreter-simple-agent" + assert agent.version is not None + + # Ask the agent to execute a complex Python calculation + # Problem: Calculate the sum of cubes from 1 to 50, then add 12!/(8!) + # Expected answer: 1637505 + print("\nAsking agent to calculate: sum of cubes from 1 to 50, plus 12!/(8!)") + + response = openai_client.responses.create( + input="Calculate this using Python: First, find the sum of cubes from 1 to 50 (1ยณ + 2ยณ + ... + 50ยณ). Then add 12 factorial divided by 8 factorial (12!/8!). What is the final result?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Get the response text + last_message = response.output[-1] + response_text = "" + + if last_message.type == "message": + for content_item in last_message.content: + if content_item.type == "output_text": + response_text += content_item.text + + print(f"Agent's response: {response_text}") + + # Verify the response contains the correct answer (1637505) + # Note: sum of cubes 1-50 = 1,625,625; 12!/8! = 11,880; total = 1,637,505 + assert "1637505" in response_text or "1,637,505" in response_text, ( + f"Expected answer 1637505 to be in response, but got: {response_text}" + ) + + print("โœ“ Code interpreter successfully executed Python code and returned correct answer") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_code_interpreter_file_generation(self, **kwargs): + """ + Test agent with Code Interpreter for file upload, processing, and download. + + This test verifies that an agent can: + 1. Work with uploaded CSV files + 2. Execute Python code to generate a chart + 3. Return a downloadable file + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /files openai_client.files.create() + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with Code Interpreter + file) + GET /containers/{container_id}/files/{file_id} openai_client.containers.files.content.retrieve() + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + DELETE /files/{file_id} openai_client.files.delete() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Get the path to the test CSV file + asset_file_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/synthetic_500_quarterly_results.csv") + ) + + assert os.path.exists(asset_file_path), f"Test CSV file not found at: {asset_file_path}" + print(f"Using test CSV file: {asset_file_path}") + + # Upload the CSV file + with open(asset_file_path, "rb") as f: + file = openai_client.files.create(purpose="assistants", file=f) + + print(f"File uploaded (id: {file.id})") + assert file.id is not None + + # Create agent with code interpreter tool and the uploaded file + agent = project_client.agents.create_version( + agent_name="code-interpreter-file-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can analyze data and create visualizations.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[file.id]))], + ), + description="Code interpreter agent for file processing and chart generation.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "code-interpreter-file-agent" + assert agent.version is not None + + # Ask the agent to create a chart from the CSV + print("\nAsking agent to create a bar chart...") + + response = openai_client.responses.create( + input="Create a bar chart showing operating profit by sector from the uploaded CSV file. Save it as a PNG file.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Extract file information from response annotations + file_id = "" + filename = "" + container_id = "" + + last_message = response.output[-1] + if last_message.type == "message": + for content_item in last_message.content: + if content_item.type == "output_text": + if content_item.annotations: + for annotation in content_item.annotations: + if annotation.type == "container_file_citation": + file_id = annotation.file_id + filename = annotation.filename + container_id = annotation.container_id + print(f"Found generated file: {filename} (ID: {file_id}, Container: {container_id})") + break + + # Verify that a file was generated + assert file_id, "Expected a file to be generated but no file ID found in response" + assert filename, "Expected a filename but none found in response" + assert container_id, "Expected a container ID but none found in response" + + print(f"โœ“ File generated successfully: {filename}") + + # Download the generated file + print(f"Downloading file {filename}...") + file_content = openai_client.containers.files.content.retrieve(file_id=file_id, container_id=container_id) + + # Read the content + content_bytes = file_content.read() + assert len(content_bytes) > 0, "Expected file content but got empty bytes" + + print(f"โœ“ File downloaded successfully ({len(content_bytes)} bytes)") + + # Verify it's a PNG file (check magic bytes) + if filename.endswith('.png'): + # PNG files start with: 89 50 4E 47 (โ€ฐPNG) + assert content_bytes[:4] == b'\x89PNG', "File does not appear to be a valid PNG" + print("โœ“ File is a valid PNG image") + + # Teardown + print("\nCleaning up...") + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + openai_client.files.delete(file.id) + print("Uploaded file deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py new file mode 100644 index 000000000000..4e34ed74153c --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py @@ -0,0 +1,327 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed----------------------------------------------------------------------------------------- +# cSpell:disable + +import os +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool + + +class TestAgentFileSearch(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_file_search(self, **kwargs): + """ + Test agent with File Search tool for document Q&A. + + This test verifies that an agent can: + 1. Upload and index documents into a vector store + 2. Use FileSearchTool to search through uploaded documents + 3. Answer questions based on document content + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /vector_stores openai_client.vector_stores.create() + POST /vector_stores/{id}/files openai_client.vector_stores.files.upload_and_poll() + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with FileSearchTool) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + DELETE /vector_stores/{id} openai_client.vector_stores.delete() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Get the path to the test file + asset_file_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/product_info.md") + ) + + assert os.path.exists(asset_file_path), f"Test file not found at: {asset_file_path}" + print(f"Using test file: {asset_file_path}") + + # Create vector store for file search + vector_store = openai_client.vector_stores.create(name="ProductInfoStore") + print(f"Vector store created (id: {vector_store.id})") + assert vector_store.id is not None + + # Upload file to vector store + with open(asset_file_path, "rb") as f: + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=f, + ) + + print(f"File uploaded (id: {file.id}, status: {file.status})") + assert file.id is not None + assert file.status == "completed", f"Expected file status 'completed', got '{file.status}'" + + # Create agent with file search tool + agent = project_client.agents.create_version( + agent_name="file-search-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can search through uploaded documents to answer questions.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Agent for testing file search capabilities.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "file-search-agent" + assert agent.version is not None + + # Ask a question about the uploaded document + print("\nAsking agent about the product information...") + + response = openai_client.responses.create( + input="What products are mentioned in the document?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Get the response text + response_text = response.output_text + print(f"\nAgent's response: {response_text[:300]}...") + + # Verify we got a meaningful response + assert len(response_text) > 50, "Expected a substantial response from the agent" + + # The response should mention finding information (indicating file search was used) + # We can't assert exact product names without knowing the file content, + # but we can verify the agent provided an answer + print("\nโœ“ Agent successfully used file search tool to answer question from uploaded document") + + # Teardown + print("\nCleaning up...") + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + openai_client.vector_stores.delete(vector_store.id) + print("Vector store deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_file_search_unsupported_file_type(self, **kwargs): + """ + Negative test: Verify that unsupported file types are rejected with clear error messages. + + This test validates that: + 1. CSV files (unsupported format) are rejected + 2. The error message clearly indicates the file type is not supported + 3. The error message lists supported file types + + This ensures good developer experience by providing actionable error messages. + """ + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create vector store + vector_store = openai_client.vector_stores.create(name="UnsupportedFileTestStore") + print(f"Vector store created (id: {vector_store.id})") + + # Create CSV file (unsupported format) + csv_content = """product,quarter,revenue +Widget A,Q1,15000 +Widget B,Q1,22000 +Widget A,Q2,18000 +Widget B,Q2,25000""" + + from io import BytesIO + csv_file = BytesIO(csv_content.encode('utf-8')) + csv_file.name = "sales_data.csv" + + # Attempt to upload unsupported file type + print("\nAttempting to upload CSV file (unsupported format)...") + try: + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=csv_file, + ) + # If we get here, the test should fail + openai_client.vector_stores.delete(vector_store.id) + pytest.fail("Expected BadRequestError for CSV file upload, but upload succeeded") + + except Exception as e: + error_message = str(e) + print(f"\nโœ“ Upload correctly rejected with error: {error_message[:200]}...") + + # Verify error message quality + assert "400" in error_message or "BadRequestError" in type(e).__name__, \ + "Should be a 400 Bad Request error" + + assert ".csv" in error_message.lower(), \ + "Error message should mention the CSV file extension" + + assert "not supported" in error_message.lower() or "unsupported" in error_message.lower(), \ + "Error message should clearly state the file type is not supported" + + # Check that supported file types are mentioned (helpful for developers) + error_lower = error_message.lower() + has_supported_list = any(ext in error_lower for ext in [".txt", ".pdf", ".md", ".py"]) + assert has_supported_list, \ + "Error message should list examples of supported file types" + + print("โœ“ Error message is clear and actionable") + print(" - Mentions unsupported file type (.csv)") + print(" - States it's not supported") + print(" - Lists supported file types") + + # Cleanup + openai_client.vector_stores.delete(vector_store.id) + print("\nVector store deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_file_search_multi_turn_conversation(self, **kwargs): + """ + Test multi-turn conversation with File Search. + + This test verifies that an agent can maintain context across multiple turns + while using File Search to answer follow-up questions. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create a document with information about products + product_info = """Product Catalog: + +Widget A: +- Price: $150 +- Category: Electronics +- Stock: 50 units +- Rating: 4.5/5 stars + +Widget B: +- Price: $220 +- Category: Electronics +- Stock: 30 units +- Rating: 4.8/5 stars + +Widget C: +- Price: $95 +- Category: Home & Garden +- Stock: 100 units +- Rating: 4.2/5 stars +""" + + # Create vector store and upload document + vector_store = openai_client.vector_stores.create(name="ProductCatalog") + print(f"Vector store created: {vector_store.id}") + + from io import BytesIO + product_file = BytesIO(product_info.encode('utf-8')) + product_file.name = "products.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=product_file, + ) + print(f"Product catalog uploaded: {file.id}") + + # Create agent with File Search + agent = project_client.agents.create_version( + agent_name="product-catalog-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a product information assistant. Use file search to answer questions about products.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Agent for multi-turn product queries.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Ask about price + print("\n--- Turn 1: Initial query ---") + response_1 = openai_client.responses.create( + input="What is the price of Widget B?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "$220" in response_1_text or "220" in response_1_text, \ + "Response should mention Widget B's price" + + # Turn 2: Follow-up question (requires context from turn 1) + print("\n--- Turn 2: Follow-up query (testing context retention) ---") + response_2 = openai_client.responses.create( + input="What about its stock level?", + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + assert "30" in response_2_text or "thirty" in response_2_text.lower(), \ + "Response should mention Widget B's stock (30 units)" + + # Turn 3: Another follow-up (compare with different product) + print("\n--- Turn 3: Comparison query ---") + response_3 = openai_client.responses.create( + input="How does that compare to Widget A's stock?", + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + assert "50" in response_3_text or "fifty" in response_3_text.lower(), \ + "Response should mention Widget A's stock (50 units)" + + # Turn 4: New topic (testing topic switching) + print("\n--- Turn 4: Topic switch ---") + response_4 = openai_client.responses.create( + input="Which widget has the highest rating?", + previous_response_id=response_3.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_4_text = response_4.output_text + print(f"Response 4: {response_4_text[:200]}...") + assert "widget b" in response_4_text.lower() or "4.8" in response_4_text, \ + "Response should identify Widget B as highest rated (4.8/5)" + + print("\nโœ“ Multi-turn conversation successful!") + print(" - Context maintained across turns") + print(" - Follow-up questions handled correctly") + print(" - Topic switching works") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py new file mode 100644 index 000000000000..de457e51809d --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py @@ -0,0 +1,137 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import os +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool + + +class TestAgentFileSearchStream(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_file_search_stream(self, **kwargs): + """ + Test agent with File Search tool using streaming responses. + + This test verifies that an agent can: + 1. Upload and index documents into a vector store + 2. Use FileSearchTool with streaming enabled + 3. Stream back responses based on document content + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /vector_stores openai_client.vector_stores.create() + POST /vector_stores/{id}/files openai_client.vector_stores.files.upload_and_poll() + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (stream=True, with FileSearchTool) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + DELETE /vector_stores/{id} openai_client.vector_stores.delete() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Get the path to the test file + asset_file_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/product_info.md") + ) + + assert os.path.exists(asset_file_path), f"Test file not found at: {asset_file_path}" + print(f"Using test file: {asset_file_path}") + + # Create vector store for file search + vector_store = openai_client.vector_stores.create(name="ProductInfoStoreStream") + print(f"Vector store created (id: {vector_store.id})") + assert vector_store.id is not None + + # Upload file to vector store + with open(asset_file_path, "rb") as f: + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=f, + ) + + print(f"File uploaded (id: {file.id}, status: {file.status})") + assert file.id is not None + assert file.status == "completed", f"Expected file status 'completed', got '{file.status}'" + + # Create agent with file search tool + agent = project_client.agents.create_version( + agent_name="file-search-stream-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can search through uploaded documents to answer questions.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Agent for testing file search with streaming.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "file-search-stream-agent" + assert agent.version is not None + + # Ask a question with streaming enabled + print("\nAsking agent about the product information (streaming)...") + + stream_response = openai_client.responses.create( + stream=True, + input="What products are mentioned in the document? Please provide a brief summary.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Collect streamed response + response_text = "" + response_id = None + events_received = 0 + + for event in stream_response: + events_received += 1 + + if event.type == "response.output_item.done": + if event.item.type == "message": + for content_item in event.item.content: + if content_item.type == "output_text": + response_text += content_item.text + + elif event.type == "response.completed": + response_id = event.response.id + # Could also use event.response.output_text + + print(f"\nStreaming completed (id: {response_id}, events: {events_received})") + assert response_id is not None, "Expected response ID from stream" + assert events_received > 0, "Expected to receive stream events" + + print(f"Agent's streamed response: {response_text[:300]}...") + + # Verify we got a meaningful response + assert len(response_text) > 50, "Expected a substantial response from the agent" + + print("\nโœ“ Agent successfully streamed responses using file search tool") + + # Teardown + print("\nCleaning up...") + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + openai_client.vector_stores.delete(vector_store.id) + print("Vector store deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py new file mode 100644 index 000000000000..38116ca09209 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py @@ -0,0 +1,504 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import json +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FunctionTool +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestAgentFunctionTool(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_function_tool(self, **kwargs): + """ + Test agent with custom function tool. + + This test verifies that an agent can: + 1. Use a custom function tool defined by the developer + 2. Request function calls when needed + 3. Receive function results and incorporate them into responses + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (triggers function) + POST /openai/responses openai_client.responses.create() (with function result) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Define a function tool for the model to use + func_tool = FunctionTool( + name="get_weather", + description="Get the current weather for a location.", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "A city name like Seattle or London", + }, + }, + "required": ["location"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent with function tool + agent = project_client.agents.create_version( + agent_name="function-tool-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can check the weather. Use the get_weather function when users ask about weather.", + tools=[func_tool], + ), + description="Agent for testing function tool capabilities.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version}, model: {agent.definition['model']})") + assert agent.id is not None + assert agent.name == "function-tool-agent" + assert agent.version is not None + + # Ask a question that should trigger the function call + print("\nAsking agent: What's the weather in Seattle?") + + response = openai_client.responses.create( + input="What's the weather in Seattle?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + + # Check for function calls in the response + function_calls_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "function_call": + function_calls_found += 1 + print(f"Found function call (id: {item.call_id}, name: {item.name})") + + # Parse the arguments + arguments = json.loads(item.arguments) + print(f"Function arguments: {arguments}") + + # Verify the function call is for get_weather + assert item.name == "get_weather", f"Expected function name 'get_weather', got '{item.name}'" + assert "location" in arguments, "Expected 'location' in function arguments" + assert "seattle" in arguments["location"].lower(), f"Expected Seattle in location, got {arguments['location']}" + + # Simulate the function execution and provide a result + weather_result = { + "location": arguments["location"], + "temperature": "72ยฐF", + "condition": "Sunny", + "humidity": "45%", + } + + # Add the function result to the input list + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(weather_result), + ) + ) + print(f"โœ“ Prepared function result: {weather_result}") + + # Verify that at least one function call was made + assert function_calls_found > 0, "Expected at least 1 function call, but found none" + print(f"\nโœ“ Processed {function_calls_found} function call(s)") + + # Send the function results back to get the final response + print("\nSending function results back to agent...") + + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Final response completed (id: {response.id})") + assert response.id is not None + + # Get the final response text + response_text = response.output_text + print(f"\nAgent's final response: {response_text}") + + # Verify the response incorporates the weather data + assert len(response_text) > 20, "Expected a meaningful response from the agent" + + # Check that the response mentions the weather information we provided + response_lower = response_text.lower() + assert any(keyword in response_lower for keyword in ["72", "sunny", "weather", "seattle"]), ( + f"Expected response to mention weather information, but got: {response_text}" + ) + + print("\nโœ“ Agent successfully used function tool and incorporated results into response") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): + """ + Test multi-turn conversation where agent calls functions multiple times. + + This tests: + - Multiple function calls across different turns + - Context retention between turns + - Ability to use previous function results in subsequent queries + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Define multiple function tools + get_weather = FunctionTool( + name="get_weather", + description="Get current weather for a city", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city name", + }, + }, + "required": ["city"], + "additionalProperties": False, + }, + strict=True, + ) + + get_temperature_forecast = FunctionTool( + name="get_temperature_forecast", + description="Get 3-day temperature forecast for a city", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city name", + }, + }, + "required": ["city"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent with multiple functions + agent = project_client.agents.create_version( + agent_name="weather-assistant-multi-turn", + definition=PromptAgentDefinition( + model=model, + instructions="You are a weather assistant. Use available functions to answer weather questions.", + tools=[get_weather, get_temperature_forecast], + ), + description="Weather assistant for multi-turn testing.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Get current weather + print("\n--- Turn 1: Current weather query ---") + response_1 = openai_client.responses.create( + input="What's the weather in New York?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_1.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + assert item.name == "get_weather" + + # Simulate weather API response + weather_data = {"temperature": 68, "condition": "Cloudy", "humidity": 65} + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(weather_data), + ) + ) + + # Get response with function results + response_1 = openai_client.responses.create( + input=input_list, + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "68" in response_1_text or "cloudy" in response_1_text.lower() + + # Turn 2: Follow-up with forecast (requires context) + print("\n--- Turn 2: Follow-up forecast query ---") + response_2 = openai_client.responses.create( + input="What about the forecast for the next few days?", + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle forecast function call + input_list = [] + for item in response_2.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + assert item.name == "get_temperature_forecast" + + # Agent should remember we're talking about New York + args = json.loads(item.arguments) + assert "new york" in args["city"].lower() + + # Simulate forecast API response + forecast_data = { + "city": "New York", + "forecast": [ + {"day": "Tomorrow", "temp": 70}, + {"day": "Day 2", "temp": 72}, + {"day": "Day 3", "temp": 69} + ] + } + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(forecast_data), + ) + ) + + # Get response with forecast + response_2 = openai_client.responses.create( + input=input_list, + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + assert "70" in response_2_text or "72" in response_2_text + + # Turn 3: Compare with another city + print("\n--- Turn 3: New city query ---") + response_3 = openai_client.responses.create( + input="How does that compare to Seattle's weather?", + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function calls for Seattle (agent might call both weather and forecast) + input_list = [] + for item in response_3.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + args = json.loads(item.arguments) + assert "seattle" in args["city"].lower() + + # Handle based on function name + if item.name == "get_weather": + weather_data = {"temperature": 58, "condition": "Rainy", "humidity": 80} + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(weather_data), + ) + ) + elif item.name == "get_temperature_forecast": + forecast_data = { + "city": "Seattle", + "forecast": [ + {"day": "Tomorrow", "temp": 56}, + {"day": "Day 2", "temp": 59}, + {"day": "Day 3", "temp": 57} + ] + } + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(forecast_data), + ) + ) + + # Get final comparison response + response_3 = openai_client.responses.create( + input=input_list, + previous_response_id=response_3.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + # Agent should mention Seattle weather (either 58 for current or comparison) + assert "seattle" in response_3_text.lower() or any(temp in response_3_text for temp in ["58", "56", "59"]) + + print("\nโœ“ Multi-turn conversation with multiple function calls successful!") + print(" - Multiple functions called across turns") + print(" - Context maintained (agent remembered New York)") + print(" - Comparison between cities works") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_function_tool_context_dependent_followup(self, **kwargs): + """ + Test deeply context-dependent follow-ups (e.g., unit conversion, clarification). + + This tests that the agent truly uses previous response content, not just + remembering parameters from the first query. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Define function tool + get_temperature = FunctionTool( + name="get_temperature", + description="Get current temperature for a city in Fahrenheit", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city name", + }, + }, + "required": ["city"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="temperature-assistant-context", + definition=PromptAgentDefinition( + model=model, + instructions="You are a temperature assistant. Answer temperature questions.", + tools=[get_temperature], + ), + description="Temperature assistant for context testing.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Get temperature in Fahrenheit + print("\n--- Turn 1: Get temperature ---") + response_1 = openai_client.responses.create( + input="What's the temperature in Boston?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_1.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"temperature": 72, "unit": "F"}), + ) + ) + + response_1 = openai_client.responses.create( + input=input_list, + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "72" in response_1_text, "Should mention 72ยฐF" + + # Turn 2: Context-dependent follow-up (convert the previous number) + print("\n--- Turn 2: Context-dependent conversion ---") + response_2 = openai_client.responses.create( + input="What is that in Celsius?", # "that" refers to the 72ยฐF from previous response + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + + # Should convert 72ยฐF to ~22ยฐC (without calling the function again) + # The agent should use the previous response's value + response_2_lower = response_2_text.lower() + assert "celsius" in response_2_lower or "ยฐc" in response_2_lower or "c" in response_2_lower, \ + "Response should mention Celsius" + assert any(temp in response_2_text for temp in ["22", "22.2", "22.22", "20", "21", "23"]), \ + f"Response should calculate Celsius from 72ยฐF (~22ยฐC), got: {response_2_text}" + + # Turn 3: Another context-dependent follow-up (comparison) + print("\n--- Turn 3: Compare to another value ---") + response_3 = openai_client.responses.create( + input="Is that warmer or colder than 25ยฐC?", # "that" refers to the Celsius value just mentioned + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + + # 22ยฐC is colder than 25ยฐC + response_3_lower = response_3_text.lower() + assert "colder" in response_3_lower or "cooler" in response_3_lower or "lower" in response_3_lower, \ + f"Response should indicate 22ยฐC is colder than 25ยฐC, got: {response_3_text}" + + print("\nโœ“ Context-dependent follow-ups successful!") + print(" - Agent converted temperature from previous response") + print(" - Agent compared values from conversation history") + print(" - No unnecessary function calls made") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py new file mode 100644 index 000000000000..5366f1b429ef --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py @@ -0,0 +1,121 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import base64 +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, ImageGenTool + + +class TestAgentImageGeneration(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_image_generation(self, **kwargs): + """ + Test agent with Image Generation tool. + + This test verifies that an agent can: + 1. Use ImageGenTool to generate images from text prompts + 2. Return base64-encoded image data + 3. Decode and validate the image format + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with ImageGenTool) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + # Skip for usw21 - image model not deployed there + endpoint = kwargs.get("azure_ai_projects_tests_agents_project_endpoint", "") + if "usw2" in endpoint.lower(): + pytest.skip("Image generation not available in usw21 (image model not deployed)") + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Disable retries for faster failure when service returns 500 + openai_client.max_retries = 0 + + # Create agent with image generation tool + agent = project_client.agents.create_version( + agent_name="image-gen-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Generate images based on user prompts", + tools=[ImageGenTool(quality="low", size="1024x1024")], + ), + description="Agent for testing image generation.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "image-gen-agent" + assert agent.version is not None + + # Request image generation + print("\nAsking agent to generate an image of a simple geometric shape...") + + response = openai_client.responses.create( + input="Generate an image of a blue circle on a white background.", + extra_headers={ + "x-ms-oai-image-generation-deployment": "gpt-image-1-mini" + }, # Required for image generation + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response created (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Extract image data from response + image_data = [output.result for output in response.output if output.type == "image_generation_call"] + + # Verify image was generated + assert len(image_data) > 0, "Expected at least one image to be generated" + assert image_data[0], "Expected image data to be non-empty" + + print(f"โœ“ Image data received ({len(image_data[0])} base64 characters)") + + # Decode the base64 image + image_bytes = b"" + try: + image_bytes = base64.b64decode(image_data[0]) + assert len(image_bytes) > 0, "Decoded image should have content" + print(f"โœ“ Image decoded successfully ({len(image_bytes)} bytes)") + except Exception as e: + pytest.fail(f"Failed to decode base64 image data: {e}") + + # Verify it's a PNG image (check magic bytes) + # PNG files start with: 89 50 4E 47 (โ€ฐPNG) + assert image_bytes[:4] == b'\x89PNG', "Image does not appear to be a valid PNG" + print("โœ“ Image is a valid PNG") + + # Verify reasonable image size (should be > 1KB for a 1024x1024 image) + assert len(image_bytes) > 1024, f"Image seems too small ({len(image_bytes)} bytes)" + print(f"โœ“ Image size is reasonable ({len(image_bytes):,} bytes)") + + print("\nโœ“ Agent successfully generated and returned a valid image") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py new file mode 100644 index 000000000000..ffb19d3be1e5 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py @@ -0,0 +1,303 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, MCPTool, Tool +from openai.types.responses.response_input_param import McpApprovalResponse, ResponseInputParam + + +class TestAgentMCP(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_mcp_basic(self, **kwargs): + """ + Test agent with MCP (Model Context Protocol) tool for external API access. + + This test verifies that an agent can: + 1. Use an MCP tool to access external resources (GitHub repo) + 2. Request approval for MCP operations + 3. Process approval responses + 4. Complete the task using the MCP tool + + The test uses a public GitHub MCP server that provides access to + Azure REST API specifications. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + POST /openai/conversations openai_client.conversations.create() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with MCP tool) + POST /openai/responses openai_client.responses.create() (with approval) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create MCP tool that connects to a public GitHub repo via MCP server + mcp_tool = MCPTool( + server_label="api-specs", + server_url="https://gitmcp.io/Azure/azure-rest-api-specs", + require_approval="always", + ) + + tools: list[Tool] = [mcp_tool] + + # Create agent with MCP tool + agent = project_client.agents.create_version( + agent_name="mcp-basic-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful agent that can use MCP tools to assist users. Use the available MCP tools to answer questions and perform tasks.", + tools=tools, + ), + description="Agent for testing basic MCP functionality.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "mcp-basic-agent" + assert agent.version is not None + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Created conversation (id: {conversation.id})") + assert conversation.id is not None + + # Send initial request that will trigger the MCP tool + print("\nAsking agent to summarize Azure REST API specs README...") + + response = openai_client.responses.create( + conversation=conversation.id, + input="Please summarize the Azure REST API specifications Readme", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Process any MCP approval requests + approval_requests_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "mcp_approval_request": + approval_requests_found += 1 + print(f"Found MCP approval request (id: {item.id}, server: {item.server_label})") + + if item.server_label == "api-specs" and item.id: + # Approve the MCP request + input_list.append( + McpApprovalResponse( + type="mcp_approval_response", + approve=True, + approval_request_id=item.id, + ) + ) + print(f"โœ“ Approved MCP request: {item.id}") + + # Verify that at least one approval request was generated + assert approval_requests_found > 0, ( + f"Expected at least 1 MCP approval request, but found {approval_requests_found}" + ) + + print(f"\nโœ“ Processed {approval_requests_found} MCP approval request(s)") + + # Send the approval response to continue the agent's work + print("\nSending approval response to continue agent execution...") + + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Final response completed (id: {response.id})") + assert response.id is not None + + # Get the final response text + response_text = response.output_text + print(f"\nAgent's response preview: {response_text[:200]}...") + + # Verify we got a meaningful response + assert len(response_text) > 100, "Expected a substantial response from the agent" + + # Check that the response mentions Azure or REST API (indicating it accessed the repo) + assert any(keyword in response_text.lower() for keyword in ["azure", "rest", "api", "specification"]), ( + f"Expected response to mention Azure/REST API, but got: {response_text[:200]}" + ) + + print("\nโœ“ Agent successfully used MCP tool to access GitHub repo and complete task") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_mcp_with_project_connection(self, **kwargs): + """ + Test agent with MCP tool using a project connection for authentication. + + This test verifies that an agent can: + 1. Use an MCP tool with a project connection (GitHub PAT) + 2. Access authenticated GitHub API endpoints + 3. Request and process approval for MCP operations + 4. Return personal GitHub profile information + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + POST /openai/conversations openai_client.conversations.create() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with MCP + connection) + POST /openai/responses openai_client.responses.create() (with approval) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Get MCP project connection from environment + mcp_project_connection_id = kwargs.get("azure_ai_projects_tests_mcp_project_connection_id") + + if not mcp_project_connection_id: + pytest.skip("MCP_PROJECT_CONNECTION_ID environment variable not set") + + assert isinstance(mcp_project_connection_id, str), "mcp_project_connection_id must be a string" + print(f"Using MCP project connection: {mcp_project_connection_id}") + + # Create MCP tool with project connection for GitHub API access + mcp_tool = MCPTool( + server_label="github-api", + server_url="https://api.githubcopilot.com/mcp", + require_approval="always", + project_connection_id=mcp_project_connection_id, + ) + + tools: list[Tool] = [mcp_tool] + + # Create agent with MCP tool + agent = project_client.agents.create_version( + agent_name="mcp-connection-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Use MCP tools as needed to access GitHub information.", + tools=tools, + ), + description="Agent for testing MCP with project connection.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "mcp-connection-agent" + assert agent.version is not None + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Created conversation (id: {conversation.id})") + assert conversation.id is not None + + # Send initial request that will trigger the MCP tool with authentication + print("\nAsking agent to get GitHub profile username...") + + response = openai_client.responses.create( + conversation=conversation.id, + input="What is my username in Github profile?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Process any MCP approval requests + approval_requests_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "mcp_approval_request": + approval_requests_found += 1 + print(f"Found MCP approval request (id: {item.id}, server: {item.server_label})") + + if item.server_label == "github-api" and item.id: + # Approve the MCP request + input_list.append( + McpApprovalResponse( + type="mcp_approval_response", + approve=True, + approval_request_id=item.id, + ) + ) + print(f"โœ“ Approved MCP request: {item.id}") + + # Verify that at least one approval request was generated + assert approval_requests_found > 0, ( + f"Expected at least 1 MCP approval request, but found {approval_requests_found}" + ) + + print(f"\nโœ“ Processed {approval_requests_found} MCP approval request(s)") + + # Send the approval response to continue the agent's work + print("\nSending approval response to continue agent execution...") + + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Final response completed (id: {response.id})") + assert response.id is not None + + # Get the final response text + response_text = response.output_text + print(f"\nAgent's response: {response_text}") + + # Verify we got a meaningful response with a GitHub username + assert len(response_text) > 5, "Expected a response with a GitHub username" + + # The response should contain some indication of a username or GitHub profile info + # We can't assert the exact username, but we can verify it's not an error + assert "error" not in response_text.lower() or "username" in response_text.lower(), ( + f"Expected response to contain GitHub profile info, but got: {response_text}" + ) + + print("\nโœ“ Agent successfully used MCP tool with project connection to access authenticated GitHub API") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py new file mode 100644 index 000000000000..01f499d4e2aa --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py @@ -0,0 +1,394 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +Test agents using tools within conversations. + +This test file demonstrates how to use various agent tools (both server-side and client-side) +within the context of conversations, testing conversation state management with tool interactions. +""" + +import json +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + FunctionTool, + FileSearchTool, + CodeInterpreterTool, + CodeInterpreterToolAuto, + PromptAgentDefinition, +) +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestAgentToolsWithConversations(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_function_tool_with_conversation(self, **kwargs): + """ + Test using FunctionTool within a conversation. + + This tests: + - Creating a conversation + - Multiple turns with function calls + - Conversation state preservation across function calls + - Using conversation_id parameter + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Define a calculator function + calculator = FunctionTool( + name="calculator", + description="Perform basic arithmetic operations", + parameters={ + "type": "object", + "properties": { + "operation": { + "type": "string", + "enum": ["add", "subtract", "multiply", "divide"], + "description": "The operation to perform", + }, + "a": {"type": "number", "description": "First number"}, + "b": {"type": "number", "description": "Second number"}, + }, + "required": ["operation", "a", "b"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="calculator-agent-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are a calculator assistant. Use the calculator function to perform operations.", + tools=[calculator], + ), + description="Calculator agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Add two numbers + print("\n--- Turn 1: Addition ---") + response_1 = openai_client.responses.create( + input="What is 15 plus 27?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_1.output: + if item.type == "function_call": + print(f"Function called: {item.name} with {item.arguments}") + args = json.loads(item.arguments) + + # Execute calculator + result = { + "add": args["a"] + args["b"], + "subtract": args["a"] - args["b"], + "multiply": args["a"] * args["b"], + "divide": args["a"] / args["b"] if args["b"] != 0 else "Error: Division by zero", + }[args["operation"]] + + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"result": result}), + ) + ) + + response_1 = openai_client.responses.create( + input=input_list, + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response 1: {response_1.output_text[:100]}...") + assert "42" in response_1.output_text + + # Turn 2: Follow-up using previous result (tests conversation memory) + print("\n--- Turn 2: Follow-up using conversation context ---") + response_2 = openai_client.responses.create( + input="Now multiply that result by 2", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list = [] + for item in response_2.output: + if item.type == "function_call": + print(f"Function called: {item.name} with {item.arguments}") + args = json.loads(item.arguments) + + # Should be multiplying 42 by 2 + assert args["operation"] == "multiply" + assert args["a"] == 42 or args["b"] == 42 + + result = args["a"] * args["b"] + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"result": result}), + ) + ) + + response_2 = openai_client.responses.create( + input=input_list, + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response 2: {response_2.output_text[:100]}...") + assert "84" in response_2.output_text + + print("\nโœ“ Function tool with conversation successful!") + print(" - Conversation preserved state across function calls") + print(" - Agent remembered previous result (42)") + + # Verify conversation state by reading items + print("\n--- Verifying conversation state ---") + all_items = list(openai_client.conversations.items.list(conversation.id)) + print(f"Total conversation items: {len(all_items)}") + + # Count different item types + user_messages = sum(1 for item in all_items if item.type == "message" and item.role == "user") + assistant_messages = sum(1 for item in all_items if item.type == "message" and item.role == "assistant") + function_calls = sum(1 for item in all_items if item.type == "function_call") + function_outputs = sum(1 for item in all_items if item.type == "function_call_output") + + print(f" User messages: {user_messages}") + print(f" Assistant messages: {assistant_messages}") + print(f" Function calls: {function_calls}") + print(f" Function outputs: {function_outputs}") + + # Verify we have expected items + assert user_messages >= 2, "Expected at least 2 user messages (two turns)" + assert function_calls >= 2, "Expected at least 2 function calls (one per turn)" + assert function_outputs >= 2, "Expected at least 2 function outputs" + print("โœ“ Conversation state verified - all items preserved") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_file_search_with_conversation(self, **kwargs): + """ + Test using FileSearchTool within a conversation. + + This tests: + - Server-side tool execution within conversation + - Multiple search queries in same conversation + - Conversation context retention + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create documents with related information + doc_content = """Product Catalog + +Widget A: +- Price: $100 +- Stock: 50 units +- Rating: 4.5/5 +- Category: Electronics + +Widget B: +- Price: $220 +- Stock: 30 units +- Rating: 4.8/5 +- Category: Electronics + +Widget C: +- Price: $75 +- Stock: 100 units +- Rating: 4.2/5 +- Category: Home Goods +""" + + # Create vector store and upload document + vector_store = openai_client.vector_stores.create(name="ConversationTestStore") + print(f"Vector store created: {vector_store.id}") + + from io import BytesIO + file = BytesIO(doc_content.encode('utf-8')) + file.name = "products.txt" + + uploaded = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=file, + ) + print(f"Document uploaded: {uploaded.id}") + + # Create agent with file search + agent = project_client.agents.create_version( + agent_name="search-agent-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are a product search assistant. Answer questions about products.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Search agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Search for highest rated + print("\n--- Turn 1: Search query ---") + response_1 = openai_client.responses.create( + input="Which widget has the highest rating?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:150]}...") + assert "Widget B" in response_1_text or "4.8" in response_1_text + + # Turn 2: Follow-up about that specific product (tests context retention) + print("\n--- Turn 2: Contextual follow-up ---") + response_2 = openai_client.responses.create( + input="What is its price?", # "its" refers to Widget B from previous turn + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:150]}...") + assert "220" in response_2_text + + # Turn 3: New search in same conversation + print("\n--- Turn 3: New search in same conversation ---") + response_3 = openai_client.responses.create( + input="Which widget is in the Home Goods category?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:150]}...") + assert "Widget C" in response_3_text + + print("\nโœ“ File search with conversation successful!") + print(" - Multiple searches in same conversation") + print(" - Context preserved (agent remembered Widget B)") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_code_interpreter_with_conversation(self, **kwargs): + """ + Test using CodeInterpreterTool within a conversation. + + This tests: + - Server-side code execution within conversation + - Multiple code executions in same conversation + - Variables/state persistence across turns + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create agent with code interpreter + agent = project_client.agents.create_version( + agent_name="code-agent-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are a data analysis assistant. Use Python to perform calculations.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[]))], + ), + description="Code interpreter agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Calculate average + print("\n--- Turn 1: Calculate average ---") + response_1 = openai_client.responses.create( + input="Calculate the average of these numbers: 10, 20, 30, 40, 50", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "30" in response_1_text + + # Turn 2: Follow-up calculation (tests conversation context) + print("\n--- Turn 2: Follow-up calculation ---") + response_2 = openai_client.responses.create( + input="Now calculate the standard deviation of those same numbers", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + # Standard deviation should be approximately 14.14 or similar + assert any(num in response_2_text for num in ["14", "15", "standard"]) + + # Turn 3: Another operation in same conversation + print("\n--- Turn 3: New calculation ---") + response_3 = openai_client.responses.create( + input="Create a list of squares from 1 to 5", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + assert "1" in response_3_text and "4" in response_3_text and "25" in response_3_text + + print("\nโœ“ Code interpreter with conversation successful!") + print(" - Multiple code executions in conversation") + print(" - Context preserved across calculations") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + print("Cleanup completed") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py new file mode 100644 index 000000000000..d0ebccafe37b --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py @@ -0,0 +1,99 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, WebSearchPreviewTool, ApproximateLocation + + +class TestAgentWebSearch(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_web_search(self, **kwargs): + """ + Test agent with Web Search tool for real-time information. + + This test verifies that an agent can: + 1. Use WebSearchPreviewTool to search the web + 2. Get current/real-time information + 3. Provide answers based on web search results + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with WebSearchPreviewTool) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create agent with web search tool + agent = project_client.agents.create_version( + agent_name="web-search-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can search the web for current information.", + tools=[ + WebSearchPreviewTool( + user_location=ApproximateLocation(country="US", city="Seattle", region="Washington") + ) + ], + ), + description="Agent for testing web search capabilities.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "web-search-agent" + assert agent.version is not None + + # Ask a question that requires web search for current information + print("\nAsking agent about current weather...") + + response = openai_client.responses.create( + input="What is the current weather in Seattle?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Get the response text + response_text = response.output_text + print(f"\nAgent's response: {response_text[:300]}...") + + # Verify we got a meaningful response + assert len(response_text) > 30, "Expected a substantial response from the agent" + + # The response should mention weather-related terms or Seattle + response_lower = response_text.lower() + assert any(keyword in response_lower for keyword in ["weather", "temperature", "seattle", "forecast"]), ( + f"Expected response to contain weather information, but got: {response_text[:200]}" + ) + + print("\nโœ“ Agent successfully used web search tool to get current information") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/test_base.py b/sdk/ai/azure-ai-projects/tests/test_base.py index c371560cb41b..857c2c6e4f90 100644 --- a/sdk/ai/azure-ai-projects/tests/test_base.py +++ b/sdk/ai/azure-ai-projects/tests/test_base.py @@ -62,6 +62,10 @@ azure_ai_projects_tests_tracing_project_endpoint="https://sanitized-account-name.services.ai.azure.com/api/projects/sanitized-project-name", azure_ai_projects_tests_container_app_resource_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.App/containerApps/00000", azure_ai_projects_tests_container_ingress_subdomain_suffix="00000", + azure_ai_projects_tests_bing_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-bing-connection", + azure_ai_projects_tests_ai_search_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-ai-search-connection", + azure_ai_projects_tests_ai_search_index_name="sanitized-index-name", + azure_ai_projects_tests_mcp_project_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-mcp-connection", ) From 86b75396f3b1931bc128305b11ab4319471d8d1c Mon Sep 17 00:00:00 2001 From: Paul Batum Date: Thu, 20 Nov 2025 18:04:41 -0800 Subject: [PATCH 2/8] Make image generation test more flexible with configurable model deployment - Image model deployment is now configurable via AZURE_AI_PROJECTS_TESTS_IMAGE_MODEL_DEPLOYMENT_NAME - Test automatically checks if the image model deployment exists in the project using deployments.get() - Gracefully skips the test if the image model is not available (instead of hardcoded region checks) - Added image_model_deployment_name to servicePreparer for proper sanitization in recordings - Defaults to 'gpt-image-1-mini' if environment variable is not set This allows the test to run across different regions/projects with varying image model availability. --- .../tools/test_agent_image_generation.py | 21 +++++++++++++------ sdk/ai/azure-ai-projects/tests/test_base.py | 1 + 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py index 5366f1b429ef..a98feed35bb5 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py @@ -5,11 +5,13 @@ # ------------------------------------ # cSpell:disable +import os import base64 import pytest from test_base import TestBase, servicePreparer from devtools_testutils import is_live_and_not_recording from azure.ai.projects.models import PromptAgentDefinition, ImageGenTool +from azure.core.exceptions import ResourceNotFoundError class TestAgentImageGeneration(TestBase): @@ -42,17 +44,24 @@ def test_agent_image_generation(self, **kwargs): DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() """ - # Skip for usw21 - image model not deployed there - endpoint = kwargs.get("azure_ai_projects_tests_agents_project_endpoint", "") - if "usw2" in endpoint.lower(): - pytest.skip("Image generation not available in usw21 (image model not deployed)") - + # Get the image model deployment name from environment variable + image_model_deployment = os.environ.get("AZURE_AI_PROJECTS_TESTS_IMAGE_MODEL_DEPLOYMENT_NAME", "gpt-image-1-mini") + model = self.test_agents_params["model_deployment_name"] # Setup project_client = self.create_client(operation_group="agents", **kwargs) openai_client = project_client.get_openai_client() + # Check if the image model deployment exists in the project + try: + deployment = project_client.deployments.get(image_model_deployment) + print(f"Image model deployment found: {deployment.name}") + except ResourceNotFoundError: + pytest.skip(f"Image generation model '{image_model_deployment}' not available in this project") + except Exception as e: + pytest.skip(f"Unable to verify image model deployment: {e}") + # Disable retries for faster failure when service returns 500 openai_client.max_retries = 0 @@ -77,7 +86,7 @@ def test_agent_image_generation(self, **kwargs): response = openai_client.responses.create( input="Generate an image of a blue circle on a white background.", extra_headers={ - "x-ms-oai-image-generation-deployment": "gpt-image-1-mini" + "x-ms-oai-image-generation-deployment": image_model_deployment }, # Required for image generation extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) diff --git a/sdk/ai/azure-ai-projects/tests/test_base.py b/sdk/ai/azure-ai-projects/tests/test_base.py index 857c2c6e4f90..c717192a28e4 100644 --- a/sdk/ai/azure-ai-projects/tests/test_base.py +++ b/sdk/ai/azure-ai-projects/tests/test_base.py @@ -66,6 +66,7 @@ azure_ai_projects_tests_ai_search_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-ai-search-connection", azure_ai_projects_tests_ai_search_index_name="sanitized-index-name", azure_ai_projects_tests_mcp_project_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-mcp-connection", + azure_ai_projects_tests_image_model_deployment_name="gpt-image-1-mini" ) From 6e5bd035cd12a3212a73913a2ab1c83cd12e19ff Mon Sep 17 00:00:00 2001 From: Paul Batum Date: Thu, 20 Nov 2025 19:09:58 -0800 Subject: [PATCH 3/8] Remove test_model_verification from main test suite This test will be maintained in a separate branch for specialized testing. --- .../tests/agents/test_model_verification.py | 93 ------------------- 1 file changed, 93 deletions(-) delete mode 100644 sdk/ai/azure-ai-projects/tests/agents/test_model_verification.py diff --git a/sdk/ai/azure-ai-projects/tests/agents/test_model_verification.py b/sdk/ai/azure-ai-projects/tests/agents/test_model_verification.py deleted file mode 100644 index 1d22c6cbf491..000000000000 --- a/sdk/ai/azure-ai-projects/tests/agents/test_model_verification.py +++ /dev/null @@ -1,93 +0,0 @@ -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ -""" -Test to verify model substitution is working correctly. -Creates an agent and asks it to identify its model. -""" - -import uuid -import pytest -from test_base import TestBase, servicePreparer -from devtools_testutils import is_live_and_not_recording -from azure.ai.projects.models import PromptAgentDefinition - - -class TestModelVerification(TestBase): - - @servicePreparer() - @pytest.mark.skipif( - condition=(not is_live_and_not_recording()), - reason="Skipped because we cannot record network calls with OpenAI client", - ) - def test_model_identity(self, **kwargs): - """ - Simple test to verify which model is actually being used. - Creates an agent and asks it to identify its model. - - DOES NOT CLEAN UP - agents are left in place for verification. - """ - - # Get model from test_agents_params (which now reads from environment if available) - model = self.test_agents_params["model_deployment_name"] - print(f"\n{'='*80}") - print(f"๐Ÿ“‹ Model: {model}") - print(f"{'='*80}\n") - - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Create unique agent name with model and random ID to avoid conflicts - random_id = str(uuid.uuid4())[:8] - agent_name = f"model-verify-{model}-{random_id}" - - # Create agent - agent = project_client.agents.create_version( - agent_name=agent_name, - definition=PromptAgentDefinition( - model=model, - instructions="You are a helpful assistant. When asked what model you are, respond with your exact model name/identifier.", - ), - description=f"Model verification test for {model}", - ) - - print(f"โœ… Agent created:") - print(f" - ID: {agent.id}") - print(f" - Name: {agent.name}") - print(f" - Version: {agent.version}") - print(f" - Model parameter passed: {model}") - - assert agent.id is not None - assert agent.name == agent_name - - # Ask the agent what model it is - print(f"\nโ“ Asking agent: What model are you?") - - response = openai_client.responses.create( - input="What model are you? Please tell me your exact model name or identifier.", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - print(f"โœ… Response completed (id: {response.id})") - - response_text = response.output_text - print(f"\n๐Ÿค– Agent's response:") - print(f"{'='*80}") - print(response_text) - print(f"{'='*80}") - - # Basic assertions - assert response.id is not None - assert len(response_text) > 0, "Expected a response from the agent" - - # Print summary - print(f"\n๐Ÿ“Š SUMMARY:") - print(f" - Expected model: {model}") - print(f" - Agent name: {agent.name}") - print(f" - Agent response: {response_text[:100]}...") - - # NOTE: NOT cleaning up - agent stays for manual verification - print(f"\nโš ๏ธ Agent NOT deleted (left for verification)") - print(f" Agent: {agent.name}:{agent.version}") From 728e798c9f1f0b91cc491af86fc33bec29ba278b Mon Sep 17 00:00:00 2001 From: Paul Batum Date: Fri, 21 Nov 2025 10:28:05 -0800 Subject: [PATCH 4/8] run black to fix formatting --- ...est_agent_code_interpreter_and_function.py | 12 +- ..._agent_file_search_and_code_interpreter.py | 35 +++--- .../test_agent_file_search_and_function.py | 90 +++++++------- ...t_file_search_code_interpreter_function.py | 34 ++++-- .../test_multitool_with_conversations.py | 17 +-- .../agents/tools/test_agent_ai_search.py | 6 +- .../tools/test_agent_ai_search_async.py | 31 +++-- .../agents/tools/test_agent_bing_grounding.py | 26 ++-- .../tools/test_agent_code_interpreter.py | 44 +++---- .../agents/tools/test_agent_file_search.py | 96 +++++++-------- .../tools/test_agent_file_search_stream.py | 24 ++-- .../agents/tools/test_agent_function_tool.py | 113 ++++++++++-------- .../tools/test_agent_image_generation.py | 20 ++-- .../tests/agents/tools/test_agent_mcp.py | 60 +++++----- .../test_agent_tools_with_conversations.py | 23 ++-- .../agents/tools/test_agent_web_search.py | 16 +-- sdk/ai/azure-ai-projects/tests/test_base.py | 2 +- 17 files changed, 341 insertions(+), 308 deletions(-) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py index d9d5d6dc5576..69033a5a3917 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py @@ -69,17 +69,17 @@ def test_calculate_and_save(self, **kwargs): description="Agent with Code Interpreter and Function Tool.", ) print(f"Agent created (id: {agent.id})") - + # Use the agent response = openai_client.responses.create( input="Calculate 5 + 3 and save the result.", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) print(f"Response received (id: {response.id})") - + assert response.id is not None print("โœ“ Code Interpreter + Function Tool works!") - + # Cleanup project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) @@ -129,16 +129,16 @@ def test_generate_data_and_report(self, **kwargs): description="Agent for data generation and reporting.", ) print(f"Agent created (id: {agent.id})") - + # Request data generation and report response = openai_client.responses.create( input="Generate a list of 10 random numbers between 1 and 100, calculate their average, and create a report.", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + print(f"Response received (id: {response.id})") assert response.id is not None print("โœ“ Data generation and reporting works!") - + # Cleanup project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py index 594ad985d9b7..3d9ee0c68fd9 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py @@ -41,11 +41,12 @@ def test_find_and_analyze_data(self, **kwargs): # Create data file txt_content = "Sample data: 10, 20, 30, 40, 50" vector_store = openai_client.vector_stores.create(name="DataStore") - + from io import BytesIO - txt_file = BytesIO(txt_content.encode('utf-8')) + + txt_file = BytesIO(txt_content.encode("utf-8")) txt_file.name = "data.txt" - + file = openai_client.vector_stores.files.upload_and_poll( vector_store_id=vector_store.id, file=txt_file, @@ -66,18 +67,18 @@ def test_find_and_analyze_data(self, **kwargs): description="Agent with File Search and Code Interpreter.", ) print(f"Agent created (id: {agent.id})") - + # Use the agent response = openai_client.responses.create( input="Find the data file and calculate the average.", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) print(f"Response received (id: {response.id})") - + assert response.id is not None assert len(response.output_text) > 20 print("โœ“ File Search + Code Interpreter works!") - + # Cleanup project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) openai_client.vector_stores.delete(vector_store.id) @@ -109,11 +110,12 @@ def test_analyze_code_file(self, **kwargs): """ vector_store = openai_client.vector_stores.create(name="CodeAnalysisStore") - + from io import BytesIO - code_file = BytesIO(python_code.encode('utf-8')) + + code_file = BytesIO(python_code.encode("utf-8")) code_file.name = "fibonacci.py" - + file = openai_client.vector_stores.files.upload_and_poll( vector_store_id=vector_store.id, file=code_file, @@ -134,23 +136,24 @@ def test_analyze_code_file(self, **kwargs): description="Agent for code analysis.", ) print(f"Agent created (id: {agent.id})") - + # Request analysis response = openai_client.responses.create( input="Find the fibonacci code and explain what it does. What is the computational complexity?", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_text = response.output_text print(f"Response: {response_text[:300]}...") - + assert len(response_text) > 50 response_lower = response_lower = response_text.lower() - assert any(keyword in response_lower for keyword in ["fibonacci", "recursive", "complexity", "exponential"]), \ - "Expected analysis of fibonacci algorithm" - + assert any( + keyword in response_lower for keyword in ["fibonacci", "recursive", "complexity", "exponential"] + ), "Expected analysis of fibonacci algorithm" + print("โœ“ Code file analysis completed") - + # Cleanup project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) openai_client.vector_stores.delete(vector_store.id) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py index 3911289b21b0..4e310a9facf6 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py @@ -61,11 +61,12 @@ def test_data_analysis_workflow(self, **kwargs): # Create vector store and upload vector_store = openai_client.vector_stores.create(name="SalesDataStore") print(f"Vector store created (id: {vector_store.id})") - + from io import BytesIO - txt_file = BytesIO(txt_content.encode('utf-8')) + + txt_file = BytesIO(txt_content.encode("utf-8")) txt_file.name = "sales_data.txt" - + file = openai_client.vector_stores.files.upload_and_poll( vector_store_id=vector_store.id, file=txt_file, @@ -108,31 +109,31 @@ def test_data_analysis_workflow(self, **kwargs): # Request analysis print("\nAsking agent to analyze the sales data...") - + response = openai_client.responses.create( input="Analyze the sales data and calculate the total revenue for each product. Then save the results.", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + print(f"Initial response completed (id: {response.id})") - + # Check if function was called function_calls_found = 0 input_list: ResponseInputParam = [] - + for item in response.output: if item.type == "function_call": function_calls_found += 1 print(f"Function call detected (id: {item.call_id}, name: {item.name})") - + assert item.name == "save_analysis_results" - + arguments = json.loads(item.arguments) print(f"Function arguments: {arguments}") - + assert "summary" in arguments assert len(arguments["summary"]) > 20 - + input_list.append( FunctionCallOutput( type="function_call_output", @@ -142,7 +143,7 @@ def test_data_analysis_workflow(self, **kwargs): ) assert function_calls_found > 0, "Expected save_analysis_results function to be called" - + # Send function results back if input_list: response = openai_client.responses.create( @@ -214,23 +215,23 @@ def test_empty_vector_store_handling(self, **kwargs): # Request analysis of non-existent file print("\nAsking agent to find non-existent data...") - + response = openai_client.responses.create( input="Find and analyze the quarterly sales report.", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_text = response.output_text print(f"Response: '{response_text[:200] if response_text else '(empty)'}...'") - + # Verify agent didn't crash assert response.id is not None, "Agent should return a valid response" assert len(response.output) >= 0, "Agent should return output items" - + # If there's text, it should be meaningful if response_text: assert len(response_text) > 10, "Non-empty response should be meaningful" - + print("\nโœ“ Agent handled missing data gracefully") # Teardown @@ -267,11 +268,12 @@ def calculate_sum(numbers): # Create vector store and upload vector_store = openai_client.vector_stores.create(name="CodeStore") - + from io import BytesIO - code_file = BytesIO(python_code.encode('utf-8')) + + code_file = BytesIO(python_code.encode("utf-8")) code_file.name = "sample_code.py" - + file = openai_client.vector_stores.files.upload_and_poll( vector_store_id=vector_store.id, file=code_file, @@ -313,22 +315,23 @@ def calculate_sum(numbers): # Request code analysis print("\nAsking agent to find and analyze the Python code...") - + response = openai_client.responses.create( input="Find the Python code file and tell me what the calculate_sum function does.", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_text = response.output_text print(f"Response: {response_text[:300]}...") - + # Verify agent found and analyzed the code assert len(response_text) > 50, "Expected detailed analysis" - + response_lower = response_text.lower() - assert any(keyword in response_lower for keyword in ["sum", "calculate", "function", "numbers", "code", "python"]), \ - "Expected response to discuss the code" - + assert any( + keyword in response_lower for keyword in ["sum", "calculate", "function", "numbers", "code", "python"] + ), "Expected response to discuss the code" + print("\nโœ“ Agent successfully found code file using File Search") # Teardown @@ -344,7 +347,7 @@ def calculate_sum(numbers): def test_multi_turn_search_and_save_workflow(self, **kwargs): """ Test multi-turn workflow: search documents, ask follow-ups, save findings. - + This tests: - File Search across multiple turns - Function calls interspersed with searches @@ -383,13 +386,14 @@ def test_multi_turn_search_and_save_workflow(self, **kwargs): # Create vector store and upload documents vector_store = openai_client.vector_stores.create(name="ResearchStore") print(f"Vector store created: {vector_store.id}") - + from io import BytesIO - file1 = BytesIO(doc1_content.encode('utf-8')) + + file1 = BytesIO(doc1_content.encode("utf-8")) file1.name = "ml_healthcare.txt" - file2 = BytesIO(doc2_content.encode('utf-8')) + file2 = BytesIO(doc2_content.encode("utf-8")) file2.name = "ai_ethics.txt" - + uploaded1 = openai_client.vector_stores.files.upload_and_poll( vector_store_id=vector_store.id, file=file1, @@ -437,11 +441,11 @@ def test_multi_turn_search_and_save_workflow(self, **kwargs): input="What does the research say about machine learning in healthcare?", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_1_text = response_1.output_text print(f"Response 1: {response_1_text[:200]}...") assert "95" in response_1_text or "accuracy" in response_1_text.lower() - + # Turn 2: Follow-up for specifics print("\n--- Turn 2: Follow-up for details ---") response_2 = openai_client.responses.create( @@ -449,12 +453,12 @@ def test_multi_turn_search_and_save_workflow(self, **kwargs): previous_response_id=response_1.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_2_text = response_2.output_text print(f"Response 2: {response_2_text[:200]}...") response_2_lower = response_2_text.lower() assert any(keyword in response_2_lower for keyword in ["imaging", "drug", "risk", "prediction"]) - + # Turn 3: Save the finding print("\n--- Turn 3: Save finding ---") response_3 = openai_client.responses.create( @@ -462,7 +466,7 @@ def test_multi_turn_search_and_save_workflow(self, **kwargs): previous_response_id=response_2.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + # Handle function call input_list: ResponseInputParam = [] function_called = False @@ -471,12 +475,12 @@ def test_multi_turn_search_and_save_workflow(self, **kwargs): function_called = True print(f"Function called: {item.name} with args: {item.arguments}") assert item.name == "save_finding" - + args = json.loads(item.arguments) assert "topic" in args and "finding" in args print(f" Topic: {args['topic']}") print(f" Finding: {args['finding'][:100]}...") - + input_list.append( FunctionCallOutput( type="function_call_output", @@ -484,9 +488,9 @@ def test_multi_turn_search_and_save_workflow(self, **kwargs): output=json.dumps({"status": "saved", "id": "finding_001"}), ) ) - + assert function_called, "Expected save_finding to be called" - + # Send function result response_3 = openai_client.responses.create( input=input_list, @@ -494,7 +498,7 @@ def test_multi_turn_search_and_save_workflow(self, **kwargs): extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) print(f"Response 3: {response_3.output_text[:150]}...") - + # Turn 4: Switch to different topic (AI ethics) print("\n--- Turn 4: New search topic ---") response_4 = openai_client.responses.create( @@ -502,7 +506,7 @@ def test_multi_turn_search_and_save_workflow(self, **kwargs): previous_response_id=response_3.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_4_text = response_4.output_text print(f"Response 4: {response_4_text[:200]}...") response_4_lower = response_4_text.lower() diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py index 1648b121b34a..c9e069268935 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py @@ -17,7 +17,13 @@ import pytest from test_base import TestBase, servicePreparer from devtools_testutils import is_live_and_not_recording -from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool, CodeInterpreterTool, CodeInterpreterToolAuto, FunctionTool +from azure.ai.projects.models import ( + PromptAgentDefinition, + FileSearchTool, + CodeInterpreterTool, + CodeInterpreterToolAuto, + FunctionTool, +) from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam @@ -43,11 +49,12 @@ def test_complete_analysis_workflow(self, **kwargs): # Create data file txt_content = "Sample data for analysis" vector_store = openai_client.vector_stores.create(name="ThreeToolStore") - + from io import BytesIO - txt_file = BytesIO(txt_content.encode('utf-8')) + + txt_file = BytesIO(txt_content.encode("utf-8")) txt_file.name = "data.txt" - + file = openai_client.vector_stores.files.upload_and_poll( vector_store_id=vector_store.id, file=txt_file, @@ -84,17 +91,17 @@ def test_complete_analysis_workflow(self, **kwargs): description="Agent using File Search, Code Interpreter, and Function Tool.", ) print(f"Agent created (id: {agent.id})") - + # Use the agent response = openai_client.responses.create( input="Find the data file, analyze it, and save the results.", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) print(f"Response received (id: {response.id})") - + assert response.id is not None print("โœ“ Three-tool combination works!") - + # Cleanup project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) openai_client.vector_stores.delete(vector_store.id) @@ -118,11 +125,12 @@ def test_four_tools_combination(self, **kwargs): # Create vector store txt_content = "Test data" vector_store = openai_client.vector_stores.create(name="FourToolStore") - + from io import BytesIO - txt_file = BytesIO(txt_content.encode('utf-8')) + + txt_file = BytesIO(txt_content.encode("utf-8")) txt_file.name = "data.txt" - + file = openai_client.vector_stores.files.upload_and_poll( vector_store_id=vector_store.id, file=txt_file, @@ -142,7 +150,7 @@ def test_four_tools_combination(self, **kwargs): }, strict=True, ) - + func_tool_2 = FunctionTool( name="log_action", description="Log an action", @@ -173,10 +181,10 @@ def test_four_tools_combination(self, **kwargs): description="Agent with 4 tools.", ) print(f"Agent with 4 tools created (id: {agent.id})") - + assert agent.id is not None print("โœ“ 4 tools works!") - + # Cleanup project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) openai_client.vector_stores.delete(vector_store.id) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py index d4d1fc68faa9..fc9b416dc7c0 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py @@ -32,7 +32,7 @@ class TestMultiToolWithConversations(TestBase): def test_file_search_and_function_with_conversation(self, **kwargs): """ Test using multiple tools (FileSearch + Function) within one conversation. - + This tests: - Mixing FileSearch (server-side) and Function (client-side) tools in same conversation - Complex multi-turn workflow with different tool types @@ -58,7 +58,8 @@ def test_file_search_and_function_with_conversation(self, **kwargs): vector_store = openai_client.vector_stores.create(name="SalesDataStore") from io import BytesIO - file = BytesIO(doc_content.encode('utf-8')) + + file = BytesIO(doc_content.encode("utf-8")) file.name = "sales.txt" openai_client.vector_stores.files.upload_and_poll(vector_store_id=vector_store.id, file=file) print(f"Vector store created: {vector_store.id}") @@ -136,7 +137,7 @@ def test_file_search_and_function_with_conversation(self, **kwargs): args = json.loads(item.arguments) print(f" Title: {args['title']}") print(f" Summary: {args['summary'][:100]}...") - + input_list.append( FunctionCallOutput( type="function_call_output", @@ -161,18 +162,18 @@ def test_file_search_and_function_with_conversation(self, **kwargs): print("\n--- Verifying multi-tool conversation state ---") all_items = list(openai_client.conversations.items.list(conversation.id)) print(f"Total conversation items: {len(all_items)}") - + # Count different item types user_messages = sum(1 for item in all_items if item.type == "message" and item.role == "user") assistant_messages = sum(1 for item in all_items if item.type == "message" and item.role == "assistant") function_calls = sum(1 for item in all_items if item.type == "function_call") function_outputs = sum(1 for item in all_items if item.type == "function_call_output") - + print(f" User messages: {user_messages}") print(f" Assistant messages: {assistant_messages}") print(f" Function calls: {function_calls}") print(f" Function outputs: {function_outputs}") - + # Print item sequence to show tool interleaving print("\n Conversation item sequence:") for i, item in enumerate(all_items, 1): @@ -181,13 +182,13 @@ def test_file_search_and_function_with_conversation(self, **kwargs): print(f" {i}. {item.type} ({item.role}): {content_preview}...") else: print(f" {i}. {item.type}") - + # Verify we have items from all three turns assert user_messages >= 3, "Expected at least 3 user messages (three turns)" assert assistant_messages >= 3, "Expected assistant responses for each turn" assert function_calls >= 1, "Expected at least 1 function call (save_report)" assert function_outputs >= 1, "Expected at least 1 function output" - + print("\nโœ“ Multi-tool conversation state verified") print(" - Both server-side (FileSearch) and client-side (Function) tools tracked") print(" - All 3 turns preserved in conversation") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py index c44a6ccb54cd..f4adb0bb9afc 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py @@ -53,7 +53,7 @@ def test_agent_ai_search_question_answering(self, **kwargs): """ Test agent with Azure AI Search capabilities for question answering. - NOTE: This test is skipped in favor of the parallel async version which is + NOTE: This test is skipped in favor of the parallel async version which is significantly faster (~3x) and provides the same coverage. See test_agent_ai_search_async.py::test_agent_ai_search_question_answering_async_parallel @@ -201,7 +201,9 @@ def test_agent_ai_search_question_answering(self, **kwargs): f"but got {correct_answers}. The agent needs to answer at least 80% correctly." ) - print(f"\nโœ“ Test passed! Agent answered {correct_answers}/{total_questions} questions correctly (>= 4 required)") + print( + f"\nโœ“ Test passed! Agent answered {correct_answers}/{total_questions} questions correctly (>= 4 required)" + ) # Teardown project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py index 1f4f498d30fe..83794b97894d 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py @@ -49,7 +49,16 @@ class TestAgentAISearchAsync(TestBase): }, ] - async def _ask_question_async(self, openai_client, agent_name: str, title: str, question: str, expected_answer: bool, question_num: int, total_questions: int): + async def _ask_question_async( + self, + openai_client, + agent_name: str, + title: str, + question: str, + expected_answer: bool, + question_num: int, + total_questions: int, + ): """Helper method to ask a single question asynchronously.""" print(f"\n{'='*80}") print(f"Q{question_num}/{total_questions}: {title}") @@ -117,7 +126,7 @@ async def test_agent_ai_search_question_answering_async_parallel(self, **kwargs) and handle multiple concurrent requests to search indexed content and provide accurate answers to questions based on the search results. - The test asks 5 true/false questions IN PARALLEL using asyncio.gather() and + The test asks 5 true/false questions IN PARALLEL using asyncio.gather() and validates that at least 4 are answered correctly by the agent using the search index. This should be significantly faster than the sequential version. @@ -140,7 +149,7 @@ async def test_agent_ai_search_question_answering_async_parallel(self, **kwargs) # Setup project_client = self.create_async_client(operation_group="agents", **kwargs) - + async with project_client: openai_client = await project_client.get_openai_client() @@ -197,21 +206,15 @@ async def test_agent_ai_search_question_answering_async_parallel(self, **kwargs) title = qa_pair["title"] question = qa_pair["question"] expected_answer = qa_pair["answer"] - + task = self._ask_question_async( - openai_client, - agent.name, - title, - question, - expected_answer, - i, - total_questions + openai_client, agent.name, title, question, expected_answer, i, total_questions ) tasks.append(task) # Run all tasks in parallel and collect results results = await asyncio.gather(*tasks) - + # Count correct answers correct_answers = sum(1 for is_correct in results if is_correct) @@ -226,7 +229,9 @@ async def test_agent_ai_search_question_answering_async_parallel(self, **kwargs) f"but got {correct_answers}. The agent needs to answer at least 80% correctly." ) - print(f"\nโœ“ Test passed! Agent answered {correct_answers}/{total_questions} questions correctly (>= 4 required)") + print( + f"\nโœ“ Test passed! Agent answered {correct_answers}/{total_questions} questions correctly (>= 4 required)" + ) # Teardown await project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py index 109a0aac716a..3e9f81c32660 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py @@ -54,10 +54,10 @@ def test_agent_bing_grounding(self, **kwargs): # Note: This test requires BING_PROJECT_CONNECTION_ID environment variable # to be set with a valid Bing connection ID from the project bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_connection_id") - + if not bing_connection_id: pytest.skip("BING_PROJECT_CONNECTION_ID environment variable not set") - + assert isinstance(bing_connection_id, str), "bing_connection_id must be a string" # Create agent with Bing grounding tool @@ -70,9 +70,7 @@ def test_agent_bing_grounding(self, **kwargs): BingGroundingAgentTool( bing_grounding=BingGroundingSearchToolParameters( search_configurations=[ - BingGroundingSearchConfiguration( - project_connection_id=bing_connection_id - ) + BingGroundingSearchConfiguration(project_connection_id=bing_connection_id) ] ) ) @@ -88,7 +86,7 @@ def test_agent_bing_grounding(self, **kwargs): # Test agent with a query that requires current web information output_text = "" url_citations = [] - + stream_response = openai_client.responses.create( stream=True, tool_choice="required", @@ -121,14 +119,14 @@ def test_agent_bing_grounding(self, **kwargs): # Verify that we got a response assert len(output_text) > 0, "Expected non-empty response text" - + # Verify that we got URL citations (Bing grounding should provide sources) assert len(url_citations) > 0, "Expected URL citations from Bing grounding" - + # Verify that citations are valid URLs for url in url_citations: assert url.startswith("http://") or url.startswith("https://"), f"Invalid URL citation: {url}" - + print(f"Test completed successfully with {len(url_citations)} URL citations") # Teardown @@ -155,10 +153,10 @@ def test_agent_bing_grounding_multiple_queries(self, **kwargs): openai_client = project_client.get_openai_client() bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_connection_id") - + if not bing_connection_id: pytest.skip("BING_PROJECT_CONNECTION_ID environment variable not set") - + assert isinstance(bing_connection_id, str), "bing_connection_id must be a string" # Create agent with Bing grounding tool @@ -171,9 +169,7 @@ def test_agent_bing_grounding_multiple_queries(self, **kwargs): BingGroundingAgentTool( bing_grounding=BingGroundingSearchToolParameters( search_configurations=[ - BingGroundingSearchConfiguration( - project_connection_id=bing_connection_id - ) + BingGroundingSearchConfiguration(project_connection_id=bing_connection_id) ] ) ) @@ -193,7 +189,7 @@ def test_agent_bing_grounding_multiple_queries(self, **kwargs): print(f"\nTesting query: {query}") output_text = "" url_citations = [] - + stream_response = openai_client.responses.create( stream=True, tool_choice="required", diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py index 17b864dc3820..a9e3daff2605 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py @@ -69,12 +69,12 @@ def test_agent_code_interpreter_simple_math(self, **kwargs): # Problem: Calculate the sum of cubes from 1 to 50, then add 12!/(8!) # Expected answer: 1637505 print("\nAsking agent to calculate: sum of cubes from 1 to 50, plus 12!/(8!)") - + response = openai_client.responses.create( input="Calculate this using Python: First, find the sum of cubes from 1 to 50 (1ยณ + 2ยณ + ... + 50ยณ). Then add 12 factorial divided by 8 factorial (12!/8!). What is the final result?", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + print(f"Response completed (id: {response.id})") assert response.id is not None assert response.output is not None @@ -83,20 +83,20 @@ def test_agent_code_interpreter_simple_math(self, **kwargs): # Get the response text last_message = response.output[-1] response_text = "" - + if last_message.type == "message": for content_item in last_message.content: if content_item.type == "output_text": response_text += content_item.text - + print(f"Agent's response: {response_text}") - + # Verify the response contains the correct answer (1637505) # Note: sum of cubes 1-50 = 1,625,625; 12!/8! = 11,880; total = 1,637,505 - assert "1637505" in response_text or "1,637,505" in response_text, ( - f"Expected answer 1637505 to be in response, but got: {response_text}" - ) - + assert ( + "1637505" in response_text or "1,637,505" in response_text + ), f"Expected answer 1637505 to be in response, but got: {response_text}" + print("โœ“ Code interpreter successfully executed Python code and returned correct answer") # Teardown @@ -142,16 +142,18 @@ def test_agent_code_interpreter_file_generation(self, **kwargs): # Get the path to the test CSV file asset_file_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/synthetic_500_quarterly_results.csv") + os.path.join( + os.path.dirname(__file__), "../../../samples/agents/assets/synthetic_500_quarterly_results.csv" + ) ) - + assert os.path.exists(asset_file_path), f"Test CSV file not found at: {asset_file_path}" print(f"Using test CSV file: {asset_file_path}") # Upload the CSV file with open(asset_file_path, "rb") as f: file = openai_client.files.create(purpose="assistants", file=f) - + print(f"File uploaded (id: {file.id})") assert file.id is not None @@ -172,12 +174,12 @@ def test_agent_code_interpreter_file_generation(self, **kwargs): # Ask the agent to create a chart from the CSV print("\nAsking agent to create a bar chart...") - + response = openai_client.responses.create( input="Create a bar chart showing operating profit by sector from the uploaded CSV file. Save it as a PNG file.", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + print(f"Response completed (id: {response.id})") assert response.id is not None assert response.output is not None @@ -205,29 +207,29 @@ def test_agent_code_interpreter_file_generation(self, **kwargs): assert file_id, "Expected a file to be generated but no file ID found in response" assert filename, "Expected a filename but none found in response" assert container_id, "Expected a container ID but none found in response" - + print(f"โœ“ File generated successfully: {filename}") # Download the generated file print(f"Downloading file {filename}...") file_content = openai_client.containers.files.content.retrieve(file_id=file_id, container_id=container_id) - + # Read the content content_bytes = file_content.read() assert len(content_bytes) > 0, "Expected file content but got empty bytes" - + print(f"โœ“ File downloaded successfully ({len(content_bytes)} bytes)") - + # Verify it's a PNG file (check magic bytes) - if filename.endswith('.png'): + if filename.endswith(".png"): # PNG files start with: 89 50 4E 47 (โ€ฐPNG) - assert content_bytes[:4] == b'\x89PNG', "File does not appear to be a valid PNG" + assert content_bytes[:4] == b"\x89PNG", "File does not appear to be a valid PNG" print("โœ“ File is a valid PNG image") # Teardown print("\nCleaning up...") project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) print("Agent deleted") - + openai_client.files.delete(file.id) print("Uploaded file deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py index 4e34ed74153c..13ceb8fc6695 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py @@ -54,7 +54,7 @@ def test_agent_file_search(self, **kwargs): asset_file_path = os.path.abspath( os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/product_info.md") ) - + assert os.path.exists(asset_file_path), f"Test file not found at: {asset_file_path}" print(f"Using test file: {asset_file_path}") @@ -69,7 +69,7 @@ def test_agent_file_search(self, **kwargs): vector_store_id=vector_store.id, file=f, ) - + print(f"File uploaded (id: {file.id}, status: {file.status})") assert file.id is not None assert file.status == "completed", f"Expected file status 'completed', got '{file.status}'" @@ -91,12 +91,12 @@ def test_agent_file_search(self, **kwargs): # Ask a question about the uploaded document print("\nAsking agent about the product information...") - + response = openai_client.responses.create( input="What products are mentioned in the document?", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + print(f"Response completed (id: {response.id})") assert response.id is not None assert response.output is not None @@ -105,10 +105,10 @@ def test_agent_file_search(self, **kwargs): # Get the response text response_text = response.output_text print(f"\nAgent's response: {response_text[:300]}...") - + # Verify we got a meaningful response assert len(response_text) > 50, "Expected a substantial response from the agent" - + # The response should mention finding information (indicating file search was used) # We can't assert exact product names without knowing the file content, # but we can verify the agent provided an answer @@ -118,7 +118,7 @@ def test_agent_file_search(self, **kwargs): print("\nCleaning up...") project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) print("Agent deleted") - + openai_client.vector_stores.delete(vector_store.id) print("Vector store deleted") @@ -130,12 +130,12 @@ def test_agent_file_search(self, **kwargs): def test_agent_file_search_unsupported_file_type(self, **kwargs): """ Negative test: Verify that unsupported file types are rejected with clear error messages. - + This test validates that: 1. CSV files (unsupported format) are rejected 2. The error message clearly indicates the file type is not supported 3. The error message lists supported file types - + This ensures good developer experience by providing actionable error messages. """ @@ -146,18 +146,19 @@ def test_agent_file_search_unsupported_file_type(self, **kwargs): # Create vector store vector_store = openai_client.vector_stores.create(name="UnsupportedFileTestStore") print(f"Vector store created (id: {vector_store.id})") - + # Create CSV file (unsupported format) csv_content = """product,quarter,revenue Widget A,Q1,15000 Widget B,Q1,22000 Widget A,Q2,18000 Widget B,Q2,25000""" - + from io import BytesIO - csv_file = BytesIO(csv_content.encode('utf-8')) + + csv_file = BytesIO(csv_content.encode("utf-8")) csv_file.name = "sales_data.csv" - + # Attempt to upload unsupported file type print("\nAttempting to upload CSV file (unsupported format)...") try: @@ -168,32 +169,30 @@ def test_agent_file_search_unsupported_file_type(self, **kwargs): # If we get here, the test should fail openai_client.vector_stores.delete(vector_store.id) pytest.fail("Expected BadRequestError for CSV file upload, but upload succeeded") - + except Exception as e: error_message = str(e) print(f"\nโœ“ Upload correctly rejected with error: {error_message[:200]}...") - + # Verify error message quality - assert "400" in error_message or "BadRequestError" in type(e).__name__, \ - "Should be a 400 Bad Request error" - - assert ".csv" in error_message.lower(), \ - "Error message should mention the CSV file extension" - - assert "not supported" in error_message.lower() or "unsupported" in error_message.lower(), \ - "Error message should clearly state the file type is not supported" - + assert "400" in error_message or "BadRequestError" in type(e).__name__, "Should be a 400 Bad Request error" + + assert ".csv" in error_message.lower(), "Error message should mention the CSV file extension" + + assert ( + "not supported" in error_message.lower() or "unsupported" in error_message.lower() + ), "Error message should clearly state the file type is not supported" + # Check that supported file types are mentioned (helpful for developers) error_lower = error_message.lower() has_supported_list = any(ext in error_lower for ext in [".txt", ".pdf", ".md", ".py"]) - assert has_supported_list, \ - "Error message should list examples of supported file types" - + assert has_supported_list, "Error message should list examples of supported file types" + print("โœ“ Error message is clear and actionable") print(" - Mentions unsupported file type (.csv)") print(" - States it's not supported") print(" - Lists supported file types") - + # Cleanup openai_client.vector_stores.delete(vector_store.id) print("\nVector store deleted") @@ -206,7 +205,7 @@ def test_agent_file_search_unsupported_file_type(self, **kwargs): def test_agent_file_search_multi_turn_conversation(self, **kwargs): """ Test multi-turn conversation with File Search. - + This test verifies that an agent can maintain context across multiple turns while using File Search to answer follow-up questions. """ @@ -242,11 +241,12 @@ def test_agent_file_search_multi_turn_conversation(self, **kwargs): # Create vector store and upload document vector_store = openai_client.vector_stores.create(name="ProductCatalog") print(f"Vector store created: {vector_store.id}") - + from io import BytesIO - product_file = BytesIO(product_info.encode('utf-8')) + + product_file = BytesIO(product_info.encode("utf-8")) product_file.name = "products.txt" - + file = openai_client.vector_stores.files.upload_and_poll( vector_store_id=vector_store.id, file=product_file, @@ -271,12 +271,11 @@ def test_agent_file_search_multi_turn_conversation(self, **kwargs): input="What is the price of Widget B?", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_1_text = response_1.output_text print(f"Response 1: {response_1_text[:200]}...") - assert "$220" in response_1_text or "220" in response_1_text, \ - "Response should mention Widget B's price" - + assert "$220" in response_1_text or "220" in response_1_text, "Response should mention Widget B's price" + # Turn 2: Follow-up question (requires context from turn 1) print("\n--- Turn 2: Follow-up query (testing context retention) ---") response_2 = openai_client.responses.create( @@ -284,12 +283,13 @@ def test_agent_file_search_multi_turn_conversation(self, **kwargs): previous_response_id=response_1.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_2_text = response_2.output_text print(f"Response 2: {response_2_text[:200]}...") - assert "30" in response_2_text or "thirty" in response_2_text.lower(), \ - "Response should mention Widget B's stock (30 units)" - + assert ( + "30" in response_2_text or "thirty" in response_2_text.lower() + ), "Response should mention Widget B's stock (30 units)" + # Turn 3: Another follow-up (compare with different product) print("\n--- Turn 3: Comparison query ---") response_3 = openai_client.responses.create( @@ -297,12 +297,13 @@ def test_agent_file_search_multi_turn_conversation(self, **kwargs): previous_response_id=response_2.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_3_text = response_3.output_text print(f"Response 3: {response_3_text[:200]}...") - assert "50" in response_3_text or "fifty" in response_3_text.lower(), \ - "Response should mention Widget A's stock (50 units)" - + assert ( + "50" in response_3_text or "fifty" in response_3_text.lower() + ), "Response should mention Widget A's stock (50 units)" + # Turn 4: New topic (testing topic switching) print("\n--- Turn 4: Topic switch ---") response_4 = openai_client.responses.create( @@ -310,11 +311,12 @@ def test_agent_file_search_multi_turn_conversation(self, **kwargs): previous_response_id=response_3.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_4_text = response_4.output_text print(f"Response 4: {response_4_text[:200]}...") - assert "widget b" in response_4_text.lower() or "4.8" in response_4_text, \ - "Response should identify Widget B as highest rated (4.8/5)" + assert ( + "widget b" in response_4_text.lower() or "4.8" in response_4_text + ), "Response should identify Widget B as highest rated (4.8/5)" print("\nโœ“ Multi-turn conversation successful!") print(" - Context maintained across turns") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py index de457e51809d..04b4ced316dc 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py @@ -55,7 +55,7 @@ def test_agent_file_search_stream(self, **kwargs): asset_file_path = os.path.abspath( os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/product_info.md") ) - + assert os.path.exists(asset_file_path), f"Test file not found at: {asset_file_path}" print(f"Using test file: {asset_file_path}") @@ -70,7 +70,7 @@ def test_agent_file_search_stream(self, **kwargs): vector_store_id=vector_store.id, file=f, ) - + print(f"File uploaded (id: {file.id}, status: {file.status})") assert file.id is not None assert file.status == "completed", f"Expected file status 'completed', got '{file.status}'" @@ -92,46 +92,46 @@ def test_agent_file_search_stream(self, **kwargs): # Ask a question with streaming enabled print("\nAsking agent about the product information (streaming)...") - + stream_response = openai_client.responses.create( stream=True, input="What products are mentioned in the document? Please provide a brief summary.", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + # Collect streamed response response_text = "" response_id = None events_received = 0 - + for event in stream_response: events_received += 1 - + if event.type == "response.output_item.done": if event.item.type == "message": for content_item in event.item.content: if content_item.type == "output_text": response_text += content_item.text - + elif event.type == "response.completed": response_id = event.response.id # Could also use event.response.output_text - + print(f"\nStreaming completed (id: {response_id}, events: {events_received})") assert response_id is not None, "Expected response ID from stream" assert events_received > 0, "Expected to receive stream events" - + print(f"Agent's streamed response: {response_text[:300]}...") - + # Verify we got a meaningful response assert len(response_text) > 50, "Expected a substantial response from the agent" - + print("\nโœ“ Agent successfully streamed responses using file search tool") # Teardown print("\nCleaning up...") project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) print("Agent deleted") - + openai_client.vector_stores.delete(vector_store.id) print("Vector store deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py index 38116ca09209..6aeda3aaa5f5 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py @@ -78,14 +78,16 @@ def test_agent_function_tool(self, **kwargs): ), description="Agent for testing function tool capabilities.", ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version}, model: {agent.definition['model']})") + print( + f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version}, model: {agent.definition['model']})" + ) assert agent.id is not None assert agent.name == "function-tool-agent" assert agent.version is not None # Ask a question that should trigger the function call print("\nAsking agent: What's the weather in Seattle?") - + response = openai_client.responses.create( input="What's the weather in Seattle?", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, @@ -103,16 +105,18 @@ def test_agent_function_tool(self, **kwargs): if item.type == "function_call": function_calls_found += 1 print(f"Found function call (id: {item.call_id}, name: {item.name})") - + # Parse the arguments arguments = json.loads(item.arguments) print(f"Function arguments: {arguments}") - + # Verify the function call is for get_weather assert item.name == "get_weather", f"Expected function name 'get_weather', got '{item.name}'" assert "location" in arguments, "Expected 'location' in function arguments" - assert "seattle" in arguments["location"].lower(), f"Expected Seattle in location, got {arguments['location']}" - + assert ( + "seattle" in arguments["location"].lower() + ), f"Expected Seattle in location, got {arguments['location']}" + # Simulate the function execution and provide a result weather_result = { "location": arguments["location"], @@ -120,7 +124,7 @@ def test_agent_function_tool(self, **kwargs): "condition": "Sunny", "humidity": "45%", } - + # Add the function result to the input list input_list.append( FunctionCallOutput( @@ -137,7 +141,7 @@ def test_agent_function_tool(self, **kwargs): # Send the function results back to get the final response print("\nSending function results back to agent...") - + response = openai_client.responses.create( input=input_list, previous_response_id=response.id, @@ -146,20 +150,20 @@ def test_agent_function_tool(self, **kwargs): print(f"Final response completed (id: {response.id})") assert response.id is not None - + # Get the final response text response_text = response.output_text print(f"\nAgent's final response: {response_text}") - + # Verify the response incorporates the weather data assert len(response_text) > 20, "Expected a meaningful response from the agent" - + # Check that the response mentions the weather information we provided response_lower = response_text.lower() - assert any(keyword in response_lower for keyword in ["72", "sunny", "weather", "seattle"]), ( - f"Expected response to mention weather information, but got: {response_text}" - ) - + assert any( + keyword in response_lower for keyword in ["72", "sunny", "weather", "seattle"] + ), f"Expected response to mention weather information, but got: {response_text}" + print("\nโœ“ Agent successfully used function tool and incorporated results into response") # Teardown @@ -174,7 +178,7 @@ def test_agent_function_tool(self, **kwargs): def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): """ Test multi-turn conversation where agent calls functions multiple times. - + This tests: - Multiple function calls across different turns - Context retention between turns @@ -204,7 +208,7 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): }, strict=True, ) - + get_temperature_forecast = FunctionTool( name="get_temperature_forecast", description="Get 3-day temperature forecast for a city", @@ -240,14 +244,14 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): input="What's the weather in New York?", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + # Handle function call input_list: ResponseInputParam = [] for item in response_1.output: if item.type == "function_call": print(f"Function called: {item.name} with args: {item.arguments}") assert item.name == "get_weather" - + # Simulate weather API response weather_data = {"temperature": 68, "condition": "Cloudy", "humidity": 65} input_list.append( @@ -257,18 +261,18 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): output=json.dumps(weather_data), ) ) - + # Get response with function results response_1 = openai_client.responses.create( input=input_list, previous_response_id=response_1.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_1_text = response_1.output_text print(f"Response 1: {response_1_text[:200]}...") assert "68" in response_1_text or "cloudy" in response_1_text.lower() - + # Turn 2: Follow-up with forecast (requires context) print("\n--- Turn 2: Follow-up forecast query ---") response_2 = openai_client.responses.create( @@ -276,26 +280,26 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): previous_response_id=response_1.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + # Handle forecast function call input_list = [] for item in response_2.output: if item.type == "function_call": print(f"Function called: {item.name} with args: {item.arguments}") assert item.name == "get_temperature_forecast" - + # Agent should remember we're talking about New York args = json.loads(item.arguments) assert "new york" in args["city"].lower() - + # Simulate forecast API response forecast_data = { "city": "New York", "forecast": [ {"day": "Tomorrow", "temp": 70}, {"day": "Day 2", "temp": 72}, - {"day": "Day 3", "temp": 69} - ] + {"day": "Day 3", "temp": 69}, + ], } input_list.append( FunctionCallOutput( @@ -304,18 +308,18 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): output=json.dumps(forecast_data), ) ) - + # Get response with forecast response_2 = openai_client.responses.create( input=input_list, previous_response_id=response_2.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_2_text = response_2.output_text print(f"Response 2: {response_2_text[:200]}...") assert "70" in response_2_text or "72" in response_2_text - + # Turn 3: Compare with another city print("\n--- Turn 3: New city query ---") response_3 = openai_client.responses.create( @@ -323,7 +327,7 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): previous_response_id=response_2.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + # Handle function calls for Seattle (agent might call both weather and forecast) input_list = [] for item in response_3.output: @@ -331,7 +335,7 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): print(f"Function called: {item.name} with args: {item.arguments}") args = json.loads(item.arguments) assert "seattle" in args["city"].lower() - + # Handle based on function name if item.name == "get_weather": weather_data = {"temperature": 58, "condition": "Rainy", "humidity": 80} @@ -348,8 +352,8 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): "forecast": [ {"day": "Tomorrow", "temp": 56}, {"day": "Day 2", "temp": 59}, - {"day": "Day 3", "temp": 57} - ] + {"day": "Day 3", "temp": 57}, + ], } input_list.append( FunctionCallOutput( @@ -358,14 +362,14 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): output=json.dumps(forecast_data), ) ) - + # Get final comparison response response_3 = openai_client.responses.create( input=input_list, previous_response_id=response_3.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_3_text = response_3.output_text print(f"Response 3: {response_3_text[:200]}...") # Agent should mention Seattle weather (either 58 for current or comparison) @@ -388,7 +392,7 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): def test_agent_function_tool_context_dependent_followup(self, **kwargs): """ Test deeply context-dependent follow-ups (e.g., unit conversion, clarification). - + This tests that the agent truly uses previous response content, not just remembering parameters from the first query. """ @@ -435,7 +439,7 @@ def test_agent_function_tool_context_dependent_followup(self, **kwargs): input="What's the temperature in Boston?", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + # Handle function call input_list: ResponseInputParam = [] for item in response_1.output: @@ -448,17 +452,17 @@ def test_agent_function_tool_context_dependent_followup(self, **kwargs): output=json.dumps({"temperature": 72, "unit": "F"}), ) ) - + response_1 = openai_client.responses.create( input=input_list, previous_response_id=response_1.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_1_text = response_1.output_text print(f"Response 1: {response_1_text[:200]}...") assert "72" in response_1_text, "Should mention 72ยฐF" - + # Turn 2: Context-dependent follow-up (convert the previous number) print("\n--- Turn 2: Context-dependent conversion ---") response_2 = openai_client.responses.create( @@ -466,18 +470,20 @@ def test_agent_function_tool_context_dependent_followup(self, **kwargs): previous_response_id=response_1.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_2_text = response_2.output_text print(f"Response 2: {response_2_text[:200]}...") - + # Should convert 72ยฐF to ~22ยฐC (without calling the function again) # The agent should use the previous response's value response_2_lower = response_2_text.lower() - assert "celsius" in response_2_lower or "ยฐc" in response_2_lower or "c" in response_2_lower, \ - "Response should mention Celsius" - assert any(temp in response_2_text for temp in ["22", "22.2", "22.22", "20", "21", "23"]), \ - f"Response should calculate Celsius from 72ยฐF (~22ยฐC), got: {response_2_text}" - + assert ( + "celsius" in response_2_lower or "ยฐc" in response_2_lower or "c" in response_2_lower + ), "Response should mention Celsius" + assert any( + temp in response_2_text for temp in ["22", "22.2", "22.22", "20", "21", "23"] + ), f"Response should calculate Celsius from 72ยฐF (~22ยฐC), got: {response_2_text}" + # Turn 3: Another context-dependent follow-up (comparison) print("\n--- Turn 3: Compare to another value ---") response_3 = openai_client.responses.create( @@ -485,15 +491,16 @@ def test_agent_function_tool_context_dependent_followup(self, **kwargs): previous_response_id=response_2.id, extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + response_3_text = response_3.output_text print(f"Response 3: {response_3_text[:200]}...") - + # 22ยฐC is colder than 25ยฐC response_3_lower = response_3_text.lower() - assert "colder" in response_3_lower or "cooler" in response_3_lower or "lower" in response_3_lower, \ - f"Response should indicate 22ยฐC is colder than 25ยฐC, got: {response_3_text}" - + assert ( + "colder" in response_3_lower or "cooler" in response_3_lower or "lower" in response_3_lower + ), f"Response should indicate 22ยฐC is colder than 25ยฐC, got: {response_3_text}" + print("\nโœ“ Context-dependent follow-ups successful!") print(" - Agent converted temperature from previous response") print(" - Agent compared values from conversation history") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py index a98feed35bb5..cdc157030c72 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py @@ -43,16 +43,18 @@ def test_agent_image_generation(self, **kwargs): # Teardown: DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() """ - + # Get the image model deployment name from environment variable - image_model_deployment = os.environ.get("AZURE_AI_PROJECTS_TESTS_IMAGE_MODEL_DEPLOYMENT_NAME", "gpt-image-1-mini") - + image_model_deployment = os.environ.get( + "AZURE_AI_PROJECTS_TESTS_IMAGE_MODEL_DEPLOYMENT_NAME", "gpt-image-1-mini" + ) + model = self.test_agents_params["model_deployment_name"] # Setup project_client = self.create_client(operation_group="agents", **kwargs) openai_client = project_client.get_openai_client() - + # Check if the image model deployment exists in the project try: deployment = project_client.deployments.get(image_model_deployment) @@ -61,7 +63,7 @@ def test_agent_image_generation(self, **kwargs): pytest.skip(f"Image generation model '{image_model_deployment}' not available in this project") except Exception as e: pytest.skip(f"Unable to verify image model deployment: {e}") - + # Disable retries for faster failure when service returns 500 openai_client.max_retries = 0 @@ -82,7 +84,7 @@ def test_agent_image_generation(self, **kwargs): # Request image generation print("\nAsking agent to generate an image of a simple geometric shape...") - + response = openai_client.responses.create( input="Generate an image of a blue circle on a white background.", extra_headers={ @@ -90,7 +92,7 @@ def test_agent_image_generation(self, **kwargs): }, # Required for image generation extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + print(f"Response created (id: {response.id})") assert response.id is not None assert response.output is not None @@ -102,7 +104,7 @@ def test_agent_image_generation(self, **kwargs): # Verify image was generated assert len(image_data) > 0, "Expected at least one image to be generated" assert image_data[0], "Expected image data to be non-empty" - + print(f"โœ“ Image data received ({len(image_data[0])} base64 characters)") # Decode the base64 image @@ -116,7 +118,7 @@ def test_agent_image_generation(self, **kwargs): # Verify it's a PNG image (check magic bytes) # PNG files start with: 89 50 4E 47 (โ€ฐPNG) - assert image_bytes[:4] == b'\x89PNG', "Image does not appear to be a valid PNG" + assert image_bytes[:4] == b"\x89PNG", "Image does not appear to be a valid PNG" print("โœ“ Image is a valid PNG") # Verify reasonable image size (should be > 1KB for a 1024x1024 image) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py index ffb19d3be1e5..ef5753773676 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py @@ -85,7 +85,7 @@ def test_agent_mcp_basic(self, **kwargs): # Send initial request that will trigger the MCP tool print("\nAsking agent to summarize Azure REST API specs README...") - + response = openai_client.responses.create( conversation=conversation.id, input="Please summarize the Azure REST API specifications Readme", @@ -100,12 +100,12 @@ def test_agent_mcp_basic(self, **kwargs): # Process any MCP approval requests approval_requests_found = 0 input_list: ResponseInputParam = [] - + for item in response.output: if item.type == "mcp_approval_request": approval_requests_found += 1 print(f"Found MCP approval request (id: {item.id}, server: {item.server_label})") - + if item.server_label == "api-specs" and item.id: # Approve the MCP request input_list.append( @@ -118,15 +118,15 @@ def test_agent_mcp_basic(self, **kwargs): print(f"โœ“ Approved MCP request: {item.id}") # Verify that at least one approval request was generated - assert approval_requests_found > 0, ( - f"Expected at least 1 MCP approval request, but found {approval_requests_found}" - ) - + assert ( + approval_requests_found > 0 + ), f"Expected at least 1 MCP approval request, but found {approval_requests_found}" + print(f"\nโœ“ Processed {approval_requests_found} MCP approval request(s)") # Send the approval response to continue the agent's work print("\nSending approval response to continue agent execution...") - + response = openai_client.responses.create( input=input_list, previous_response_id=response.id, @@ -135,19 +135,19 @@ def test_agent_mcp_basic(self, **kwargs): print(f"Final response completed (id: {response.id})") assert response.id is not None - + # Get the final response text response_text = response.output_text print(f"\nAgent's response preview: {response_text[:200]}...") - + # Verify we got a meaningful response assert len(response_text) > 100, "Expected a substantial response from the agent" - + # Check that the response mentions Azure or REST API (indicating it accessed the repo) - assert any(keyword in response_text.lower() for keyword in ["azure", "rest", "api", "specification"]), ( - f"Expected response to mention Azure/REST API, but got: {response_text[:200]}" - ) - + assert any( + keyword in response_text.lower() for keyword in ["azure", "rest", "api", "specification"] + ), f"Expected response to mention Azure/REST API, but got: {response_text[:200]}" + print("\nโœ“ Agent successfully used MCP tool to access GitHub repo and complete task") # Teardown @@ -232,7 +232,7 @@ def test_agent_mcp_with_project_connection(self, **kwargs): # Send initial request that will trigger the MCP tool with authentication print("\nAsking agent to get GitHub profile username...") - + response = openai_client.responses.create( conversation=conversation.id, input="What is my username in Github profile?", @@ -247,12 +247,12 @@ def test_agent_mcp_with_project_connection(self, **kwargs): # Process any MCP approval requests approval_requests_found = 0 input_list: ResponseInputParam = [] - + for item in response.output: if item.type == "mcp_approval_request": approval_requests_found += 1 print(f"Found MCP approval request (id: {item.id}, server: {item.server_label})") - + if item.server_label == "github-api" and item.id: # Approve the MCP request input_list.append( @@ -265,15 +265,15 @@ def test_agent_mcp_with_project_connection(self, **kwargs): print(f"โœ“ Approved MCP request: {item.id}") # Verify that at least one approval request was generated - assert approval_requests_found > 0, ( - f"Expected at least 1 MCP approval request, but found {approval_requests_found}" - ) - + assert ( + approval_requests_found > 0 + ), f"Expected at least 1 MCP approval request, but found {approval_requests_found}" + print(f"\nโœ“ Processed {approval_requests_found} MCP approval request(s)") # Send the approval response to continue the agent's work print("\nSending approval response to continue agent execution...") - + response = openai_client.responses.create( input=input_list, previous_response_id=response.id, @@ -282,20 +282,20 @@ def test_agent_mcp_with_project_connection(self, **kwargs): print(f"Final response completed (id: {response.id})") assert response.id is not None - + # Get the final response text response_text = response.output_text print(f"\nAgent's response: {response_text}") - + # Verify we got a meaningful response with a GitHub username assert len(response_text) > 5, "Expected a response with a GitHub username" - + # The response should contain some indication of a username or GitHub profile info # We can't assert the exact username, but we can verify it's not an error - assert "error" not in response_text.lower() or "username" in response_text.lower(), ( - f"Expected response to contain GitHub profile info, but got: {response_text}" - ) - + assert ( + "error" not in response_text.lower() or "username" in response_text.lower() + ), f"Expected response to contain GitHub profile info, but got: {response_text}" + print("\nโœ“ Agent successfully used MCP tool with project connection to access authenticated GitHub API") # Teardown diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py index 01f499d4e2aa..cb322e771714 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py @@ -34,7 +34,7 @@ class TestAgentToolsWithConversations(TestBase): def test_function_tool_with_conversation(self, **kwargs): """ Test using FunctionTool within a conversation. - + This tests: - Creating a conversation - Multiple turns with function calls @@ -99,7 +99,7 @@ def test_function_tool_with_conversation(self, **kwargs): if item.type == "function_call": print(f"Function called: {item.name} with {item.arguments}") args = json.loads(item.arguments) - + # Execute calculator result = { "add": args["a"] + args["b"], @@ -107,7 +107,7 @@ def test_function_tool_with_conversation(self, **kwargs): "multiply": args["a"] * args["b"], "divide": args["a"] / args["b"] if args["b"] != 0 else "Error: Division by zero", }[args["operation"]] - + input_list.append( FunctionCallOutput( type="function_call_output", @@ -138,11 +138,11 @@ def test_function_tool_with_conversation(self, **kwargs): if item.type == "function_call": print(f"Function called: {item.name} with {item.arguments}") args = json.loads(item.arguments) - + # Should be multiplying 42 by 2 assert args["operation"] == "multiply" assert args["a"] == 42 or args["b"] == 42 - + result = args["a"] * args["b"] input_list.append( FunctionCallOutput( @@ -168,18 +168,18 @@ def test_function_tool_with_conversation(self, **kwargs): print("\n--- Verifying conversation state ---") all_items = list(openai_client.conversations.items.list(conversation.id)) print(f"Total conversation items: {len(all_items)}") - + # Count different item types user_messages = sum(1 for item in all_items if item.type == "message" and item.role == "user") assistant_messages = sum(1 for item in all_items if item.type == "message" and item.role == "assistant") function_calls = sum(1 for item in all_items if item.type == "function_call") function_outputs = sum(1 for item in all_items if item.type == "function_call_output") - + print(f" User messages: {user_messages}") print(f" Assistant messages: {assistant_messages}") print(f" Function calls: {function_calls}") print(f" Function outputs: {function_outputs}") - + # Verify we have expected items assert user_messages >= 2, "Expected at least 2 user messages (two turns)" assert function_calls >= 2, "Expected at least 2 function calls (one per turn)" @@ -199,7 +199,7 @@ def test_function_tool_with_conversation(self, **kwargs): def test_file_search_with_conversation(self, **kwargs): """ Test using FileSearchTool within a conversation. - + This tests: - Server-side tool execution within conversation - Multiple search queries in same conversation @@ -239,7 +239,8 @@ def test_file_search_with_conversation(self, **kwargs): print(f"Vector store created: {vector_store.id}") from io import BytesIO - file = BytesIO(doc_content.encode('utf-8')) + + file = BytesIO(doc_content.encode("utf-8")) file.name = "products.txt" uploaded = openai_client.vector_stores.files.upload_and_poll( @@ -318,7 +319,7 @@ def test_file_search_with_conversation(self, **kwargs): def test_code_interpreter_with_conversation(self, **kwargs): """ Test using CodeInterpreterTool within a conversation. - + This tests: - Server-side code execution within conversation - Multiple code executions in same conversation diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py index d0ebccafe37b..a0359f7c2cdc 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py @@ -68,12 +68,12 @@ def test_agent_web_search(self, **kwargs): # Ask a question that requires web search for current information print("\nAsking agent about current weather...") - + response = openai_client.responses.create( input="What is the current weather in Seattle?", extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - + print(f"Response completed (id: {response.id})") assert response.id is not None assert response.output is not None @@ -82,16 +82,16 @@ def test_agent_web_search(self, **kwargs): # Get the response text response_text = response.output_text print(f"\nAgent's response: {response_text[:300]}...") - + # Verify we got a meaningful response assert len(response_text) > 30, "Expected a substantial response from the agent" - + # The response should mention weather-related terms or Seattle response_lower = response_text.lower() - assert any(keyword in response_lower for keyword in ["weather", "temperature", "seattle", "forecast"]), ( - f"Expected response to contain weather information, but got: {response_text[:200]}" - ) - + assert any( + keyword in response_lower for keyword in ["weather", "temperature", "seattle", "forecast"] + ), f"Expected response to contain weather information, but got: {response_text[:200]}" + print("\nโœ“ Agent successfully used web search tool to get current information") # Teardown diff --git a/sdk/ai/azure-ai-projects/tests/test_base.py b/sdk/ai/azure-ai-projects/tests/test_base.py index c717192a28e4..4055c1c294d9 100644 --- a/sdk/ai/azure-ai-projects/tests/test_base.py +++ b/sdk/ai/azure-ai-projects/tests/test_base.py @@ -66,7 +66,7 @@ azure_ai_projects_tests_ai_search_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-ai-search-connection", azure_ai_projects_tests_ai_search_index_name="sanitized-index-name", azure_ai_projects_tests_mcp_project_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-mcp-connection", - azure_ai_projects_tests_image_model_deployment_name="gpt-image-1-mini" + azure_ai_projects_tests_image_model_deployment_name="gpt-image-1-mini", ) From 7325d0344b18698a09bee912047526a2c63bcffe Mon Sep 17 00:00:00 2001 From: Paul Batum Date: Fri, 21 Nov 2025 14:05:42 -0800 Subject: [PATCH 5/8] code review feedback --- sdk/ai/azure-ai-projects/.env.template | 6 + .../tools/sample_agent_bing_grounding.py | 2 +- .../tests/agents/tools/README.md | 307 ------------------ .../tests/agents/tools/multitool/__init__.py | 4 - ..._agent_file_search_and_code_interpreter.py | 6 +- .../agents/tools/test_agent_ai_search.py | 6 +- .../tools/test_agent_ai_search_async.py | 8 +- .../agents/tools/test_agent_bing_grounding.py | 10 +- .../tests/agents/tools/test_agent_mcp.py | 2 +- sdk/ai/azure-ai-projects/tests/test_base.py | 4 +- 10 files changed, 24 insertions(+), 331 deletions(-) delete mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/README.md delete mode 100644 sdk/ai/azure-ai-projects/tests/agents/tools/multitool/__init__.py diff --git a/sdk/ai/azure-ai-projects/.env.template b/sdk/ai/azure-ai-projects/.env.template index 4c5e51b6c5d3..ea2b0c167ea6 100644 --- a/sdk/ai/azure-ai-projects/.env.template +++ b/sdk/ai/azure-ai-projects/.env.template @@ -44,6 +44,12 @@ AZURE_AI_PROJECTS_TESTS_CONTAINER_PROJECT_ENDPOINT= AZURE_AI_PROJECTS_TESTS_CONTAINER_APP_RESOURCE_ID= AZURE_AI_PROJECTS_TESTS_CONTAINER_INGRESS_SUBDOMAIN_SUFFIX= +# Connection IDs and settings used in agent tool tests +AZURE_AI_PROJECTS_TESTS_BING_PROJECT_CONNECTION_ID= +AZURE_AI_PROJECTS_TESTS_AI_SEARCH_PROJECT_CONNECTION_ID= +AZURE_AI_PROJECTS_TESTS_AI_SEARCH_INDEX_NAME= +AZURE_AI_PROJECTS_TESTS_MCP_PROJECT_CONNECTION_ID= + # Used in tools BING_PROJECT_CONNECTION_ID= MCP_PROJECT_CONNECTION_ID= diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_bing_grounding.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_bing_grounding.py index 28bb5d47a165..c6110d4d3c79 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_bing_grounding.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_bing_grounding.py @@ -64,7 +64,7 @@ # [END tool_declaration] agent = project_client.agents.create_version( - agent_name="bing-grounding-agent", + agent_name="MyAgent", definition=PromptAgentDefinition( model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"], instructions="You are a helpful assistant.", diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/README.md b/sdk/ai/azure-ai-projects/tests/agents/tools/README.md deleted file mode 100644 index 3a5bc8c62066..000000000000 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/README.md +++ /dev/null @@ -1,307 +0,0 @@ -# Azure AI Agent Tools Tests - -This directory contains comprehensive tests for Azure AI Agents with various tool capabilities. These tests demonstrate how agents can be enhanced with different tools to perform specialized tasks like searching documents, executing code, calling custom functions, and more. - -## ๐Ÿ“ Directory Structure - -``` -tests/agents/tools/ -โ”œโ”€โ”€ README.md # This file -โ”œโ”€โ”€ test_agent_*.py # Single-tool tests -โ”œโ”€โ”€ test_agent_tools_with_conversations.py # Single tools + conversations -โ””โ”€โ”€ multitool/ # Multi-tool combinations - โ”œโ”€โ”€ test_agent_*_and_*.py # Dual-tool tests - โ”œโ”€โ”€ test_agent_*_*_*.py # Three-tool tests - โ””โ”€โ”€ test_multitool_with_conversations.py # Multi-tools + conversations -``` - -## ๐Ÿ”ง Tool Types & Architecture - -### Server-Side Tools (Automatic Execution) -These tools are executed entirely on the server. No client-side dispatch loop required. - -| Tool | Test File | What It Does | -|------|-----------|--------------| -| **FileSearchTool** | `test_agent_file_search.py` | Searches uploaded documents using vector stores | -| **CodeInterpreterTool** | `test_agent_code_interpreter.py` | Executes Python code in sandboxed environment | -| **AzureAISearchAgentTool** | `test_agent_ai_search.py` | Queries Azure AI Search indexes | -| **BingGroundingTool** | `test_agent_bing_grounding.py` | Searches the web using Bing API | -| **WebSearchTool** | `test_agent_web_search.py` | Performs web searches | -| **MCPTool** | `test_agent_mcp.py` | Model Context Protocol integrations | - -**Usage Pattern:** -```python -# Server-side tools - just create and go! -response = openai_client.responses.create( - input="Search for information about...", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}} -) -# Server handles everything - no loop needed -print(response.output_text) -``` - -### Client-Side Tools (Manual Dispatch Loop) -These tools require client-side execution logic. You must implement a dispatch loop. - -| Tool | Test File | What It Does | -|------|-----------|--------------| -| **FunctionTool** | `test_agent_function_tool.py` | Calls custom Python functions defined in your code | - -**Usage Pattern:** -```python -# Client-side tools - requires dispatch loop -response = openai_client.responses.create( - input="Calculate 15 plus 27", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}} -) - -# Check for function calls -for item in response.output: - if item.type == "function_call": - # Execute function locally - result = my_function(json.loads(item.arguments)) - - # Send result back - response = openai_client.responses.create( - input=[FunctionCallOutput( - call_id=item.call_id, - output=json.dumps(result) - )], - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}} - ) -``` - -## ๐Ÿ“ Test Categories - -### 1. Single-Tool Tests -Each test focuses on one tool type: - -- **`test_agent_file_search.py`** - Vector store document search - - Upload files to vector store - - Search across multiple documents - - Citation handling - - Stream vs non-stream responses - -- **`test_agent_code_interpreter.py`** - Python code execution - - Execute Python calculations - - Generate data files (CSV, images) - - File upload and download - - Error handling - -- **`test_agent_function_tool.py`** - Custom function calling - - Define function schemas - - Client-side execution loop - - Multi-turn function calls - - JSON parameter handling - -- **`test_agent_ai_search.py`** - Azure AI Search integration - - Connect to existing indexes - - Query with citations - - Multiple index support - -- **`test_agent_bing_grounding.py`** - Bing web search - - Real-time web queries - - URL citations - - Grounding with web sources - -- **`test_agent_mcp.py`** - Model Context Protocol - - GitHub integration - - Custom MCP servers - - Tool discovery - -- **`test_agent_web_search.py`** - Web search capabilities -- **`test_agent_image_generation.py`** - DALL-E image generation - -### 2. Multi-Tool Tests (`multitool/`) -Tests combining multiple tools in a single agent: - -- **`test_agent_file_search_and_function.py`** - - Search documents, then save results via function - - 4 comprehensive tests demonstrating different workflows - -- **`test_agent_code_interpreter_and_function.py`** - - Generate data with code, save via function - - Calculate and persist results - -- **`test_agent_file_search_and_code_interpreter.py`** - - Search docs, analyze with Python code - - Data extraction and processing - -- **`test_agent_file_search_code_interpreter_function.py`** - - All three tools working together - - Complete analysis workflows - -### 3. Conversation State Management -Tests demonstrating multi-turn interactions with state preservation: - -#### Single-Tool Conversations (`test_agent_tools_with_conversations.py`) -- **`test_function_tool_with_conversation`** - - Multiple function calls in one conversation - - Context preservation (agent remembers previous results) - - Conversation state verification - -- **`test_file_search_with_conversation`** - - Multiple searches in one conversation - - Follow-up questions with context - -- **`test_code_interpreter_with_conversation`** - - Sequential code executions - - Variable/state management across turns - -#### Multi-Tool Conversations (`multitool/test_multitool_with_conversations.py`) -- **`test_file_search_and_function_with_conversation`** - - Mix server-side (FileSearch) and client-side (Function) tools - - Complex workflow: Search โ†’ Follow-up โ†’ Save report - - Verifies both tool types tracked in conversation - -## ๐Ÿ”„ Conversation Patterns - -Tests demonstrate three patterns for multi-turn interactions: - -### Pattern 1: Manual History Management -```python -history = [{"role": "user", "content": "first message"}] -response = client.responses.create(input=history) -history += [{"role": el.role, "content": el.content} for el in response.output] -history.append({"role": "user", "content": "follow-up"}) -response = client.responses.create(input=history) -``` - -### Pattern 2: Using `previous_response_id` -```python -response_1 = client.responses.create(input="first message") -response_2 = client.responses.create( - input="follow-up", - previous_response_id=response_1.id -) -``` - -### Pattern 3: Using Conversations API (Recommended) -```python -conversation = client.conversations.create() -response_1 = client.responses.create( - input="first message", - conversation=conversation.id -) -response_2 = client.responses.create( - input="follow-up", - conversation=conversation.id -) -``` - -โš ๏ธ **Important:** When using `conversation` parameter, do NOT use `previous_response_id` - they are mutually exclusive. - -### Conversation State Verification -Tests verify conversation state by reading back all items: -```python -all_items = list(client.conversations.items.list(conversation.id)) -# Verify all user messages, assistant messages, tool calls preserved -``` - -## โš™๏ธ Environment Setup - -### Required Environment Variables - -Tool tests require additional environment variables beyond the basic SDK tests: - -```bash -# Core settings (already in base .env) -AZURE_AI_PROJECTS_TESTS_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com/api/projects/your-project -AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-4o - -# Tool-specific connections (needed for tool tests) -AZURE_AI_PROJECTS_TESTS_BING_CONNECTION_ID=/subscriptions/.../connections/your-bing-connection -AZURE_AI_PROJECTS_TESTS_AI_SEARCH_CONNECTION_ID=/subscriptions/.../connections/your-search-connection -AZURE_AI_PROJECTS_TESTS_AI_SEARCH_INDEX_NAME=your-index-name -AZURE_AI_PROJECTS_TESTS_MCP_PROJECT_CONNECTION_ID=/subscriptions/.../connections/your-mcp-connection -``` - -### ๐Ÿš€ Quick Setup with Auto-Discovery - -Use the `generate_env_file.py` script to automatically discover your project's configuration: - -```bash -# Navigate to the project directory -cd sdk/ai/azure-ai-projects - -# Run the generator with your project endpoint -uv run python scripts/generate_env_file.py \ - "https://your-project.services.ai.azure.com/api/projects/your-project" \ - myproject - -# This creates: .env.generated.myproject -``` - -The script will automatically discover: -- โœ… Available model deployments -- โœ… Bing connection (if configured) -- โœ… AI Search connection (if configured) -- โœ… GitHub/MCP connections (if configured) - -**Then copy the discovered values to your main `.env` file:** - -```bash -# Review the generated file -cat .env.generated.myproject - -# Copy relevant values to your .env -# You may need to manually add the AI Search index name -``` - -### Manual Setup - -If you prefer manual setup or need specific connections: - -1. **Go to Azure AI Foundry**: https://ai.azure.com -2. **Open your project** -3. **Navigate to "Connections" tab** -4. **Copy connection IDs** (full resource paths starting with `/subscriptions/...`) -5. **Add to your `.env` file** - -For AI Search index name: -1. Go to your AI Search service -2. Navigate to "Indexes" tab -3. Copy the index name - -## ๐Ÿงช Running Tests - -### Run All Tool Tests -```bash -pytest tests/agents/tools/ -v -``` - -### Run Single-Tool Tests Only -```bash -pytest tests/agents/tools/test_agent_*.py -v -``` - -### Run Multi-Tool Tests Only -```bash -pytest tests/agents/tools/multitool/ -v -``` - -### Run Conversation Tests -```bash -# Single-tool conversations -pytest tests/agents/tools/test_agent_tools_with_conversations.py -v - -# Multi-tool conversations -pytest tests/agents/tools/multitool/test_multitool_with_conversations.py -v -``` - -### Run Specific Tool Tests -```bash -# Function tool -pytest tests/agents/tools/test_agent_function_tool.py -v - -# File search -pytest tests/agents/tools/test_agent_file_search.py -v - -# Code interpreter -pytest tests/agents/tools/test_agent_code_interpreter.py -v -``` - -### Run Single Test -```bash -pytest tests/agents/tools/test_agent_function_tool.py::TestAgentFunctionTool::test_agent_function_tool -v -s -``` \ No newline at end of file diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/__init__.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/__init__.py deleted file mode 100644 index b74cfa3b899c..000000000000 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py index 3d9ee0c68fd9..73d31f5275fb 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py @@ -14,11 +14,11 @@ import os import pytest +from io import BytesIO from test_base import TestBase, servicePreparer from devtools_testutils import is_live_and_not_recording from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool, CodeInterpreterTool, CodeInterpreterToolAuto - class TestAgentFileSearchAndCodeInterpreter(TestBase): """Tests for agents using File Search + Code Interpreter combination.""" @@ -40,9 +40,7 @@ def test_find_and_analyze_data(self, **kwargs): # Create data file txt_content = "Sample data: 10, 20, 30, 40, 50" - vector_store = openai_client.vector_stores.create(name="DataStore") - - from io import BytesIO + vector_store = openai_client.vector_stores.create(name="DataStore") txt_file = BytesIO(txt_content.encode("utf-8")) txt_file.name = "data.txt" diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py index f4adb0bb9afc..db57ce91e8c9 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py @@ -85,14 +85,14 @@ def test_agent_ai_search_question_answering(self, **kwargs): openai_client = project_client.get_openai_client() # Get AI Search connection and index from environment - ai_search_connection_id = kwargs.get("azure_ai_projects_tests_ai_search_connection_id") + ai_search_connection_id = kwargs.get("azure_ai_projects_tests_ai_search_project_connection_id") ai_search_index_name = kwargs.get("azure_ai_projects_tests_ai_search_index_name") if not ai_search_connection_id: - pytest.skip("AI_SEARCH_PROJECT_CONNECTION_ID environment variable not set") + pytest.skip("AZURE_AI_PROJECTS_TESTS_AI_SEARCH_PROJECT_CONNECTION_ID environment variable not set") if not ai_search_index_name: - pytest.skip("AI_SEARCH_INDEX_NAME environment variable not set") + pytest.skip("AZURE_AI_PROJECTS_TESTS_AI_SEARCH_INDEX_NAME environment variable not set") assert isinstance(ai_search_connection_id, str), "ai_search_connection_id must be a string" assert isinstance(ai_search_index_name, str), "ai_search_index_name must be a string" diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py index 83794b97894d..1bd914c7d513 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py @@ -151,17 +151,17 @@ async def test_agent_ai_search_question_answering_async_parallel(self, **kwargs) project_client = self.create_async_client(operation_group="agents", **kwargs) async with project_client: - openai_client = await project_client.get_openai_client() + openai_client = project_client.get_openai_client() # Get AI Search connection and index from environment - ai_search_connection_id = kwargs.get("azure_ai_projects_tests_ai_search_connection_id") + ai_search_connection_id = kwargs.get("azure_ai_projects_tests_ai_search_project_connection_id") ai_search_index_name = kwargs.get("azure_ai_projects_tests_ai_search_index_name") if not ai_search_connection_id: - pytest.skip("AI_SEARCH_PROJECT_CONNECTION_ID environment variable not set") + pytest.skip("AZURE_AI_PROJECTS_TESTS_AI_SEARCH_PROJECT_CONNECTION_ID environment variable not set") if not ai_search_index_name: - pytest.skip("AI_SEARCH_INDEX_NAME environment variable not set") + pytest.skip("AZURE_AI_PROJECTS_TESTS_AI_SEARCH_INDEX_NAME environment variable not set") assert isinstance(ai_search_connection_id, str), "ai_search_connection_id must be a string" assert isinstance(ai_search_index_name, str), "ai_search_index_name must be a string" diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py index 3e9f81c32660..487f655ecf92 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py @@ -51,12 +51,12 @@ def test_agent_bing_grounding(self, **kwargs): project_client = self.create_client(operation_group="agents", **kwargs) openai_client = project_client.get_openai_client() - # Note: This test requires BING_PROJECT_CONNECTION_ID environment variable + # Note: This test requires AZURE_AI_PROJECTS_TESTS_BING_PROJECT_CONNECTION_ID environment variable # to be set with a valid Bing connection ID from the project - bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_connection_id") + bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_project_connection_id") if not bing_connection_id: - pytest.skip("BING_PROJECT_CONNECTION_ID environment variable not set") + pytest.skip("AZURE_AI_PROJECTS_TESTS_BING_PROJECT_CONNECTION_ID environment variable not set") assert isinstance(bing_connection_id, str), "bing_connection_id must be a string" @@ -152,10 +152,10 @@ def test_agent_bing_grounding_multiple_queries(self, **kwargs): project_client = self.create_client(operation_group="agents", **kwargs) openai_client = project_client.get_openai_client() - bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_connection_id") + bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_project_connection_id") if not bing_connection_id: - pytest.skip("BING_PROJECT_CONNECTION_ID environment variable not set") + pytest.skip("AZURE_AI_PROJECTS_TESTS_BING_PROJECT_CONNECTION_ID environment variable not set") assert isinstance(bing_connection_id, str), "bing_connection_id must be a string" diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py index ef5753773676..1f51d2261293 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py @@ -195,7 +195,7 @@ def test_agent_mcp_with_project_connection(self, **kwargs): mcp_project_connection_id = kwargs.get("azure_ai_projects_tests_mcp_project_connection_id") if not mcp_project_connection_id: - pytest.skip("MCP_PROJECT_CONNECTION_ID environment variable not set") + pytest.skip("AZURE_AI_PROJECTS_TESTS_MCP_PROJECT_CONNECTION_ID environment variable not set") assert isinstance(mcp_project_connection_id, str), "mcp_project_connection_id must be a string" print(f"Using MCP project connection: {mcp_project_connection_id}") diff --git a/sdk/ai/azure-ai-projects/tests/test_base.py b/sdk/ai/azure-ai-projects/tests/test_base.py index 4055c1c294d9..5260e44c38d6 100644 --- a/sdk/ai/azure-ai-projects/tests/test_base.py +++ b/sdk/ai/azure-ai-projects/tests/test_base.py @@ -62,8 +62,8 @@ azure_ai_projects_tests_tracing_project_endpoint="https://sanitized-account-name.services.ai.azure.com/api/projects/sanitized-project-name", azure_ai_projects_tests_container_app_resource_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.App/containerApps/00000", azure_ai_projects_tests_container_ingress_subdomain_suffix="00000", - azure_ai_projects_tests_bing_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-bing-connection", - azure_ai_projects_tests_ai_search_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-ai-search-connection", + azure_ai_projects_tests_bing_project_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-bing-connection", + azure_ai_projects_tests_ai_search_project_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-ai-search-connection", azure_ai_projects_tests_ai_search_index_name="sanitized-index-name", azure_ai_projects_tests_mcp_project_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-mcp-connection", azure_ai_projects_tests_image_model_deployment_name="gpt-image-1-mini", From 3f7d30f23948401374fbb038eef634901a847018 Mon Sep 17 00:00:00 2001 From: Paul Batum Date: Fri, 21 Nov 2025 17:04:37 -0800 Subject: [PATCH 6/8] code review feedback --- ..._agent_file_search_and_code_interpreter.py | 5 +- .../test_agent_file_search_and_function.py | 4 - ...t_file_search_code_interpreter_function.py | 5 +- .../test_multitool_with_conversations.py | 2 +- .../agents/tools/test_agent_ai_search.py | 210 ++--- .../agents/tools/test_agent_bing_grounding.py | 272 +++--- .../tools/test_agent_code_interpreter.py | 307 +++---- .../agents/tools/test_agent_file_search.py | 429 +++++----- .../tools/test_agent_file_search_stream.py | 172 ++-- .../agents/tools/test_agent_function_tool.py | 780 +++++++++--------- .../tools/test_agent_image_generation.py | 158 ++-- .../tests/agents/tools/test_agent_mcp.py | 390 ++++----- .../test_agent_tools_with_conversations.py | 578 ++++++------- .../agents/tools/test_agent_web_search.py | 108 +-- 14 files changed, 1708 insertions(+), 1712 deletions(-) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py index 73d31f5275fb..d673e31f6419 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py @@ -19,6 +19,7 @@ from devtools_testutils import is_live_and_not_recording from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool, CodeInterpreterTool, CodeInterpreterToolAuto + class TestAgentFileSearchAndCodeInterpreter(TestBase): """Tests for agents using File Search + Code Interpreter combination.""" @@ -40,7 +41,7 @@ def test_find_and_analyze_data(self, **kwargs): # Create data file txt_content = "Sample data: 10, 20, 30, 40, 50" - vector_store = openai_client.vector_stores.create(name="DataStore") + vector_store = openai_client.vector_stores.create(name="DataStore") txt_file = BytesIO(txt_content.encode("utf-8")) txt_file.name = "data.txt" @@ -109,8 +110,6 @@ def test_analyze_code_file(self, **kwargs): vector_store = openai_client.vector_stores.create(name="CodeAnalysisStore") - from io import BytesIO - code_file = BytesIO(python_code.encode("utf-8")) code_file.name = "fibonacci.py" diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py index 4e310a9facf6..4e65fae75254 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py @@ -62,8 +62,6 @@ def test_data_analysis_workflow(self, **kwargs): vector_store = openai_client.vector_stores.create(name="SalesDataStore") print(f"Vector store created (id: {vector_store.id})") - from io import BytesIO - txt_file = BytesIO(txt_content.encode("utf-8")) txt_file.name = "sales_data.txt" @@ -269,8 +267,6 @@ def calculate_sum(numbers): # Create vector store and upload vector_store = openai_client.vector_stores.create(name="CodeStore") - from io import BytesIO - code_file = BytesIO(python_code.encode("utf-8")) code_file.name = "sample_code.py" diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py index c9e069268935..b24c6d491e22 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py @@ -15,6 +15,7 @@ import os import json import pytest +from io import BytesIO from test_base import TestBase, servicePreparer from devtools_testutils import is_live_and_not_recording from azure.ai.projects.models import ( @@ -50,8 +51,6 @@ def test_complete_analysis_workflow(self, **kwargs): txt_content = "Sample data for analysis" vector_store = openai_client.vector_stores.create(name="ThreeToolStore") - from io import BytesIO - txt_file = BytesIO(txt_content.encode("utf-8")) txt_file.name = "data.txt" @@ -126,8 +125,6 @@ def test_four_tools_combination(self, **kwargs): txt_content = "Test data" vector_store = openai_client.vector_stores.create(name="FourToolStore") - from io import BytesIO - txt_file = BytesIO(txt_content.encode("utf-8")) txt_file.name = "data.txt" diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py index fc9b416dc7c0..35bd8339821e 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py @@ -12,6 +12,7 @@ import json import pytest +from io import BytesIO from test_base import TestBase, servicePreparer from devtools_testutils import is_live_and_not_recording from azure.ai.projects.models import ( @@ -57,7 +58,6 @@ def test_file_search_and_function_with_conversation(self, **kwargs): """ vector_store = openai_client.vector_stores.create(name="SalesDataStore") - from io import BytesIO file = BytesIO(doc_content.encode("utf-8")) file.name = "sales.txt" diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py index db57ce91e8c9..7d506fc5bc42 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py @@ -80,10 +80,6 @@ def test_agent_ai_search_question_answering(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - # Get AI Search connection and index from environment ai_search_connection_id = kwargs.get("azure_ai_projects_tests_ai_search_project_connection_id") ai_search_index_name = kwargs.get("azure_ai_projects_tests_ai_search_index_name") @@ -97,114 +93,118 @@ def test_agent_ai_search_question_answering(self, **kwargs): assert isinstance(ai_search_connection_id, str), "ai_search_connection_id must be a string" assert isinstance(ai_search_index_name, str), "ai_search_index_name must be a string" - # Create agent with Azure AI Search tool - agent = project_client.agents.create_version( - agent_name="ai-search-qa-agent", - definition=PromptAgentDefinition( - model=model, - instructions="""You are a helpful assistant that answers true/false questions based on the provided search results. + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with Azure AI Search tool + agent = project_client.agents.create_version( + agent_name="ai-search-qa-agent", + definition=PromptAgentDefinition( + model=model, + instructions="""You are a helpful assistant that answers true/false questions based on the provided search results. Always use the Azure AI Search tool to find relevant information before answering. Respond with only 'True' or 'False' based on what you find in the search results. If you cannot find clear evidence in the search results, answer 'False'.""", - tools=[ - AzureAISearchAgentTool( - azure_ai_search=AzureAISearchToolResource( - indexes=[ - AISearchIndexResource( - project_connection_id=ai_search_connection_id, - index_name=ai_search_index_name, - query_type=AzureAISearchQueryType.SIMPLE, - ), - ] + tools=[ + AzureAISearchAgentTool( + azure_ai_search=AzureAISearchToolResource( + indexes=[ + AISearchIndexResource( + project_connection_id=ai_search_connection_id, + index_name=ai_search_index_name, + query_type=AzureAISearchQueryType.SIMPLE, + ), + ] + ) ) - ) - ], - ), - description="Agent for testing AI Search question answering.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - assert agent.id is not None - assert agent.name == "ai-search-qa-agent" - assert agent.version is not None - - # Test each question - correct_answers = 0 - total_questions = len(self.TEST_QUESTIONS) - - for i, qa_pair in enumerate(self.TEST_QUESTIONS, 1): - question = qa_pair["question"] - expected_answer = qa_pair["answer"] + ], + ), + description="Agent for testing AI Search question answering.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "ai-search-qa-agent" + assert agent.version is not None + + # Test each question + correct_answers = 0 + total_questions = len(self.TEST_QUESTIONS) + + for i, qa_pair in enumerate(self.TEST_QUESTIONS, 1): + question = qa_pair["question"] + expected_answer = qa_pair["answer"] + + print(f"\n{'='*80}") + print(f"Question {i}/{total_questions}:") + print(f"Q: {question}") + print(f"Expected: {expected_answer}") + + output_text = "" + + stream_response = openai_client.responses.create( + stream=True, + tool_choice="required", + input=f"Answer this question with only 'True' or 'False': {question}", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + for event in stream_response: + if event.type == "response.created": + print(f"Response created with ID: {event.response.id}") + elif event.type == "response.output_text.delta": + pass # Don't print deltas to reduce output + elif event.type == "response.completed": + output_text = event.response.output_text + print(f"Agent's answer: {output_text}") + + # Parse the answer from the output + # Look for "True" or "False" in the response + output_lower = output_text.lower() + agent_answer = None + + # Try to extract boolean answer + if "true" in output_lower and "false" not in output_lower: + agent_answer = True + elif "false" in output_lower and "true" not in output_lower: + agent_answer = False + elif output_lower.strip() in ["true", "false"]: + agent_answer = output_lower.strip() == "true" + else: + # Try to determine based on more complex responses + # Count occurrences + true_count = output_lower.count("true") + false_count = output_lower.count("false") + if true_count > false_count: + agent_answer = True + elif false_count > true_count: + agent_answer = False + + if agent_answer is not None: + is_correct = agent_answer == expected_answer + if is_correct: + correct_answers += 1 + print(f"โœ“ CORRECT (Agent: {agent_answer}, Expected: {expected_answer})") + else: + print(f"โœ— INCORRECT (Agent: {agent_answer}, Expected: {expected_answer})") + else: + print(f"โœ— UNABLE TO PARSE ANSWER from: {output_text}") + # Print summary print(f"\n{'='*80}") - print(f"Question {i}/{total_questions}:") - print(f"Q: {question}") - print(f"Expected: {expected_answer}") + print(f"SUMMARY: {correct_answers}/{total_questions} questions answered correctly") + print(f"{'='*80}") - output_text = "" - - stream_response = openai_client.responses.create( - stream=True, - tool_choice="required", - input=f"Answer this question with only 'True' or 'False': {question}", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + # Verify that at least 4 out of 5 questions were answered correctly + assert correct_answers >= 4, ( + f"Expected at least 4 correct answers out of {total_questions}, " + f"but got {correct_answers}. The agent needs to answer at least 80% correctly." ) - for event in stream_response: - if event.type == "response.created": - print(f"Response created with ID: {event.response.id}") - elif event.type == "response.output_text.delta": - pass # Don't print deltas to reduce output - elif event.type == "response.completed": - output_text = event.response.output_text - print(f"Agent's answer: {output_text}") - - # Parse the answer from the output - # Look for "True" or "False" in the response - output_lower = output_text.lower() - agent_answer = None - - # Try to extract boolean answer - if "true" in output_lower and "false" not in output_lower: - agent_answer = True - elif "false" in output_lower and "true" not in output_lower: - agent_answer = False - elif output_lower.strip() in ["true", "false"]: - agent_answer = output_lower.strip() == "true" - else: - # Try to determine based on more complex responses - # Count occurrences - true_count = output_lower.count("true") - false_count = output_lower.count("false") - if true_count > false_count: - agent_answer = True - elif false_count > true_count: - agent_answer = False + print( + f"\nโœ“ Test passed! Agent answered {correct_answers}/{total_questions} questions correctly (>= 4 required)" + ) - if agent_answer is not None: - is_correct = agent_answer == expected_answer - if is_correct: - correct_answers += 1 - print(f"โœ“ CORRECT (Agent: {agent_answer}, Expected: {expected_answer})") - else: - print(f"โœ— INCORRECT (Agent: {agent_answer}, Expected: {expected_answer})") - else: - print(f"โœ— UNABLE TO PARSE ANSWER from: {output_text}") - - # Print summary - print(f"\n{'='*80}") - print(f"SUMMARY: {correct_answers}/{total_questions} questions answered correctly") - print(f"{'='*80}") - - # Verify that at least 4 out of 5 questions were answered correctly - assert correct_answers >= 4, ( - f"Expected at least 4 correct answers out of {total_questions}, " - f"but got {correct_answers}. The agent needs to answer at least 80% correctly." - ) - - print( - f"\nโœ“ Test passed! Agent answered {correct_answers}/{total_questions} questions correctly (>= 4 required)" - ) - - # Teardown - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py index 487f655ecf92..8373963e63e5 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py @@ -47,10 +47,6 @@ def test_agent_bing_grounding(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - # Note: This test requires AZURE_AI_PROJECTS_TESTS_BING_PROJECT_CONNECTION_ID environment variable # to be set with a valid Bing connection ID from the project bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_project_connection_id") @@ -60,78 +56,82 @@ def test_agent_bing_grounding(self, **kwargs): assert isinstance(bing_connection_id, str), "bing_connection_id must be a string" - # Create agent with Bing grounding tool - agent = project_client.agents.create_version( - agent_name="bing-grounding-agent", - definition=PromptAgentDefinition( - model=model, - instructions="You are a helpful assistant.", - tools=[ - BingGroundingAgentTool( - bing_grounding=BingGroundingSearchToolParameters( - search_configurations=[ - BingGroundingSearchConfiguration(project_connection_id=bing_connection_id) - ] + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with Bing grounding tool + agent = project_client.agents.create_version( + agent_name="bing-grounding-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant.", + tools=[ + BingGroundingAgentTool( + bing_grounding=BingGroundingSearchToolParameters( + search_configurations=[ + BingGroundingSearchConfiguration(project_connection_id=bing_connection_id) + ] + ) ) - ) - ], - ), - description="You are a helpful agent.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - assert agent.id is not None - assert agent.name == "bing-grounding-agent" - assert agent.version is not None - - # Test agent with a query that requires current web information - output_text = "" - url_citations = [] - - stream_response = openai_client.responses.create( - stream=True, - tool_choice="required", - input="What is the current weather in Seattle?", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - for event in stream_response: - if event.type == "response.created": - print(f"Follow-up response created with ID: {event.response.id}") - assert event.response.id is not None - elif event.type == "response.output_text.delta": - print(f"Delta: {event.delta}") - elif event.type == "response.text.done": - print(f"Follow-up response done!") - elif event.type == "response.output_item.done": - if event.item.type == "message": - item = event.item - if item.content and len(item.content) > 0: - if item.content[-1].type == "output_text": - text_content = item.content[-1] - for annotation in text_content.annotations: - if annotation.type == "url_citation": - print(f"URL Citation: {annotation.url}") - url_citations.append(annotation.url) - elif event.type == "response.completed": - print(f"Follow-up completed!") - print(f"Full response: {event.response.output_text}") - output_text = event.response.output_text - - # Verify that we got a response - assert len(output_text) > 0, "Expected non-empty response text" - - # Verify that we got URL citations (Bing grounding should provide sources) - assert len(url_citations) > 0, "Expected URL citations from Bing grounding" - - # Verify that citations are valid URLs - for url in url_citations: - assert url.startswith("http://") or url.startswith("https://"), f"Invalid URL citation: {url}" - - print(f"Test completed successfully with {len(url_citations)} URL citations") - - # Teardown - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") + ], + ), + description="You are a helpful agent.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "bing-grounding-agent" + assert agent.version is not None + + # Test agent with a query that requires current web information + output_text = "" + url_citations = [] + + stream_response = openai_client.responses.create( + stream=True, + tool_choice="required", + input="What is the current weather in Seattle?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + for event in stream_response: + if event.type == "response.created": + print(f"Follow-up response created with ID: {event.response.id}") + assert event.response.id is not None + elif event.type == "response.output_text.delta": + print(f"Delta: {event.delta}") + elif event.type == "response.text.done": + print(f"Follow-up response done!") + elif event.type == "response.output_item.done": + if event.item.type == "message": + item = event.item + if item.content and len(item.content) > 0: + if item.content[-1].type == "output_text": + text_content = item.content[-1] + for annotation in text_content.annotations: + if annotation.type == "url_citation": + print(f"URL Citation: {annotation.url}") + url_citations.append(annotation.url) + elif event.type == "response.completed": + print(f"Follow-up completed!") + print(f"Full response: {event.response.output_text}") + output_text = event.response.output_text + + # Verify that we got a response + assert len(output_text) > 0, "Expected non-empty response text" + + # Verify that we got URL citations (Bing grounding should provide sources) + assert len(url_citations) > 0, "Expected URL citations from Bing grounding" + + # Verify that citations are valid URLs + for url in url_citations: + assert url.startswith("http://") or url.startswith("https://"), f"Invalid URL citation: {url}" + + print(f"Test completed successfully with {len(url_citations)} URL citations") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") @servicePreparer() @pytest.mark.skipif( @@ -148,10 +148,6 @@ def test_agent_bing_grounding_multiple_queries(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_project_connection_id") if not bing_connection_id: @@ -159,62 +155,66 @@ def test_agent_bing_grounding_multiple_queries(self, **kwargs): assert isinstance(bing_connection_id, str), "bing_connection_id must be a string" - # Create agent with Bing grounding tool - agent = project_client.agents.create_version( - agent_name="bing-grounding-multi-query-agent", - definition=PromptAgentDefinition( - model=model, - instructions="You are a helpful assistant that provides current information.", - tools=[ - BingGroundingAgentTool( - bing_grounding=BingGroundingSearchToolParameters( - search_configurations=[ - BingGroundingSearchConfiguration(project_connection_id=bing_connection_id) - ] + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with Bing grounding tool + agent = project_client.agents.create_version( + agent_name="bing-grounding-multi-query-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that provides current information.", + tools=[ + BingGroundingAgentTool( + bing_grounding=BingGroundingSearchToolParameters( + search_configurations=[ + BingGroundingSearchConfiguration(project_connection_id=bing_connection_id) + ] + ) ) - ) - ], - ), - description="Agent for testing multiple Bing grounding queries.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - - # Test with multiple different queries - queries = [ - "What is today's date?", - "What are the latest news about AI?", - ] - - for query in queries: - print(f"\nTesting query: {query}") - output_text = "" - url_citations = [] - - stream_response = openai_client.responses.create( - stream=True, - tool_choice="required", - input=query, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ], + ), + description="Agent for testing multiple Bing grounding queries.", ) - - for event in stream_response: - if event.type == "response.output_item.done": - if event.item.type == "message": - item = event.item - if item.content and len(item.content) > 0: - if item.content[-1].type == "output_text": - text_content = item.content[-1] - for annotation in text_content.annotations: - if annotation.type == "url_citation": - url_citations.append(annotation.url) - elif event.type == "response.completed": - output_text = event.response.output_text - - # Verify that we got a response for each query - assert len(output_text) > 0, f"Expected non-empty response text for query: {query}" - print(f"Response length: {len(output_text)} characters") - print(f"URL citations found: {len(url_citations)}") - - # Teardown - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + + # Test with multiple different queries + queries = [ + "What is today's date?", + "What are the latest news about AI?", + ] + + for query in queries: + print(f"\nTesting query: {query}") + output_text = "" + url_citations = [] + + stream_response = openai_client.responses.create( + stream=True, + tool_choice="required", + input=query, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + for event in stream_response: + if event.type == "response.output_item.done": + if event.item.type == "message": + item = event.item + if item.content and len(item.content) > 0: + if item.content[-1].type == "output_text": + text_content = item.content[-1] + for annotation in text_content.annotations: + if annotation.type == "url_citation": + url_citations.append(annotation.url) + elif event.type == "response.completed": + output_text = event.response.output_text + + # Verify that we got a response for each query + assert len(output_text) > 0, f"Expected non-empty response text for query: {query}" + print(f"Response length: {len(output_text)} characters") + print(f"URL citations found: {len(url_citations)}") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py index a9e3daff2605..b360d2bd2d32 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py @@ -46,64 +46,67 @@ def test_agent_code_interpreter_simple_math(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Create agent with code interpreter tool (no files) - agent = project_client.agents.create_version( - agent_name="code-interpreter-simple-agent", - definition=PromptAgentDefinition( - model=model, - instructions="You are a helpful assistant that can execute Python code.", - tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[]))], - ), - description="Simple code interpreter agent for basic Python execution.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - assert agent.id is not None - assert agent.name == "code-interpreter-simple-agent" - assert agent.version is not None - - # Ask the agent to execute a complex Python calculation - # Problem: Calculate the sum of cubes from 1 to 50, then add 12!/(8!) - # Expected answer: 1637505 - print("\nAsking agent to calculate: sum of cubes from 1 to 50, plus 12!/(8!)") - - response = openai_client.responses.create( - input="Calculate this using Python: First, find the sum of cubes from 1 to 50 (1ยณ + 2ยณ + ... + 50ยณ). Then add 12 factorial divided by 8 factorial (12!/8!). What is the final result?", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - print(f"Response completed (id: {response.id})") - assert response.id is not None - assert response.output is not None - assert len(response.output) > 0 - - # Get the response text - last_message = response.output[-1] - response_text = "" - - if last_message.type == "message": - for content_item in last_message.content: - if content_item.type == "output_text": - response_text += content_item.text - - print(f"Agent's response: {response_text}") - - # Verify the response contains the correct answer (1637505) - # Note: sum of cubes 1-50 = 1,625,625; 12!/8! = 11,880; total = 1,637,505 - assert ( - "1637505" in response_text or "1,637,505" in response_text - ), f"Expected answer 1637505 to be in response, but got: {response_text}" - - print("โœ“ Code interpreter successfully executed Python code and returned correct answer") - - # Teardown - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with code interpreter tool (no files) + agent = project_client.agents.create_version( + agent_name="code-interpreter-simple-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can execute Python code.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[]))], + ), + description="Simple code interpreter agent for basic Python execution.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "code-interpreter-simple-agent" + assert agent.version is not None + + # Ask the agent to execute a complex Python calculation + # Problem: Calculate the sum of cubes from 1 to 50, then add 12!/(8!) + # Expected answer: 1637505 + print("\nAsking agent to calculate: sum of cubes from 1 to 50, plus 12!/(8!)") + + response = openai_client.responses.create( + input="Calculate this using Python: First, find the sum of cubes from 1 to 50 (1ยณ + 2ยณ + ... + 50ยณ). Then add 12 factorial divided by 8 factorial (12!/8!). What is the final result?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Get the response text + last_message = response.output[-1] + response_text = "" + + if last_message.type == "message": + for content_item in last_message.content: + if content_item.type == "output_text": + response_text += content_item.text + + print(f"Agent's response: {response_text}") + + # Verify the response contains the correct answer (1637505) + # Note: sum of cubes 1-50 = 1,625,625; 12!/8! = 11,880; total = 1,637,505 + assert ( + "1637505" in response_text or "1,637,505" in response_text + ), f"Expected answer 1637505 to be in response, but got: {response_text}" + + print("โœ“ Code interpreter successfully executed Python code and returned correct answer") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") @servicePreparer() + @pytest.mark.skip( + reason="Skipped due to known server bug. Enable once https://msdata.visualstudio.com/Vienna/_workitems/edit/4841313 is resolved" + ) @pytest.mark.skipif( condition=(not is_live_and_not_recording()), reason="Skipped because we cannot record network calls with OpenAI client", @@ -136,100 +139,102 @@ def test_agent_code_interpreter_file_generation(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Get the path to the test CSV file + asset_file_path = os.path.abspath( + os.path.join( + os.path.dirname(__file__), "../../../samples/agents/assets/synthetic_500_quarterly_results.csv" + ) + ) - # Get the path to the test CSV file - asset_file_path = os.path.abspath( - os.path.join( - os.path.dirname(__file__), "../../../samples/agents/assets/synthetic_500_quarterly_results.csv" + assert os.path.exists(asset_file_path), f"Test CSV file not found at: {asset_file_path}" + print(f"Using test CSV file: {asset_file_path}") + + # Upload the CSV file + with open(asset_file_path, "rb") as f: + file = openai_client.files.create(purpose="assistants", file=f) + + print(f"File uploaded (id: {file.id})") + assert file.id is not None + + # Create agent with code interpreter tool and the uploaded file + agent = project_client.agents.create_version( + agent_name="code-interpreter-file-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can analyze data and create visualizations.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[file.id]))], + ), + description="Code interpreter agent for file processing and chart generation.", ) - ) - - assert os.path.exists(asset_file_path), f"Test CSV file not found at: {asset_file_path}" - print(f"Using test CSV file: {asset_file_path}") - - # Upload the CSV file - with open(asset_file_path, "rb") as f: - file = openai_client.files.create(purpose="assistants", file=f) - - print(f"File uploaded (id: {file.id})") - assert file.id is not None - - # Create agent with code interpreter tool and the uploaded file - agent = project_client.agents.create_version( - agent_name="code-interpreter-file-agent", - definition=PromptAgentDefinition( - model=model, - instructions="You are a helpful assistant that can analyze data and create visualizations.", - tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[file.id]))], - ), - description="Code interpreter agent for file processing and chart generation.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - assert agent.id is not None - assert agent.name == "code-interpreter-file-agent" - assert agent.version is not None - - # Ask the agent to create a chart from the CSV - print("\nAsking agent to create a bar chart...") - - response = openai_client.responses.create( - input="Create a bar chart showing operating profit by sector from the uploaded CSV file. Save it as a PNG file.", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - print(f"Response completed (id: {response.id})") - assert response.id is not None - assert response.output is not None - assert len(response.output) > 0 - - # Extract file information from response annotations - file_id = "" - filename = "" - container_id = "" - - last_message = response.output[-1] - if last_message.type == "message": - for content_item in last_message.content: - if content_item.type == "output_text": - if content_item.annotations: - for annotation in content_item.annotations: - if annotation.type == "container_file_citation": - file_id = annotation.file_id - filename = annotation.filename - container_id = annotation.container_id - print(f"Found generated file: {filename} (ID: {file_id}, Container: {container_id})") - break - - # Verify that a file was generated - assert file_id, "Expected a file to be generated but no file ID found in response" - assert filename, "Expected a filename but none found in response" - assert container_id, "Expected a container ID but none found in response" - - print(f"โœ“ File generated successfully: {filename}") - - # Download the generated file - print(f"Downloading file {filename}...") - file_content = openai_client.containers.files.content.retrieve(file_id=file_id, container_id=container_id) - - # Read the content - content_bytes = file_content.read() - assert len(content_bytes) > 0, "Expected file content but got empty bytes" - - print(f"โœ“ File downloaded successfully ({len(content_bytes)} bytes)") - - # Verify it's a PNG file (check magic bytes) - if filename.endswith(".png"): - # PNG files start with: 89 50 4E 47 (โ€ฐPNG) - assert content_bytes[:4] == b"\x89PNG", "File does not appear to be a valid PNG" - print("โœ“ File is a valid PNG image") - - # Teardown - print("\nCleaning up...") - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") - - openai_client.files.delete(file.id) - print("Uploaded file deleted") + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "code-interpreter-file-agent" + assert agent.version is not None + + # Ask the agent to create a chart from the CSV + print("\nAsking agent to create a bar chart...") + + response = openai_client.responses.create( + input="Create a bar chart showing operating profit by sector from the uploaded CSV file. Save it as a PNG file.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Extract file information from response annotations + file_id = "" + filename = "" + container_id = "" + + last_message = response.output[-1] + if last_message.type == "message": + for content_item in last_message.content: + if content_item.type == "output_text": + if content_item.annotations: + for annotation in content_item.annotations: + if annotation.type == "container_file_citation": + file_id = annotation.file_id + filename = annotation.filename + container_id = annotation.container_id + print( + f"Found generated file: {filename} (ID: {file_id}, Container: {container_id})" + ) + break + + # Verify that a file was generated + assert file_id, "Expected a file to be generated but no file ID found in response" + assert filename, "Expected a filename but none found in response" + assert container_id, "Expected a container ID but none found in response" + + print(f"โœ“ File generated successfully: {filename}") + + # Download the generated file + print(f"Downloading file {filename}...") + file_content = openai_client.containers.files.content.retrieve(file_id=file_id, container_id=container_id) + + # Read the content + content_bytes = file_content.read() + assert len(content_bytes) > 0, "Expected file content but got empty bytes" + + print(f"โœ“ File downloaded successfully ({len(content_bytes)} bytes)") + + # Verify it's a PNG file (check magic bytes) + if filename.endswith(".png"): + # PNG files start with: 89 50 4E 47 (โ€ฐPNG) + assert content_bytes[:4] == b"\x89PNG", "File does not appear to be a valid PNG" + print("โœ“ File is a valid PNG image") + + # Teardown + print("\nCleaning up...") + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + openai_client.files.delete(file.id) + print("Uploaded file deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py index 13ceb8fc6695..59cde4dd72d2 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py @@ -6,6 +6,7 @@ import os import pytest +from io import BytesIO from test_base import TestBase, servicePreparer from devtools_testutils import is_live_and_not_recording from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool @@ -46,81 +47,81 @@ def test_agent_file_search(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Get the path to the test file - asset_file_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/product_info.md") - ) + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Get the path to the test file + asset_file_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/product_info.md") + ) - assert os.path.exists(asset_file_path), f"Test file not found at: {asset_file_path}" - print(f"Using test file: {asset_file_path}") + assert os.path.exists(asset_file_path), f"Test file not found at: {asset_file_path}" + print(f"Using test file: {asset_file_path}") + + # Create vector store for file search + vector_store = openai_client.vector_stores.create(name="ProductInfoStore") + print(f"Vector store created (id: {vector_store.id})") + assert vector_store.id is not None + + # Upload file to vector store + with open(asset_file_path, "rb") as f: + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=f, + ) + + print(f"File uploaded (id: {file.id}, status: {file.status})") + assert file.id is not None + assert file.status == "completed", f"Expected file status 'completed', got '{file.status}'" + + # Create agent with file search tool + agent = project_client.agents.create_version( + agent_name="file-search-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can search through uploaded documents to answer questions.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Agent for testing file search capabilities.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "file-search-agent" + assert agent.version is not None - # Create vector store for file search - vector_store = openai_client.vector_stores.create(name="ProductInfoStore") - print(f"Vector store created (id: {vector_store.id})") - assert vector_store.id is not None + # Ask a question about the uploaded document + print("\nAsking agent about the product information...") - # Upload file to vector store - with open(asset_file_path, "rb") as f: - file = openai_client.vector_stores.files.upload_and_poll( - vector_store_id=vector_store.id, - file=f, + response = openai_client.responses.create( + input="What products are mentioned in the document?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, ) - print(f"File uploaded (id: {file.id}, status: {file.status})") - assert file.id is not None - assert file.status == "completed", f"Expected file status 'completed', got '{file.status}'" - - # Create agent with file search tool - agent = project_client.agents.create_version( - agent_name="file-search-agent", - definition=PromptAgentDefinition( - model=model, - instructions="You are a helpful assistant that can search through uploaded documents to answer questions.", - tools=[FileSearchTool(vector_store_ids=[vector_store.id])], - ), - description="Agent for testing file search capabilities.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - assert agent.id is not None - assert agent.name == "file-search-agent" - assert agent.version is not None - - # Ask a question about the uploaded document - print("\nAsking agent about the product information...") - - response = openai_client.responses.create( - input="What products are mentioned in the document?", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - print(f"Response completed (id: {response.id})") - assert response.id is not None - assert response.output is not None - assert len(response.output) > 0 - - # Get the response text - response_text = response.output_text - print(f"\nAgent's response: {response_text[:300]}...") - - # Verify we got a meaningful response - assert len(response_text) > 50, "Expected a substantial response from the agent" - - # The response should mention finding information (indicating file search was used) - # We can't assert exact product names without knowing the file content, - # but we can verify the agent provided an answer - print("\nโœ“ Agent successfully used file search tool to answer question from uploaded document") - - # Teardown - print("\nCleaning up...") - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") - - openai_client.vector_stores.delete(vector_store.id) - print("Vector store deleted") + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Get the response text + response_text = response.output_text + print(f"\nAgent's response: {response_text[:300]}...") + + # Verify we got a meaningful response + assert len(response_text) > 50, "Expected a substantial response from the agent" + + # The response should mention finding information (indicating file search was used) + # We can't assert exact product names without knowing the file content, + # but we can verify the agent provided an answer + print("\nโœ“ Agent successfully used file search tool to answer question from uploaded document") + + # Teardown + print("\nCleaning up...") + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + openai_client.vector_stores.delete(vector_store.id) + print("Vector store deleted") @servicePreparer() @pytest.mark.skipif( @@ -139,63 +140,63 @@ def test_agent_file_search_unsupported_file_type(self, **kwargs): This ensures good developer experience by providing actionable error messages. """ - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Create vector store - vector_store = openai_client.vector_stores.create(name="UnsupportedFileTestStore") - print(f"Vector store created (id: {vector_store.id})") + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create vector store + vector_store = openai_client.vector_stores.create(name="UnsupportedFileTestStore") + print(f"Vector store created (id: {vector_store.id})") - # Create CSV file (unsupported format) - csv_content = """product,quarter,revenue + # Create CSV file (unsupported format) + csv_content = """product,quarter,revenue Widget A,Q1,15000 Widget B,Q1,22000 Widget A,Q2,18000 Widget B,Q2,25000""" - from io import BytesIO - - csv_file = BytesIO(csv_content.encode("utf-8")) - csv_file.name = "sales_data.csv" - - # Attempt to upload unsupported file type - print("\nAttempting to upload CSV file (unsupported format)...") - try: - file = openai_client.vector_stores.files.upload_and_poll( - vector_store_id=vector_store.id, - file=csv_file, - ) - # If we get here, the test should fail + csv_file = BytesIO(csv_content.encode("utf-8")) + csv_file.name = "sales_data.csv" + + # Attempt to upload unsupported file type + print("\nAttempting to upload CSV file (unsupported format)...") + try: + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=csv_file, + ) + # If we get here, the test should fail + openai_client.vector_stores.delete(vector_store.id) + pytest.fail("Expected BadRequestError for CSV file upload, but upload succeeded") + + except Exception as e: + error_message = str(e) + print(f"\nโœ“ Upload correctly rejected with error: {error_message[:200]}...") + + # Verify error message quality + assert ( + "400" in error_message or "BadRequestError" in type(e).__name__ + ), "Should be a 400 Bad Request error" + + assert ".csv" in error_message.lower(), "Error message should mention the CSV file extension" + + assert ( + "not supported" in error_message.lower() or "unsupported" in error_message.lower() + ), "Error message should clearly state the file type is not supported" + + # Check that supported file types are mentioned (helpful for developers) + error_lower = error_message.lower() + has_supported_list = any(ext in error_lower for ext in [".txt", ".pdf", ".md", ".py"]) + assert has_supported_list, "Error message should list examples of supported file types" + + print("โœ“ Error message is clear and actionable") + print(" - Mentions unsupported file type (.csv)") + print(" - States it's not supported") + print(" - Lists supported file types") + + # Cleanup openai_client.vector_stores.delete(vector_store.id) - pytest.fail("Expected BadRequestError for CSV file upload, but upload succeeded") - - except Exception as e: - error_message = str(e) - print(f"\nโœ“ Upload correctly rejected with error: {error_message[:200]}...") - - # Verify error message quality - assert "400" in error_message or "BadRequestError" in type(e).__name__, "Should be a 400 Bad Request error" - - assert ".csv" in error_message.lower(), "Error message should mention the CSV file extension" - - assert ( - "not supported" in error_message.lower() or "unsupported" in error_message.lower() - ), "Error message should clearly state the file type is not supported" - - # Check that supported file types are mentioned (helpful for developers) - error_lower = error_message.lower() - has_supported_list = any(ext in error_lower for ext in [".txt", ".pdf", ".md", ".py"]) - assert has_supported_list, "Error message should list examples of supported file types" - - print("โœ“ Error message is clear and actionable") - print(" - Mentions unsupported file type (.csv)") - print(" - States it's not supported") - print(" - Lists supported file types") - - # Cleanup - openai_client.vector_stores.delete(vector_store.id) - print("\nVector store deleted") + print("\nVector store deleted") @servicePreparer() @pytest.mark.skipif( @@ -212,12 +213,12 @@ def test_agent_file_search_multi_turn_conversation(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Create a document with information about products - product_info = """Product Catalog: + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create a document with information about products + product_info = """Product Catalog: Widget A: - Price: $150 @@ -238,92 +239,90 @@ def test_agent_file_search_multi_turn_conversation(self, **kwargs): - Rating: 4.2/5 stars """ - # Create vector store and upload document - vector_store = openai_client.vector_stores.create(name="ProductCatalog") - print(f"Vector store created: {vector_store.id}") - - from io import BytesIO - - product_file = BytesIO(product_info.encode("utf-8")) - product_file.name = "products.txt" - - file = openai_client.vector_stores.files.upload_and_poll( - vector_store_id=vector_store.id, - file=product_file, - ) - print(f"Product catalog uploaded: {file.id}") - - # Create agent with File Search - agent = project_client.agents.create_version( - agent_name="product-catalog-agent", - definition=PromptAgentDefinition( - model=model, - instructions="You are a product information assistant. Use file search to answer questions about products.", - tools=[FileSearchTool(vector_store_ids=[vector_store.id])], - ), - description="Agent for multi-turn product queries.", - ) - print(f"Agent created: {agent.id}") - - # Turn 1: Ask about price - print("\n--- Turn 1: Initial query ---") - response_1 = openai_client.responses.create( - input="What is the price of Widget B?", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_1_text = response_1.output_text - print(f"Response 1: {response_1_text[:200]}...") - assert "$220" in response_1_text or "220" in response_1_text, "Response should mention Widget B's price" - - # Turn 2: Follow-up question (requires context from turn 1) - print("\n--- Turn 2: Follow-up query (testing context retention) ---") - response_2 = openai_client.responses.create( - input="What about its stock level?", - previous_response_id=response_1.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_2_text = response_2.output_text - print(f"Response 2: {response_2_text[:200]}...") - assert ( - "30" in response_2_text or "thirty" in response_2_text.lower() - ), "Response should mention Widget B's stock (30 units)" - - # Turn 3: Another follow-up (compare with different product) - print("\n--- Turn 3: Comparison query ---") - response_3 = openai_client.responses.create( - input="How does that compare to Widget A's stock?", - previous_response_id=response_2.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_3_text = response_3.output_text - print(f"Response 3: {response_3_text[:200]}...") - assert ( - "50" in response_3_text or "fifty" in response_3_text.lower() - ), "Response should mention Widget A's stock (50 units)" - - # Turn 4: New topic (testing topic switching) - print("\n--- Turn 4: Topic switch ---") - response_4 = openai_client.responses.create( - input="Which widget has the highest rating?", - previous_response_id=response_3.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_4_text = response_4.output_text - print(f"Response 4: {response_4_text[:200]}...") - assert ( - "widget b" in response_4_text.lower() or "4.8" in response_4_text - ), "Response should identify Widget B as highest rated (4.8/5)" - - print("\nโœ“ Multi-turn conversation successful!") - print(" - Context maintained across turns") - print(" - Follow-up questions handled correctly") - print(" - Topic switching works") - - # Cleanup - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - openai_client.vector_stores.delete(vector_store.id) - print("Cleanup completed") + # Create vector store and upload document + vector_store = openai_client.vector_stores.create(name="ProductCatalog") + print(f"Vector store created: {vector_store.id}") + + product_file = BytesIO(product_info.encode("utf-8")) + product_file.name = "products.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=product_file, + ) + print(f"Product catalog uploaded: {file.id}") + + # Create agent with File Search + agent = project_client.agents.create_version( + agent_name="product-catalog-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a product information assistant. Use file search to answer questions about products.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Agent for multi-turn product queries.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Ask about price + print("\n--- Turn 1: Initial query ---") + response_1 = openai_client.responses.create( + input="What is the price of Widget B?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "$220" in response_1_text or "220" in response_1_text, "Response should mention Widget B's price" + + # Turn 2: Follow-up question (requires context from turn 1) + print("\n--- Turn 2: Follow-up query (testing context retention) ---") + response_2 = openai_client.responses.create( + input="What about its stock level?", + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + assert ( + "30" in response_2_text or "thirty" in response_2_text.lower() + ), "Response should mention Widget B's stock (30 units)" + + # Turn 3: Another follow-up (compare with different product) + print("\n--- Turn 3: Comparison query ---") + response_3 = openai_client.responses.create( + input="How does that compare to Widget A's stock?", + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + assert ( + "50" in response_3_text or "fifty" in response_3_text.lower() + ), "Response should mention Widget A's stock (50 units)" + + # Turn 4: New topic (testing topic switching) + print("\n--- Turn 4: Topic switch ---") + response_4 = openai_client.responses.create( + input="Which widget has the highest rating?", + previous_response_id=response_3.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_4_text = response_4.output_text + print(f"Response 4: {response_4_text[:200]}...") + assert ( + "widget b" in response_4_text.lower() or "4.8" in response_4_text + ), "Response should identify Widget B as highest rated (4.8/5)" + + print("\nโœ“ Multi-turn conversation successful!") + print(" - Context maintained across turns") + print(" - Follow-up questions handled correctly") + print(" - Topic switching works") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py index 04b4ced316dc..a207ab32eb9f 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py @@ -47,91 +47,91 @@ def test_agent_file_search_stream(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Get the path to the test file - asset_file_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/product_info.md") - ) - - assert os.path.exists(asset_file_path), f"Test file not found at: {asset_file_path}" - print(f"Using test file: {asset_file_path}") - - # Create vector store for file search - vector_store = openai_client.vector_stores.create(name="ProductInfoStoreStream") - print(f"Vector store created (id: {vector_store.id})") - assert vector_store.id is not None - - # Upload file to vector store - with open(asset_file_path, "rb") as f: - file = openai_client.vector_stores.files.upload_and_poll( - vector_store_id=vector_store.id, - file=f, + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Get the path to the test file + asset_file_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/product_info.md") ) - print(f"File uploaded (id: {file.id}, status: {file.status})") - assert file.id is not None - assert file.status == "completed", f"Expected file status 'completed', got '{file.status}'" - - # Create agent with file search tool - agent = project_client.agents.create_version( - agent_name="file-search-stream-agent", - definition=PromptAgentDefinition( - model=model, - instructions="You are a helpful assistant that can search through uploaded documents to answer questions.", - tools=[FileSearchTool(vector_store_ids=[vector_store.id])], - ), - description="Agent for testing file search with streaming.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - assert agent.id is not None - assert agent.name == "file-search-stream-agent" - assert agent.version is not None - - # Ask a question with streaming enabled - print("\nAsking agent about the product information (streaming)...") - - stream_response = openai_client.responses.create( - stream=True, - input="What products are mentioned in the document? Please provide a brief summary.", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - # Collect streamed response - response_text = "" - response_id = None - events_received = 0 - - for event in stream_response: - events_received += 1 - - if event.type == "response.output_item.done": - if event.item.type == "message": - for content_item in event.item.content: - if content_item.type == "output_text": - response_text += content_item.text - - elif event.type == "response.completed": - response_id = event.response.id - # Could also use event.response.output_text - - print(f"\nStreaming completed (id: {response_id}, events: {events_received})") - assert response_id is not None, "Expected response ID from stream" - assert events_received > 0, "Expected to receive stream events" - - print(f"Agent's streamed response: {response_text[:300]}...") - - # Verify we got a meaningful response - assert len(response_text) > 50, "Expected a substantial response from the agent" - - print("\nโœ“ Agent successfully streamed responses using file search tool") - - # Teardown - print("\nCleaning up...") - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") - - openai_client.vector_stores.delete(vector_store.id) - print("Vector store deleted") + assert os.path.exists(asset_file_path), f"Test file not found at: {asset_file_path}" + print(f"Using test file: {asset_file_path}") + + # Create vector store for file search + vector_store = openai_client.vector_stores.create(name="ProductInfoStoreStream") + print(f"Vector store created (id: {vector_store.id})") + assert vector_store.id is not None + + # Upload file to vector store + with open(asset_file_path, "rb") as f: + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=f, + ) + + print(f"File uploaded (id: {file.id}, status: {file.status})") + assert file.id is not None + assert file.status == "completed", f"Expected file status 'completed', got '{file.status}'" + + # Create agent with file search tool + agent = project_client.agents.create_version( + agent_name="file-search-stream-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can search through uploaded documents to answer questions.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Agent for testing file search with streaming.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "file-search-stream-agent" + assert agent.version is not None + + # Ask a question with streaming enabled + print("\nAsking agent about the product information (streaming)...") + + stream_response = openai_client.responses.create( + stream=True, + input="What products are mentioned in the document? Please provide a brief summary.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Collect streamed response + response_text = "" + response_id = None + events_received = 0 + + for event in stream_response: + events_received += 1 + + if event.type == "response.output_item.done": + if event.item.type == "message": + for content_item in event.item.content: + if content_item.type == "output_text": + response_text += content_item.text + + elif event.type == "response.completed": + response_id = event.response.id + # Could also use event.response.output_text + + print(f"\nStreaming completed (id: {response_id}, events: {events_received})") + assert response_id is not None, "Expected response ID from stream" + assert events_received > 0, "Expected to receive stream events" + + print(f"Agent's streamed response: {response_text[:300]}...") + + # Verify we got a meaningful response + assert len(response_text) > 50, "Expected a substantial response from the agent" + + print("\nโœ“ Agent successfully streamed responses using file search tool") + + # Teardown + print("\nCleaning up...") + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + openai_client.vector_stores.delete(vector_store.id) + print("Vector store deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py index 6aeda3aaa5f5..eaf6d7ec3617 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py @@ -46,128 +46,128 @@ def test_agent_function_tool(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Define a function tool for the model to use - func_tool = FunctionTool( - name="get_weather", - description="Get the current weather for a location.", - parameters={ - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "A city name like Seattle or London", + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Define a function tool for the model to use + func_tool = FunctionTool( + name="get_weather", + description="Get the current weather for a location.", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "A city name like Seattle or London", + }, }, + "required": ["location"], + "additionalProperties": False, }, - "required": ["location"], - "additionalProperties": False, - }, - strict=True, - ) - - # Create agent with function tool - agent = project_client.agents.create_version( - agent_name="function-tool-agent", - definition=PromptAgentDefinition( - model=model, - instructions="You are a helpful assistant that can check the weather. Use the get_weather function when users ask about weather.", - tools=[func_tool], - ), - description="Agent for testing function tool capabilities.", - ) - print( - f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version}, model: {agent.definition['model']})" - ) - assert agent.id is not None - assert agent.name == "function-tool-agent" - assert agent.version is not None - - # Ask a question that should trigger the function call - print("\nAsking agent: What's the weather in Seattle?") - - response = openai_client.responses.create( - input="What's the weather in Seattle?", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - print(f"Initial response completed (id: {response.id})") - assert response.id is not None - assert response.output is not None - - # Check for function calls in the response - function_calls_found = 0 - input_list: ResponseInputParam = [] - - for item in response.output: - if item.type == "function_call": - function_calls_found += 1 - print(f"Found function call (id: {item.call_id}, name: {item.name})") - - # Parse the arguments - arguments = json.loads(item.arguments) - print(f"Function arguments: {arguments}") - - # Verify the function call is for get_weather - assert item.name == "get_weather", f"Expected function name 'get_weather', got '{item.name}'" - assert "location" in arguments, "Expected 'location' in function arguments" - assert ( - "seattle" in arguments["location"].lower() - ), f"Expected Seattle in location, got {arguments['location']}" - - # Simulate the function execution and provide a result - weather_result = { - "location": arguments["location"], - "temperature": "72ยฐF", - "condition": "Sunny", - "humidity": "45%", - } - - # Add the function result to the input list - input_list.append( - FunctionCallOutput( - type="function_call_output", - call_id=item.call_id, - output=json.dumps(weather_result), + strict=True, + ) + + # Create agent with function tool + agent = project_client.agents.create_version( + agent_name="function-tool-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can check the weather. Use the get_weather function when users ask about weather.", + tools=[func_tool], + ), + description="Agent for testing function tool capabilities.", + ) + print( + f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version}, model: {agent.definition['model']})" + ) + assert agent.id is not None + assert agent.name == "function-tool-agent" + assert agent.version is not None + + # Ask a question that should trigger the function call + print("\nAsking agent: What's the weather in Seattle?") + + response = openai_client.responses.create( + input="What's the weather in Seattle?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + + # Check for function calls in the response + function_calls_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "function_call": + function_calls_found += 1 + print(f"Found function call (id: {item.call_id}, name: {item.name})") + + # Parse the arguments + arguments = json.loads(item.arguments) + print(f"Function arguments: {arguments}") + + # Verify the function call is for get_weather + assert item.name == "get_weather", f"Expected function name 'get_weather', got '{item.name}'" + assert "location" in arguments, "Expected 'location' in function arguments" + assert ( + "seattle" in arguments["location"].lower() + ), f"Expected Seattle in location, got {arguments['location']}" + + # Simulate the function execution and provide a result + weather_result = { + "location": arguments["location"], + "temperature": "72ยฐF", + "condition": "Sunny", + "humidity": "45%", + } + + # Add the function result to the input list + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(weather_result), + ) ) - ) - print(f"โœ“ Prepared function result: {weather_result}") + print(f"โœ“ Prepared function result: {weather_result}") - # Verify that at least one function call was made - assert function_calls_found > 0, "Expected at least 1 function call, but found none" - print(f"\nโœ“ Processed {function_calls_found} function call(s)") + # Verify that at least one function call was made + assert function_calls_found > 0, "Expected at least 1 function call, but found none" + print(f"\nโœ“ Processed {function_calls_found} function call(s)") - # Send the function results back to get the final response - print("\nSending function results back to agent...") + # Send the function results back to get the final response + print("\nSending function results back to agent...") - response = openai_client.responses.create( - input=input_list, - previous_response_id=response.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) - print(f"Final response completed (id: {response.id})") - assert response.id is not None + print(f"Final response completed (id: {response.id})") + assert response.id is not None - # Get the final response text - response_text = response.output_text - print(f"\nAgent's final response: {response_text}") + # Get the final response text + response_text = response.output_text + print(f"\nAgent's final response: {response_text}") - # Verify the response incorporates the weather data - assert len(response_text) > 20, "Expected a meaningful response from the agent" + # Verify the response incorporates the weather data + assert len(response_text) > 20, "Expected a meaningful response from the agent" - # Check that the response mentions the weather information we provided - response_lower = response_text.lower() - assert any( - keyword in response_lower for keyword in ["72", "sunny", "weather", "seattle"] - ), f"Expected response to mention weather information, but got: {response_text}" + # Check that the response mentions the weather information we provided + response_lower = response_text.lower() + assert any( + keyword in response_lower for keyword in ["72", "sunny", "weather", "seattle"] + ), f"Expected response to mention weather information, but got: {response_text}" - print("\nโœ“ Agent successfully used function tool and incorporated results into response") + print("\nโœ“ Agent successfully used function tool and incorporated results into response") - # Teardown - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) print("Agent deleted") @servicePreparer() @@ -187,158 +187,73 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Define multiple function tools - get_weather = FunctionTool( - name="get_weather", - description="Get current weather for a city", - parameters={ - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city name", + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Define multiple function tools + get_weather = FunctionTool( + name="get_weather", + description="Get current weather for a city", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city name", + }, }, + "required": ["city"], + "additionalProperties": False, }, - "required": ["city"], - "additionalProperties": False, - }, - strict=True, - ) - - get_temperature_forecast = FunctionTool( - name="get_temperature_forecast", - description="Get 3-day temperature forecast for a city", - parameters={ - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city name", + strict=True, + ) + + get_temperature_forecast = FunctionTool( + name="get_temperature_forecast", + description="Get 3-day temperature forecast for a city", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city name", + }, }, + "required": ["city"], + "additionalProperties": False, }, - "required": ["city"], - "additionalProperties": False, - }, - strict=True, - ) - - # Create agent with multiple functions - agent = project_client.agents.create_version( - agent_name="weather-assistant-multi-turn", - definition=PromptAgentDefinition( - model=model, - instructions="You are a weather assistant. Use available functions to answer weather questions.", - tools=[get_weather, get_temperature_forecast], - ), - description="Weather assistant for multi-turn testing.", - ) - print(f"Agent created: {agent.id}") - - # Turn 1: Get current weather - print("\n--- Turn 1: Current weather query ---") - response_1 = openai_client.responses.create( - input="What's the weather in New York?", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - # Handle function call - input_list: ResponseInputParam = [] - for item in response_1.output: - if item.type == "function_call": - print(f"Function called: {item.name} with args: {item.arguments}") - assert item.name == "get_weather" - - # Simulate weather API response - weather_data = {"temperature": 68, "condition": "Cloudy", "humidity": 65} - input_list.append( - FunctionCallOutput( - type="function_call_output", - call_id=item.call_id, - output=json.dumps(weather_data), - ) - ) - - # Get response with function results - response_1 = openai_client.responses.create( - input=input_list, - previous_response_id=response_1.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_1_text = response_1.output_text - print(f"Response 1: {response_1_text[:200]}...") - assert "68" in response_1_text or "cloudy" in response_1_text.lower() - - # Turn 2: Follow-up with forecast (requires context) - print("\n--- Turn 2: Follow-up forecast query ---") - response_2 = openai_client.responses.create( - input="What about the forecast for the next few days?", - previous_response_id=response_1.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - # Handle forecast function call - input_list = [] - for item in response_2.output: - if item.type == "function_call": - print(f"Function called: {item.name} with args: {item.arguments}") - assert item.name == "get_temperature_forecast" - - # Agent should remember we're talking about New York - args = json.loads(item.arguments) - assert "new york" in args["city"].lower() - - # Simulate forecast API response - forecast_data = { - "city": "New York", - "forecast": [ - {"day": "Tomorrow", "temp": 70}, - {"day": "Day 2", "temp": 72}, - {"day": "Day 3", "temp": 69}, - ], - } - input_list.append( - FunctionCallOutput( - type="function_call_output", - call_id=item.call_id, - output=json.dumps(forecast_data), - ) - ) - - # Get response with forecast - response_2 = openai_client.responses.create( - input=input_list, - previous_response_id=response_2.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_2_text = response_2.output_text - print(f"Response 2: {response_2_text[:200]}...") - assert "70" in response_2_text or "72" in response_2_text - - # Turn 3: Compare with another city - print("\n--- Turn 3: New city query ---") - response_3 = openai_client.responses.create( - input="How does that compare to Seattle's weather?", - previous_response_id=response_2.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - # Handle function calls for Seattle (agent might call both weather and forecast) - input_list = [] - for item in response_3.output: - if item.type == "function_call": - print(f"Function called: {item.name} with args: {item.arguments}") - args = json.loads(item.arguments) - assert "seattle" in args["city"].lower() - - # Handle based on function name - if item.name == "get_weather": - weather_data = {"temperature": 58, "condition": "Rainy", "humidity": 80} + strict=True, + ) + + # Create agent with multiple functions + agent = project_client.agents.create_version( + agent_name="weather-assistant-multi-turn", + definition=PromptAgentDefinition( + model=model, + instructions="You are a weather assistant. Use available functions to answer weather questions.", + tools=[get_weather, get_temperature_forecast], + ), + description="Weather assistant for multi-turn testing.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Get current weather + print("\n--- Turn 1: Current weather query ---") + response_1 = openai_client.responses.create( + input="What's the weather in New York?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_1.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + assert item.name == "get_weather" + + # Simulate weather API response + weather_data = {"temperature": 68, "condition": "Cloudy", "humidity": 65} input_list.append( FunctionCallOutput( type="function_call_output", @@ -346,13 +261,44 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): output=json.dumps(weather_data), ) ) - elif item.name == "get_temperature_forecast": + + # Get response with function results + response_1 = openai_client.responses.create( + input=input_list, + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "68" in response_1_text or "cloudy" in response_1_text.lower() + + # Turn 2: Follow-up with forecast (requires context) + print("\n--- Turn 2: Follow-up forecast query ---") + response_2 = openai_client.responses.create( + input="What about the forecast for the next few days?", + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle forecast function call + input_list = [] + for item in response_2.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + assert item.name == "get_temperature_forecast" + + # Agent should remember we're talking about New York + args = json.loads(item.arguments) + assert "new york" in args["city"].lower() + + # Simulate forecast API response forecast_data = { - "city": "Seattle", + "city": "New York", "forecast": [ - {"day": "Tomorrow", "temp": 56}, - {"day": "Day 2", "temp": 59}, - {"day": "Day 3", "temp": 57}, + {"day": "Tomorrow", "temp": 70}, + {"day": "Day 2", "temp": 72}, + {"day": "Day 3", "temp": 69}, ], } input_list.append( @@ -363,26 +309,80 @@ def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): ) ) - # Get final comparison response - response_3 = openai_client.responses.create( - input=input_list, - previous_response_id=response_3.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_3_text = response_3.output_text - print(f"Response 3: {response_3_text[:200]}...") - # Agent should mention Seattle weather (either 58 for current or comparison) - assert "seattle" in response_3_text.lower() or any(temp in response_3_text for temp in ["58", "56", "59"]) - - print("\nโœ“ Multi-turn conversation with multiple function calls successful!") - print(" - Multiple functions called across turns") - print(" - Context maintained (agent remembered New York)") - print(" - Comparison between cities works") - - # Cleanup - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") + # Get response with forecast + response_2 = openai_client.responses.create( + input=input_list, + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + assert "70" in response_2_text or "72" in response_2_text + + # Turn 3: Compare with another city + print("\n--- Turn 3: New city query ---") + response_3 = openai_client.responses.create( + input="How does that compare to Seattle's weather?", + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function calls for Seattle (agent might call both weather and forecast) + input_list = [] + for item in response_3.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + args = json.loads(item.arguments) + assert "seattle" in args["city"].lower() + + # Handle based on function name + if item.name == "get_weather": + weather_data = {"temperature": 58, "condition": "Rainy", "humidity": 80} + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(weather_data), + ) + ) + elif item.name == "get_temperature_forecast": + forecast_data = { + "city": "Seattle", + "forecast": [ + {"day": "Tomorrow", "temp": 56}, + {"day": "Day 2", "temp": 59}, + {"day": "Day 3", "temp": 57}, + ], + } + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(forecast_data), + ) + ) + + # Get final comparison response + response_3 = openai_client.responses.create( + input=input_list, + previous_response_id=response_3.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + # Agent should mention Seattle weather (either 58 for current or comparison) + assert "seattle" in response_3_text.lower() or any(temp in response_3_text for temp in ["58", "56", "59"]) + + print("\nโœ“ Multi-turn conversation with multiple function calls successful!") + print(" - Multiple functions called across turns") + print(" - Context maintained (agent remembered New York)") + print(" - Comparison between cities works") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") @servicePreparer() @pytest.mark.skipif( @@ -399,113 +399,113 @@ def test_agent_function_tool_context_dependent_followup(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Define function tool - get_temperature = FunctionTool( - name="get_temperature", - description="Get current temperature for a city in Fahrenheit", - parameters={ - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city name", + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Define function tool + get_temperature = FunctionTool( + name="get_temperature", + description="Get current temperature for a city in Fahrenheit", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city name", + }, }, + "required": ["city"], + "additionalProperties": False, }, - "required": ["city"], - "additionalProperties": False, - }, - strict=True, - ) - - # Create agent - agent = project_client.agents.create_version( - agent_name="temperature-assistant-context", - definition=PromptAgentDefinition( - model=model, - instructions="You are a temperature assistant. Answer temperature questions.", - tools=[get_temperature], - ), - description="Temperature assistant for context testing.", - ) - print(f"Agent created: {agent.id}") - - # Turn 1: Get temperature in Fahrenheit - print("\n--- Turn 1: Get temperature ---") - response_1 = openai_client.responses.create( - input="What's the temperature in Boston?", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - # Handle function call - input_list: ResponseInputParam = [] - for item in response_1.output: - if item.type == "function_call": - print(f"Function called: {item.name} with args: {item.arguments}") - input_list.append( - FunctionCallOutput( - type="function_call_output", - call_id=item.call_id, - output=json.dumps({"temperature": 72, "unit": "F"}), + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="temperature-assistant-context", + definition=PromptAgentDefinition( + model=model, + instructions="You are a temperature assistant. Answer temperature questions.", + tools=[get_temperature], + ), + description="Temperature assistant for context testing.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Get temperature in Fahrenheit + print("\n--- Turn 1: Get temperature ---") + response_1 = openai_client.responses.create( + input="What's the temperature in Boston?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_1.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"temperature": 72, "unit": "F"}), + ) ) - ) - - response_1 = openai_client.responses.create( - input=input_list, - previous_response_id=response_1.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_1_text = response_1.output_text - print(f"Response 1: {response_1_text[:200]}...") - assert "72" in response_1_text, "Should mention 72ยฐF" - - # Turn 2: Context-dependent follow-up (convert the previous number) - print("\n--- Turn 2: Context-dependent conversion ---") - response_2 = openai_client.responses.create( - input="What is that in Celsius?", # "that" refers to the 72ยฐF from previous response - previous_response_id=response_1.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_2_text = response_2.output_text - print(f"Response 2: {response_2_text[:200]}...") - - # Should convert 72ยฐF to ~22ยฐC (without calling the function again) - # The agent should use the previous response's value - response_2_lower = response_2_text.lower() - assert ( - "celsius" in response_2_lower or "ยฐc" in response_2_lower or "c" in response_2_lower - ), "Response should mention Celsius" - assert any( - temp in response_2_text for temp in ["22", "22.2", "22.22", "20", "21", "23"] - ), f"Response should calculate Celsius from 72ยฐF (~22ยฐC), got: {response_2_text}" - - # Turn 3: Another context-dependent follow-up (comparison) - print("\n--- Turn 3: Compare to another value ---") - response_3 = openai_client.responses.create( - input="Is that warmer or colder than 25ยฐC?", # "that" refers to the Celsius value just mentioned - previous_response_id=response_2.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_3_text = response_3.output_text - print(f"Response 3: {response_3_text[:200]}...") - - # 22ยฐC is colder than 25ยฐC - response_3_lower = response_3_text.lower() - assert ( - "colder" in response_3_lower or "cooler" in response_3_lower or "lower" in response_3_lower - ), f"Response should indicate 22ยฐC is colder than 25ยฐC, got: {response_3_text}" - - print("\nโœ“ Context-dependent follow-ups successful!") - print(" - Agent converted temperature from previous response") - print(" - Agent compared values from conversation history") - print(" - No unnecessary function calls made") - - # Cleanup - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") + + response_1 = openai_client.responses.create( + input=input_list, + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "72" in response_1_text, "Should mention 72ยฐF" + + # Turn 2: Context-dependent follow-up (convert the previous number) + print("\n--- Turn 2: Context-dependent conversion ---") + response_2 = openai_client.responses.create( + input="What is that in Celsius?", # "that" refers to the 72ยฐF from previous response + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + + # Should convert 72ยฐF to ~22ยฐC (without calling the function again) + # The agent should use the previous response's value + response_2_lower = response_2_text.lower() + assert ( + "celsius" in response_2_lower or "ยฐc" in response_2_lower or "c" in response_2_lower + ), "Response should mention Celsius" + assert any( + temp in response_2_text for temp in ["22", "22.2", "22.22", "20", "21", "23"] + ), f"Response should calculate Celsius from 72ยฐF (~22ยฐC), got: {response_2_text}" + + # Turn 3: Another context-dependent follow-up (comparison) + print("\n--- Turn 3: Compare to another value ---") + response_3 = openai_client.responses.create( + input="Is that warmer or colder than 25ยฐC?", # "that" refers to the Celsius value just mentioned + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + + # 22ยฐC is colder than 25ยฐC + response_3_lower = response_3_text.lower() + assert ( + "colder" in response_3_lower or "cooler" in response_3_lower or "lower" in response_3_lower + ), f"Response should indicate 22ยฐC is colder than 25ยฐC, got: {response_3_text}" + + print("\nโœ“ Context-dependent follow-ups successful!") + print(" - Agent converted temperature from previous response") + print(" - Agent compared values from conversation history") + print(" - No unnecessary function calls made") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py index cdc157030c72..7a8c34ed184f 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py @@ -51,82 +51,82 @@ def test_agent_image_generation(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Check if the image model deployment exists in the project - try: - deployment = project_client.deployments.get(image_model_deployment) - print(f"Image model deployment found: {deployment.name}") - except ResourceNotFoundError: - pytest.skip(f"Image generation model '{image_model_deployment}' not available in this project") - except Exception as e: - pytest.skip(f"Unable to verify image model deployment: {e}") - - # Disable retries for faster failure when service returns 500 - openai_client.max_retries = 0 - - # Create agent with image generation tool - agent = project_client.agents.create_version( - agent_name="image-gen-agent", - definition=PromptAgentDefinition( - model=model, - instructions="Generate images based on user prompts", - tools=[ImageGenTool(quality="low", size="1024x1024")], - ), - description="Agent for testing image generation.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - assert agent.id is not None - assert agent.name == "image-gen-agent" - assert agent.version is not None - - # Request image generation - print("\nAsking agent to generate an image of a simple geometric shape...") - - response = openai_client.responses.create( - input="Generate an image of a blue circle on a white background.", - extra_headers={ - "x-ms-oai-image-generation-deployment": image_model_deployment - }, # Required for image generation - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - print(f"Response created (id: {response.id})") - assert response.id is not None - assert response.output is not None - assert len(response.output) > 0 - - # Extract image data from response - image_data = [output.result for output in response.output if output.type == "image_generation_call"] - - # Verify image was generated - assert len(image_data) > 0, "Expected at least one image to be generated" - assert image_data[0], "Expected image data to be non-empty" - - print(f"โœ“ Image data received ({len(image_data[0])} base64 characters)") - - # Decode the base64 image - image_bytes = b"" - try: - image_bytes = base64.b64decode(image_data[0]) - assert len(image_bytes) > 0, "Decoded image should have content" - print(f"โœ“ Image decoded successfully ({len(image_bytes)} bytes)") - except Exception as e: - pytest.fail(f"Failed to decode base64 image data: {e}") - - # Verify it's a PNG image (check magic bytes) - # PNG files start with: 89 50 4E 47 (โ€ฐPNG) - assert image_bytes[:4] == b"\x89PNG", "Image does not appear to be a valid PNG" - print("โœ“ Image is a valid PNG") - - # Verify reasonable image size (should be > 1KB for a 1024x1024 image) - assert len(image_bytes) > 1024, f"Image seems too small ({len(image_bytes)} bytes)" - print(f"โœ“ Image size is reasonable ({len(image_bytes):,} bytes)") - - print("\nโœ“ Agent successfully generated and returned a valid image") - - # Teardown - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Check if the image model deployment exists in the project + try: + deployment = project_client.deployments.get(image_model_deployment) + print(f"Image model deployment found: {deployment.name}") + except ResourceNotFoundError: + pytest.skip(f"Image generation model '{image_model_deployment}' not available in this project") + except Exception as e: + pytest.skip(f"Unable to verify image model deployment: {e}") + + # Disable retries for faster failure when service returns 500 + openai_client.max_retries = 0 + + # Create agent with image generation tool + agent = project_client.agents.create_version( + agent_name="image-gen-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Generate images based on user prompts", + tools=[ImageGenTool(quality="low", size="1024x1024")], + ), + description="Agent for testing image generation.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "image-gen-agent" + assert agent.version is not None + + # Request image generation + print("\nAsking agent to generate an image of a simple geometric shape...") + + response = openai_client.responses.create( + input="Generate an image of a blue circle on a white background.", + extra_headers={ + "x-ms-oai-image-generation-deployment": image_model_deployment + }, # Required for image generation + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response created (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Extract image data from response + image_data = [output.result for output in response.output if output.type == "image_generation_call"] + + # Verify image was generated + assert len(image_data) > 0, "Expected at least one image to be generated" + assert image_data[0], "Expected image data to be non-empty" + + print(f"โœ“ Image data received ({len(image_data[0])} base64 characters)") + + # Decode the base64 image + image_bytes = b"" + try: + image_bytes = base64.b64decode(image_data[0]) + assert len(image_bytes) > 0, "Decoded image should have content" + print(f"โœ“ Image decoded successfully ({len(image_bytes)} bytes)") + except Exception as e: + pytest.fail(f"Failed to decode base64 image data: {e}") + + # Verify it's a PNG image (check magic bytes) + # PNG files start with: 89 50 4E 47 (โ€ฐPNG) + assert image_bytes[:4] == b"\x89PNG", "Image does not appear to be a valid PNG" + print("โœ“ Image is a valid PNG") + + # Verify reasonable image size (should be > 1KB for a 1024x1024 image) + assert len(image_bytes) > 1024, f"Image seems too small ({len(image_bytes)} bytes)" + print(f"โœ“ Image size is reasonable ({len(image_bytes):,} bytes)") + + print("\nโœ“ Agent successfully generated and returned a valid image") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py index 1f51d2261293..d56d9b97adba 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py @@ -50,109 +50,109 @@ def test_agent_mcp_basic(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Create MCP tool that connects to a public GitHub repo via MCP server - mcp_tool = MCPTool( - server_label="api-specs", - server_url="https://gitmcp.io/Azure/azure-rest-api-specs", - require_approval="always", - ) - - tools: list[Tool] = [mcp_tool] - - # Create agent with MCP tool - agent = project_client.agents.create_version( - agent_name="mcp-basic-agent", - definition=PromptAgentDefinition( - model=model, - instructions="You are a helpful agent that can use MCP tools to assist users. Use the available MCP tools to answer questions and perform tasks.", - tools=tools, - ), - description="Agent for testing basic MCP functionality.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - assert agent.id is not None - assert agent.name == "mcp-basic-agent" - assert agent.version is not None - - # Create conversation - conversation = openai_client.conversations.create() - print(f"Created conversation (id: {conversation.id})") - assert conversation.id is not None - - # Send initial request that will trigger the MCP tool - print("\nAsking agent to summarize Azure REST API specs README...") - - response = openai_client.responses.create( - conversation=conversation.id, - input="Please summarize the Azure REST API specifications Readme", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - print(f"Initial response completed (id: {response.id})") - assert response.id is not None - assert response.output is not None - assert len(response.output) > 0 - - # Process any MCP approval requests - approval_requests_found = 0 - input_list: ResponseInputParam = [] - - for item in response.output: - if item.type == "mcp_approval_request": - approval_requests_found += 1 - print(f"Found MCP approval request (id: {item.id}, server: {item.server_label})") - - if item.server_label == "api-specs" and item.id: - # Approve the MCP request - input_list.append( - McpApprovalResponse( - type="mcp_approval_response", - approve=True, - approval_request_id=item.id, + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create MCP tool that connects to a public GitHub repo via MCP server + mcp_tool = MCPTool( + server_label="api-specs", + server_url="https://gitmcp.io/Azure/azure-rest-api-specs", + require_approval="always", + ) + + tools: list[Tool] = [mcp_tool] + + # Create agent with MCP tool + agent = project_client.agents.create_version( + agent_name="mcp-basic-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful agent that can use MCP tools to assist users. Use the available MCP tools to answer questions and perform tasks.", + tools=tools, + ), + description="Agent for testing basic MCP functionality.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "mcp-basic-agent" + assert agent.version is not None + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Created conversation (id: {conversation.id})") + assert conversation.id is not None + + # Send initial request that will trigger the MCP tool + print("\nAsking agent to summarize Azure REST API specs README...") + + response = openai_client.responses.create( + conversation=conversation.id, + input="Please summarize the Azure REST API specifications Readme", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Process any MCP approval requests + approval_requests_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "mcp_approval_request": + approval_requests_found += 1 + print(f"Found MCP approval request (id: {item.id}, server: {item.server_label})") + + if item.server_label == "api-specs" and item.id: + # Approve the MCP request + input_list.append( + McpApprovalResponse( + type="mcp_approval_response", + approve=True, + approval_request_id=item.id, + ) ) - ) - print(f"โœ“ Approved MCP request: {item.id}") + print(f"โœ“ Approved MCP request: {item.id}") - # Verify that at least one approval request was generated - assert ( - approval_requests_found > 0 - ), f"Expected at least 1 MCP approval request, but found {approval_requests_found}" + # Verify that at least one approval request was generated + assert ( + approval_requests_found > 0 + ), f"Expected at least 1 MCP approval request, but found {approval_requests_found}" - print(f"\nโœ“ Processed {approval_requests_found} MCP approval request(s)") + print(f"\nโœ“ Processed {approval_requests_found} MCP approval request(s)") - # Send the approval response to continue the agent's work - print("\nSending approval response to continue agent execution...") + # Send the approval response to continue the agent's work + print("\nSending approval response to continue agent execution...") - response = openai_client.responses.create( - input=input_list, - previous_response_id=response.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) - print(f"Final response completed (id: {response.id})") - assert response.id is not None + print(f"Final response completed (id: {response.id})") + assert response.id is not None - # Get the final response text - response_text = response.output_text - print(f"\nAgent's response preview: {response_text[:200]}...") + # Get the final response text + response_text = response.output_text + print(f"\nAgent's response preview: {response_text[:200]}...") - # Verify we got a meaningful response - assert len(response_text) > 100, "Expected a substantial response from the agent" + # Verify we got a meaningful response + assert len(response_text) > 100, "Expected a substantial response from the agent" - # Check that the response mentions Azure or REST API (indicating it accessed the repo) - assert any( - keyword in response_text.lower() for keyword in ["azure", "rest", "api", "specification"] - ), f"Expected response to mention Azure/REST API, but got: {response_text[:200]}" + # Check that the response mentions Azure or REST API (indicating it accessed the repo) + assert any( + keyword in response_text.lower() for keyword in ["azure", "rest", "api", "specification"] + ), f"Expected response to mention Azure/REST API, but got: {response_text[:200]}" - print("\nโœ“ Agent successfully used MCP tool to access GitHub repo and complete task") + print("\nโœ“ Agent successfully used MCP tool to access GitHub repo and complete task") - # Teardown - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") @servicePreparer() @pytest.mark.skipif( @@ -187,117 +187,117 @@ def test_agent_mcp_with_project_connection(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Get MCP project connection from environment - mcp_project_connection_id = kwargs.get("azure_ai_projects_tests_mcp_project_connection_id") - - if not mcp_project_connection_id: - pytest.skip("AZURE_AI_PROJECTS_TESTS_MCP_PROJECT_CONNECTION_ID environment variable not set") - - assert isinstance(mcp_project_connection_id, str), "mcp_project_connection_id must be a string" - print(f"Using MCP project connection: {mcp_project_connection_id}") - - # Create MCP tool with project connection for GitHub API access - mcp_tool = MCPTool( - server_label="github-api", - server_url="https://api.githubcopilot.com/mcp", - require_approval="always", - project_connection_id=mcp_project_connection_id, - ) - - tools: list[Tool] = [mcp_tool] - - # Create agent with MCP tool - agent = project_client.agents.create_version( - agent_name="mcp-connection-agent", - definition=PromptAgentDefinition( - model=model, - instructions="Use MCP tools as needed to access GitHub information.", - tools=tools, - ), - description="Agent for testing MCP with project connection.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - assert agent.id is not None - assert agent.name == "mcp-connection-agent" - assert agent.version is not None - - # Create conversation - conversation = openai_client.conversations.create() - print(f"Created conversation (id: {conversation.id})") - assert conversation.id is not None - - # Send initial request that will trigger the MCP tool with authentication - print("\nAsking agent to get GitHub profile username...") - - response = openai_client.responses.create( - conversation=conversation.id, - input="What is my username in Github profile?", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - print(f"Initial response completed (id: {response.id})") - assert response.id is not None - assert response.output is not None - assert len(response.output) > 0 - - # Process any MCP approval requests - approval_requests_found = 0 - input_list: ResponseInputParam = [] - - for item in response.output: - if item.type == "mcp_approval_request": - approval_requests_found += 1 - print(f"Found MCP approval request (id: {item.id}, server: {item.server_label})") - - if item.server_label == "github-api" and item.id: - # Approve the MCP request - input_list.append( - McpApprovalResponse( - type="mcp_approval_response", - approve=True, - approval_request_id=item.id, + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Get MCP project connection from environment + mcp_project_connection_id = kwargs.get("azure_ai_projects_tests_mcp_project_connection_id") + + if not mcp_project_connection_id: + pytest.skip("AZURE_AI_PROJECTS_TESTS_MCP_PROJECT_CONNECTION_ID environment variable not set") + + assert isinstance(mcp_project_connection_id, str), "mcp_project_connection_id must be a string" + print(f"Using MCP project connection: {mcp_project_connection_id}") + + # Create MCP tool with project connection for GitHub API access + mcp_tool = MCPTool( + server_label="github-api", + server_url="https://api.githubcopilot.com/mcp", + require_approval="always", + project_connection_id=mcp_project_connection_id, + ) + + tools: list[Tool] = [mcp_tool] + + # Create agent with MCP tool + agent = project_client.agents.create_version( + agent_name="mcp-connection-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Use MCP tools as needed to access GitHub information.", + tools=tools, + ), + description="Agent for testing MCP with project connection.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "mcp-connection-agent" + assert agent.version is not None + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Created conversation (id: {conversation.id})") + assert conversation.id is not None + + # Send initial request that will trigger the MCP tool with authentication + print("\nAsking agent to get GitHub profile username...") + + response = openai_client.responses.create( + conversation=conversation.id, + input="What is my username in Github profile?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Process any MCP approval requests + approval_requests_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "mcp_approval_request": + approval_requests_found += 1 + print(f"Found MCP approval request (id: {item.id}, server: {item.server_label})") + + if item.server_label == "github-api" and item.id: + # Approve the MCP request + input_list.append( + McpApprovalResponse( + type="mcp_approval_response", + approve=True, + approval_request_id=item.id, + ) ) - ) - print(f"โœ“ Approved MCP request: {item.id}") + print(f"โœ“ Approved MCP request: {item.id}") - # Verify that at least one approval request was generated - assert ( - approval_requests_found > 0 - ), f"Expected at least 1 MCP approval request, but found {approval_requests_found}" + # Verify that at least one approval request was generated + assert ( + approval_requests_found > 0 + ), f"Expected at least 1 MCP approval request, but found {approval_requests_found}" - print(f"\nโœ“ Processed {approval_requests_found} MCP approval request(s)") + print(f"\nโœ“ Processed {approval_requests_found} MCP approval request(s)") - # Send the approval response to continue the agent's work - print("\nSending approval response to continue agent execution...") + # Send the approval response to continue the agent's work + print("\nSending approval response to continue agent execution...") - response = openai_client.responses.create( - input=input_list, - previous_response_id=response.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) - print(f"Final response completed (id: {response.id})") - assert response.id is not None + print(f"Final response completed (id: {response.id})") + assert response.id is not None - # Get the final response text - response_text = response.output_text - print(f"\nAgent's response: {response_text}") + # Get the final response text + response_text = response.output_text + print(f"\nAgent's response: {response_text}") - # Verify we got a meaningful response with a GitHub username - assert len(response_text) > 5, "Expected a response with a GitHub username" + # Verify we got a meaningful response with a GitHub username + assert len(response_text) > 5, "Expected a response with a GitHub username" - # The response should contain some indication of a username or GitHub profile info - # We can't assert the exact username, but we can verify it's not an error - assert ( - "error" not in response_text.lower() or "username" in response_text.lower() - ), f"Expected response to contain GitHub profile info, but got: {response_text}" + # The response should contain some indication of a username or GitHub profile info + # We can't assert the exact username, but we can verify it's not an error + assert ( + "error" not in response_text.lower() or "username" in response_text.lower() + ), f"Expected response to contain GitHub profile info, but got: {response_text}" - print("\nโœ“ Agent successfully used MCP tool with project connection to access authenticated GitHub API") + print("\nโœ“ Agent successfully used MCP tool with project connection to access authenticated GitHub API") - # Teardown - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py index cb322e771714..2ec7fc478556 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py @@ -44,152 +44,152 @@ def test_function_tool_with_conversation(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Define a calculator function - calculator = FunctionTool( - name="calculator", - description="Perform basic arithmetic operations", - parameters={ - "type": "object", - "properties": { - "operation": { - "type": "string", - "enum": ["add", "subtract", "multiply", "divide"], - "description": "The operation to perform", + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Define a calculator function + calculator = FunctionTool( + name="calculator", + description="Perform basic arithmetic operations", + parameters={ + "type": "object", + "properties": { + "operation": { + "type": "string", + "enum": ["add", "subtract", "multiply", "divide"], + "description": "The operation to perform", + }, + "a": {"type": "number", "description": "First number"}, + "b": {"type": "number", "description": "Second number"}, }, - "a": {"type": "number", "description": "First number"}, - "b": {"type": "number", "description": "Second number"}, + "required": ["operation", "a", "b"], + "additionalProperties": False, }, - "required": ["operation", "a", "b"], - "additionalProperties": False, - }, - strict=True, - ) - - # Create agent - agent = project_client.agents.create_version( - agent_name="calculator-agent-conversation", - definition=PromptAgentDefinition( - model=model, - instructions="You are a calculator assistant. Use the calculator function to perform operations.", - tools=[calculator], - ), - description="Calculator agent for conversation testing.", - ) - print(f"Agent created: {agent.id}") - - # Create conversation - conversation = openai_client.conversations.create() - print(f"Conversation created: {conversation.id}") - - # Turn 1: Add two numbers - print("\n--- Turn 1: Addition ---") - response_1 = openai_client.responses.create( - input="What is 15 plus 27?", - conversation=conversation.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - # Handle function call - input_list: ResponseInputParam = [] - for item in response_1.output: - if item.type == "function_call": - print(f"Function called: {item.name} with {item.arguments}") - args = json.loads(item.arguments) - - # Execute calculator - result = { - "add": args["a"] + args["b"], - "subtract": args["a"] - args["b"], - "multiply": args["a"] * args["b"], - "divide": args["a"] / args["b"] if args["b"] != 0 else "Error: Division by zero", - }[args["operation"]] - - input_list.append( - FunctionCallOutput( - type="function_call_output", - call_id=item.call_id, - output=json.dumps({"result": result}), + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="calculator-agent-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are a calculator assistant. Use the calculator function to perform operations.", + tools=[calculator], + ), + description="Calculator agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Add two numbers + print("\n--- Turn 1: Addition ---") + response_1 = openai_client.responses.create( + input="What is 15 plus 27?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_1.output: + if item.type == "function_call": + print(f"Function called: {item.name} with {item.arguments}") + args = json.loads(item.arguments) + + # Execute calculator + result = { + "add": args["a"] + args["b"], + "subtract": args["a"] - args["b"], + "multiply": args["a"] * args["b"], + "divide": args["a"] / args["b"] if args["b"] != 0 else "Error: Division by zero", + }[args["operation"]] + + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"result": result}), + ) ) - ) - - response_1 = openai_client.responses.create( - input=input_list, - conversation=conversation.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - print(f"Response 1: {response_1.output_text[:100]}...") - assert "42" in response_1.output_text - - # Turn 2: Follow-up using previous result (tests conversation memory) - print("\n--- Turn 2: Follow-up using conversation context ---") - response_2 = openai_client.responses.create( - input="Now multiply that result by 2", - conversation=conversation.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - # Handle function call - input_list = [] - for item in response_2.output: - if item.type == "function_call": - print(f"Function called: {item.name} with {item.arguments}") - args = json.loads(item.arguments) - - # Should be multiplying 42 by 2 - assert args["operation"] == "multiply" - assert args["a"] == 42 or args["b"] == 42 - - result = args["a"] * args["b"] - input_list.append( - FunctionCallOutput( - type="function_call_output", - call_id=item.call_id, - output=json.dumps({"result": result}), + + response_1 = openai_client.responses.create( + input=input_list, + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response 1: {response_1.output_text[:100]}...") + assert "42" in response_1.output_text + + # Turn 2: Follow-up using previous result (tests conversation memory) + print("\n--- Turn 2: Follow-up using conversation context ---") + response_2 = openai_client.responses.create( + input="Now multiply that result by 2", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list = [] + for item in response_2.output: + if item.type == "function_call": + print(f"Function called: {item.name} with {item.arguments}") + args = json.loads(item.arguments) + + # Should be multiplying 42 by 2 + assert args["operation"] == "multiply" + assert args["a"] == 42 or args["b"] == 42 + + result = args["a"] * args["b"] + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"result": result}), + ) ) - ) - - response_2 = openai_client.responses.create( - input=input_list, - conversation=conversation.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - print(f"Response 2: {response_2.output_text[:100]}...") - assert "84" in response_2.output_text - - print("\nโœ“ Function tool with conversation successful!") - print(" - Conversation preserved state across function calls") - print(" - Agent remembered previous result (42)") - - # Verify conversation state by reading items - print("\n--- Verifying conversation state ---") - all_items = list(openai_client.conversations.items.list(conversation.id)) - print(f"Total conversation items: {len(all_items)}") - - # Count different item types - user_messages = sum(1 for item in all_items if item.type == "message" and item.role == "user") - assistant_messages = sum(1 for item in all_items if item.type == "message" and item.role == "assistant") - function_calls = sum(1 for item in all_items if item.type == "function_call") - function_outputs = sum(1 for item in all_items if item.type == "function_call_output") - - print(f" User messages: {user_messages}") - print(f" Assistant messages: {assistant_messages}") - print(f" Function calls: {function_calls}") - print(f" Function outputs: {function_outputs}") - - # Verify we have expected items - assert user_messages >= 2, "Expected at least 2 user messages (two turns)" - assert function_calls >= 2, "Expected at least 2 function calls (one per turn)" - assert function_outputs >= 2, "Expected at least 2 function outputs" - print("โœ“ Conversation state verified - all items preserved") - - # Cleanup - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - openai_client.conversations.delete(conversation_id=conversation.id) - print("Cleanup completed") + + response_2 = openai_client.responses.create( + input=input_list, + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response 2: {response_2.output_text[:100]}...") + assert "84" in response_2.output_text + + print("\nโœ“ Function tool with conversation successful!") + print(" - Conversation preserved state across function calls") + print(" - Agent remembered previous result (42)") + + # Verify conversation state by reading items + print("\n--- Verifying conversation state ---") + all_items = list(openai_client.conversations.items.list(conversation.id)) + print(f"Total conversation items: {len(all_items)}") + + # Count different item types + user_messages = sum(1 for item in all_items if item.type == "message" and item.role == "user") + assistant_messages = sum(1 for item in all_items if item.type == "message" and item.role == "assistant") + function_calls = sum(1 for item in all_items if item.type == "function_call") + function_outputs = sum(1 for item in all_items if item.type == "function_call_output") + + print(f" User messages: {user_messages}") + print(f" Assistant messages: {assistant_messages}") + print(f" Function calls: {function_calls}") + print(f" Function outputs: {function_outputs}") + + # Verify we have expected items + assert user_messages >= 2, "Expected at least 2 user messages (two turns)" + assert function_calls >= 2, "Expected at least 2 function calls (one per turn)" + assert function_outputs >= 2, "Expected at least 2 function outputs" + print("โœ“ Conversation state verified - all items preserved") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + print("Cleanup completed") @servicePreparer() @pytest.mark.skipif( @@ -208,12 +208,12 @@ def test_file_search_with_conversation(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Create documents with related information - doc_content = """Product Catalog + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create documents with related information + doc_content = """Product Catalog Widget A: - Price: $100 @@ -234,82 +234,82 @@ def test_file_search_with_conversation(self, **kwargs): - Category: Home Goods """ - # Create vector store and upload document - vector_store = openai_client.vector_stores.create(name="ConversationTestStore") - print(f"Vector store created: {vector_store.id}") - - from io import BytesIO - - file = BytesIO(doc_content.encode("utf-8")) - file.name = "products.txt" - - uploaded = openai_client.vector_stores.files.upload_and_poll( - vector_store_id=vector_store.id, - file=file, - ) - print(f"Document uploaded: {uploaded.id}") - - # Create agent with file search - agent = project_client.agents.create_version( - agent_name="search-agent-conversation", - definition=PromptAgentDefinition( - model=model, - instructions="You are a product search assistant. Answer questions about products.", - tools=[FileSearchTool(vector_store_ids=[vector_store.id])], - ), - description="Search agent for conversation testing.", - ) - print(f"Agent created: {agent.id}") - - # Create conversation - conversation = openai_client.conversations.create() - print(f"Conversation created: {conversation.id}") - - # Turn 1: Search for highest rated - print("\n--- Turn 1: Search query ---") - response_1 = openai_client.responses.create( - input="Which widget has the highest rating?", - conversation=conversation.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_1_text = response_1.output_text - print(f"Response 1: {response_1_text[:150]}...") - assert "Widget B" in response_1_text or "4.8" in response_1_text - - # Turn 2: Follow-up about that specific product (tests context retention) - print("\n--- Turn 2: Contextual follow-up ---") - response_2 = openai_client.responses.create( - input="What is its price?", # "its" refers to Widget B from previous turn - conversation=conversation.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_2_text = response_2.output_text - print(f"Response 2: {response_2_text[:150]}...") - assert "220" in response_2_text - - # Turn 3: New search in same conversation - print("\n--- Turn 3: New search in same conversation ---") - response_3 = openai_client.responses.create( - input="Which widget is in the Home Goods category?", - conversation=conversation.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_3_text = response_3.output_text - print(f"Response 3: {response_3_text[:150]}...") - assert "Widget C" in response_3_text - - print("\nโœ“ File search with conversation successful!") - print(" - Multiple searches in same conversation") - print(" - Context preserved (agent remembered Widget B)") - - # Cleanup - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - openai_client.conversations.delete(conversation_id=conversation.id) - openai_client.vector_stores.delete(vector_store.id) - print("Cleanup completed") + # Create vector store and upload document + vector_store = openai_client.vector_stores.create(name="ConversationTestStore") + print(f"Vector store created: {vector_store.id}") + + from io import BytesIO + + file = BytesIO(doc_content.encode("utf-8")) + file.name = "products.txt" + + uploaded = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=file, + ) + print(f"Document uploaded: {uploaded.id}") + + # Create agent with file search + agent = project_client.agents.create_version( + agent_name="search-agent-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are a product search assistant. Answer questions about products.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Search agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Search for highest rated + print("\n--- Turn 1: Search query ---") + response_1 = openai_client.responses.create( + input="Which widget has the highest rating?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:150]}...") + assert "Widget B" in response_1_text or "4.8" in response_1_text + + # Turn 2: Follow-up about that specific product (tests context retention) + print("\n--- Turn 2: Contextual follow-up ---") + response_2 = openai_client.responses.create( + input="What is its price?", # "its" refers to Widget B from previous turn + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:150]}...") + assert "220" in response_2_text + + # Turn 3: New search in same conversation + print("\n--- Turn 3: New search in same conversation ---") + response_3 = openai_client.responses.create( + input="Which widget is in the Home Goods category?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:150]}...") + assert "Widget C" in response_3_text + + print("\nโœ“ File search with conversation successful!") + print(" - Multiple searches in same conversation") + print(" - Context preserved (agent remembered Widget B)") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") @servicePreparer() @pytest.mark.skipif( @@ -328,68 +328,68 @@ def test_code_interpreter_with_conversation(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Create agent with code interpreter - agent = project_client.agents.create_version( - agent_name="code-agent-conversation", - definition=PromptAgentDefinition( - model=model, - instructions="You are a data analysis assistant. Use Python to perform calculations.", - tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[]))], - ), - description="Code interpreter agent for conversation testing.", - ) - print(f"Agent created: {agent.id}") - - # Create conversation - conversation = openai_client.conversations.create() - print(f"Conversation created: {conversation.id}") - - # Turn 1: Calculate average - print("\n--- Turn 1: Calculate average ---") - response_1 = openai_client.responses.create( - input="Calculate the average of these numbers: 10, 20, 30, 40, 50", - conversation=conversation.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_1_text = response_1.output_text - print(f"Response 1: {response_1_text[:200]}...") - assert "30" in response_1_text - - # Turn 2: Follow-up calculation (tests conversation context) - print("\n--- Turn 2: Follow-up calculation ---") - response_2 = openai_client.responses.create( - input="Now calculate the standard deviation of those same numbers", - conversation=conversation.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_2_text = response_2.output_text - print(f"Response 2: {response_2_text[:200]}...") - # Standard deviation should be approximately 14.14 or similar - assert any(num in response_2_text for num in ["14", "15", "standard"]) - - # Turn 3: Another operation in same conversation - print("\n--- Turn 3: New calculation ---") - response_3 = openai_client.responses.create( - input="Create a list of squares from 1 to 5", - conversation=conversation.id, - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - response_3_text = response_3.output_text - print(f"Response 3: {response_3_text[:200]}...") - assert "1" in response_3_text and "4" in response_3_text and "25" in response_3_text - - print("\nโœ“ Code interpreter with conversation successful!") - print(" - Multiple code executions in conversation") - print(" - Context preserved across calculations") - - # Cleanup - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - openai_client.conversations.delete(conversation_id=conversation.id) - print("Cleanup completed") + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with code interpreter + agent = project_client.agents.create_version( + agent_name="code-agent-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are a data analysis assistant. Use Python to perform calculations.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[]))], + ), + description="Code interpreter agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Calculate average + print("\n--- Turn 1: Calculate average ---") + response_1 = openai_client.responses.create( + input="Calculate the average of these numbers: 10, 20, 30, 40, 50", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "30" in response_1_text + + # Turn 2: Follow-up calculation (tests conversation context) + print("\n--- Turn 2: Follow-up calculation ---") + response_2 = openai_client.responses.create( + input="Now calculate the standard deviation of those same numbers", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + # Standard deviation should be approximately 14.14 or similar + assert any(num in response_2_text for num in ["14", "15", "standard"]) + + # Turn 3: Another operation in same conversation + print("\n--- Turn 3: New calculation ---") + response_3 = openai_client.responses.create( + input="Create a list of squares from 1 to 5", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + assert "1" in response_3_text and "4" in response_3_text and "25" in response_3_text + + print("\nโœ“ Code interpreter with conversation successful!") + print(" - Multiple code executions in conversation") + print(" - Context preserved across calculations") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + print("Cleanup completed") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py index a0359f7c2cdc..30e896825e64 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py @@ -43,57 +43,57 @@ def test_agent_web_search(self, **kwargs): model = self.test_agents_params["model_deployment_name"] - # Setup - project_client = self.create_client(operation_group="agents", **kwargs) - openai_client = project_client.get_openai_client() - - # Create agent with web search tool - agent = project_client.agents.create_version( - agent_name="web-search-agent", - definition=PromptAgentDefinition( - model=model, - instructions="You are a helpful assistant that can search the web for current information.", - tools=[ - WebSearchPreviewTool( - user_location=ApproximateLocation(country="US", city="Seattle", region="Washington") - ) - ], - ), - description="Agent for testing web search capabilities.", - ) - print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") - assert agent.id is not None - assert agent.name == "web-search-agent" - assert agent.version is not None - - # Ask a question that requires web search for current information - print("\nAsking agent about current weather...") - - response = openai_client.responses.create( - input="What is the current weather in Seattle?", - extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, - ) - - print(f"Response completed (id: {response.id})") - assert response.id is not None - assert response.output is not None - assert len(response.output) > 0 - - # Get the response text - response_text = response.output_text - print(f"\nAgent's response: {response_text[:300]}...") - - # Verify we got a meaningful response - assert len(response_text) > 30, "Expected a substantial response from the agent" - - # The response should mention weather-related terms or Seattle - response_lower = response_text.lower() - assert any( - keyword in response_lower for keyword in ["weather", "temperature", "seattle", "forecast"] - ), f"Expected response to contain weather information, but got: {response_text[:200]}" - - print("\nโœ“ Agent successfully used web search tool to get current information") - - # Teardown - project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) - print("Agent deleted") + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with web search tool + agent = project_client.agents.create_version( + agent_name="web-search-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can search the web for current information.", + tools=[ + WebSearchPreviewTool( + user_location=ApproximateLocation(country="US", city="Seattle", region="Washington") + ) + ], + ), + description="Agent for testing web search capabilities.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "web-search-agent" + assert agent.version is not None + + # Ask a question that requires web search for current information + print("\nAsking agent about current weather...") + + response = openai_client.responses.create( + input="What is the current weather in Seattle?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Get the response text + response_text = response.output_text + print(f"\nAgent's response: {response_text[:300]}...") + + # Verify we got a meaningful response + assert len(response_text) > 30, "Expected a substantial response from the agent" + + # The response should mention weather-related terms or Seattle + response_lower = response_text.lower() + assert any( + keyword in response_lower for keyword in ["weather", "temperature", "seattle", "forecast"] + ), f"Expected response to contain weather information, but got: {response_text[:200]}" + + print("\nโœ“ Agent successfully used web search tool to get current information") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") From 0101ab268b1039d2a14b6f42260f0696906dc086 Mon Sep 17 00:00:00 2001 From: Paul Batum Date: Fri, 21 Nov 2025 18:10:02 -0800 Subject: [PATCH 7/8] add test case for conversation + code interpreter + file upload --- .../test_agent_tools_with_conversations.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py index 2ec7fc478556..bad368125fdc 100644 --- a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py @@ -393,3 +393,76 @@ def test_code_interpreter_with_conversation(self, **kwargs): project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) openai_client.conversations.delete(conversation_id=conversation.id) print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_code_interpreter_with_file_in_conversation(self, **kwargs): + """ + Test using CodeInterpreterTool with file upload within a conversation. + + This test reproduces the 500 error seen in the sample when using + code interpreter with uploaded files in conversations. + + This tests: + - Uploading a real file (not BytesIO) for code interpreter + - Using code interpreter with files in conversation + - Server-side code execution with file access and chart generation + """ + + model = self.test_agents_params["model_deployment_name"] + import os + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Use the same CSV file as the sample + asset_file_path = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "../../../samples/agents/assets/synthetic_500_quarterly_results.csv", + ) + ) + + # Upload file using open() with rb mode, just like the sample + with open(asset_file_path, "rb") as f: + uploaded_file = openai_client.files.create(file=f, purpose="assistants") + print(f"File uploaded: {uploaded_file.id}") + + # Create agent with code interpreter - matching sample exactly + agent = project_client.agents.create_version( + agent_name="agent-code-interpreter-with-file-pbatum1", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[uploaded_file.id]))], + ), + description="Code interpreter agent for data analysis and visualization.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Use the same prompt as the sample - requesting chart generation + print("\n--- Turn 1: Create bar chart ---") + response_1 = openai_client.responses.create( + conversation=conversation.id, + input="Could you please create bar chart in TRANSPORTATION sector for the operating profit from the uploaded csv file and provide file to me?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + + print("\nโœ“ Code interpreter with file in conversation successful!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + openai_client.files.delete(uploaded_file.id) + print("Cleanup completed") From 601677ad0a4c6d85db79b0898be92346f0ad0129 Mon Sep 17 00:00:00 2001 From: Paul Batum Date: Fri, 21 Nov 2025 18:17:34 -0800 Subject: [PATCH 8/8] add image generation model to env template --- sdk/ai/azure-ai-projects/.env.template | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/ai/azure-ai-projects/.env.template b/sdk/ai/azure-ai-projects/.env.template index ea2b0c167ea6..9635fbe711bd 100644 --- a/sdk/ai/azure-ai-projects/.env.template +++ b/sdk/ai/azure-ai-projects/.env.template @@ -50,6 +50,9 @@ AZURE_AI_PROJECTS_TESTS_AI_SEARCH_PROJECT_CONNECTION_ID= AZURE_AI_PROJECTS_TESTS_AI_SEARCH_INDEX_NAME= AZURE_AI_PROJECTS_TESTS_MCP_PROJECT_CONNECTION_ID= +# Used in Image generation agent tools tests +AZURE_AI_PROJECTS_TESTS_IMAGE_MODEL_DEPLOYMENT_NAME= + # Used in tools BING_PROJECT_CONNECTION_ID= MCP_PROJECT_CONNECTION_ID=