diff --git a/sdk/ai/azure-ai-projects/.env.template b/sdk/ai/azure-ai-projects/.env.template index 4c5e51b6c5d3..9635fbe711bd 100644 --- a/sdk/ai/azure-ai-projects/.env.template +++ b/sdk/ai/azure-ai-projects/.env.template @@ -44,6 +44,15 @@ AZURE_AI_PROJECTS_TESTS_CONTAINER_PROJECT_ENDPOINT= AZURE_AI_PROJECTS_TESTS_CONTAINER_APP_RESOURCE_ID= AZURE_AI_PROJECTS_TESTS_CONTAINER_INGRESS_SUBDOMAIN_SUFFIX= +# Connection IDs and settings used in agent tool tests +AZURE_AI_PROJECTS_TESTS_BING_PROJECT_CONNECTION_ID= +AZURE_AI_PROJECTS_TESTS_AI_SEARCH_PROJECT_CONNECTION_ID= +AZURE_AI_PROJECTS_TESTS_AI_SEARCH_INDEX_NAME= +AZURE_AI_PROJECTS_TESTS_MCP_PROJECT_CONNECTION_ID= + +# Used in Image generation agent tools tests +AZURE_AI_PROJECTS_TESTS_IMAGE_MODEL_DEPLOYMENT_NAME= + # Used in tools BING_PROJECT_CONNECTION_ID= MCP_PROJECT_CONNECTION_ID= diff --git a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_with_project_connection.py b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_with_project_connection.py index c5a62a3213c1..3534a00ae3eb 100644 --- a/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_with_project_connection.py +++ b/sdk/ai/azure-ai-projects/samples/agents/tools/sample_agent_mcp_with_project_connection.py @@ -22,7 +22,7 @@ 2) AZURE_AI_MODEL_DEPLOYMENT_NAME - The deployment name of the AI model, as found under the "Name" column in the "Models + endpoints" tab in your Microsoft Foundry project. 3) MCP_PROJECT_CONNECTION_ID - The connection resource ID in Custom keys - with key equals to "Authorization" and value to be "Bear ". + with key equals to "Authorization" and value to be "Bearer ". Token can be created in https://github.com/settings/personal-access-tokens/new """ diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/__init__.py b/sdk/ai/azure-ai-projects/tests/agents/tools/__init__.py new file mode 100644 index 000000000000..b74cfa3b899c --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/__init__.py @@ -0,0 +1,4 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py new file mode 100644 index 000000000000..69033a5a3917 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_code_interpreter_and_function.py @@ -0,0 +1,144 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Multi-Tool Tests: Code Interpreter + Function Tool + +Tests various scenarios using an agent with Code Interpreter and Function Tool. +All tests use the same tool combination but different inputs and workflows. +""" + +import os +import json +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, CodeInterpreterTool, CodeInterpreterToolAuto, FunctionTool +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestAgentCodeInterpreterAndFunction(TestBase): + """Tests for agents using Code Interpreter + Function Tool combination.""" + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_calculate_and_save(self, **kwargs): + """ + Test calculation with Code Interpreter and saving with Function Tool. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Define function tool + func_tool = FunctionTool( + name="save_result", + description="Save analysis result", + parameters={ + "type": "object", + "properties": { + "result": {"type": "string", "description": "The result"}, + }, + "required": ["result"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="code-func-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Run calculations and save results.", + tools=[ + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + func_tool, + ], + ), + description="Agent with Code Interpreter and Function Tool.", + ) + print(f"Agent created (id: {agent.id})") + + # Use the agent + response = openai_client.responses.create( + input="Calculate 5 + 3 and save the result.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response received (id: {response.id})") + + assert response.id is not None + print("✓ Code Interpreter + Function Tool works!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_generate_data_and_report(self, **kwargs): + """ + Test generating data with Code Interpreter and reporting with Function. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Define function tool + report_function = FunctionTool( + name="generate_report", + description="Generate a report with the provided data", + parameters={ + "type": "object", + "properties": { + "title": {"type": "string", "description": "Report title"}, + "summary": {"type": "string", "description": "Report summary"}, + }, + "required": ["title", "summary"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="code-func-report-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Generate data using code and create reports with the generate_report function.", + tools=[ + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + report_function, + ], + ), + description="Agent for data generation and reporting.", + ) + print(f"Agent created (id: {agent.id})") + + # Request data generation and report + response = openai_client.responses.create( + input="Generate a list of 10 random numbers between 1 and 100, calculate their average, and create a report.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response received (id: {response.id})") + assert response.id is not None + print("✓ Data generation and reporting works!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py new file mode 100644 index 000000000000..d673e31f6419 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_code_interpreter.py @@ -0,0 +1,156 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Multi-Tool Tests: File Search + Code Interpreter + +Tests various scenarios using an agent with File Search and Code Interpreter. +All tests use the same tool combination but different inputs and workflows. +""" + +import os +import pytest +from io import BytesIO +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool, CodeInterpreterTool, CodeInterpreterToolAuto + + +class TestAgentFileSearchAndCodeInterpreter(TestBase): + """Tests for agents using File Search + Code Interpreter combination.""" + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_find_and_analyze_data(self, **kwargs): + """ + Test finding data with File Search and analyzing with Code Interpreter. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create data file + txt_content = "Sample data: 10, 20, 30, 40, 50" + vector_store = openai_client.vector_stores.create(name="DataStore") + + txt_file = BytesIO(txt_content.encode("utf-8")) + txt_file.name = "data.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=txt_file, + ) + print(f"File uploaded (id: {file.id})") + + # Create agent + agent = project_client.agents.create_version( + agent_name="file-search-code-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Find data and analyze it.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + ], + ), + description="Agent with File Search and Code Interpreter.", + ) + print(f"Agent created (id: {agent.id})") + + # Use the agent + response = openai_client.responses.create( + input="Find the data file and calculate the average.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response received (id: {response.id})") + + assert response.id is not None + assert len(response.output_text) > 20 + print("✓ File Search + Code Interpreter works!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_analyze_code_file(self, **kwargs): + """ + Test finding code file and analyzing it. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create Python code file + python_code = """def fibonacci(n): + if n <= 1: + return n + return fibonacci(n-1) + fibonacci(n-2) + +result = fibonacci(10) +print(f"Fibonacci(10) = {result}") +""" + + vector_store = openai_client.vector_stores.create(name="CodeAnalysisStore") + + code_file = BytesIO(python_code.encode("utf-8")) + code_file.name = "fibonacci.py" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=code_file, + ) + print(f"Code file uploaded (id: {file.id})") + + # Create agent + agent = project_client.agents.create_version( + agent_name="file-search-code-analysis-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Find code files and analyze them. You can run code to test it.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + ], + ), + description="Agent for code analysis.", + ) + print(f"Agent created (id: {agent.id})") + + # Request analysis + response = openai_client.responses.create( + input="Find the fibonacci code and explain what it does. What is the computational complexity?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_text = response.output_text + print(f"Response: {response_text[:300]}...") + + assert len(response_text) > 50 + response_lower = response_lower = response_text.lower() + assert any( + keyword in response_lower for keyword in ["fibonacci", "recursive", "complexity", "exponential"] + ), "Expected analysis of fibonacci algorithm" + + print("✓ Code file analysis completed") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py new file mode 100644 index 000000000000..4e65fae75254 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_and_function.py @@ -0,0 +1,519 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Multi-Tool Tests: File Search + Function Tool + +Tests various scenarios using an agent with File Search and Function Tool. +All tests use the same tool combination but different inputs and workflows. +""" + +import os +import json +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool, FunctionTool +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestAgentFileSearchAndFunction(TestBase): + """Tests for agents using File Search + Function Tool combination.""" + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_data_analysis_workflow(self, **kwargs): + """ + Test data analysis workflow: upload data, search, save results. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create sample data file + txt_content = """Sales Data Q1-Q3: + +Product: Widget A +Q1: $15,000 +Q2: $18,000 +Q3: $21,000 +Total: $54,000 + +Product: Widget B +Q1: $22,000 +Q2: $25,000 +Q3: $28,000 +Total: $75,000 + +Overall Total Revenue: $129,000 +""" + + # Create vector store and upload + vector_store = openai_client.vector_stores.create(name="SalesDataStore") + print(f"Vector store created (id: {vector_store.id})") + + txt_file = BytesIO(txt_content.encode("utf-8")) + txt_file.name = "sales_data.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=txt_file, + ) + print(f"File uploaded (id: {file.id})") + assert file.status == "completed" + + # Define function tool + save_results_function = FunctionTool( + name="save_analysis_results", + description="Save the analysis results", + parameters={ + "type": "object", + "properties": { + "summary": { + "type": "string", + "description": "Summary of the analysis", + }, + }, + "required": ["summary"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="file-search-function-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a data analyst. Use file search to find data and save_analysis_results function to save your findings.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + save_results_function, + ], + ), + description="Agent with File Search and Function Tool.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + + # Request analysis + print("\nAsking agent to analyze the sales data...") + + response = openai_client.responses.create( + input="Analyze the sales data and calculate the total revenue for each product. Then save the results.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + + # Check if function was called + function_calls_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "function_call": + function_calls_found += 1 + print(f"Function call detected (id: {item.call_id}, name: {item.name})") + + assert item.name == "save_analysis_results" + + arguments = json.loads(item.arguments) + print(f"Function arguments: {arguments}") + + assert "summary" in arguments + assert len(arguments["summary"]) > 20 + + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"status": "success", "saved": True}), + ) + ) + + assert function_calls_found > 0, "Expected save_analysis_results function to be called" + + # Send function results back + if input_list: + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Final response: {response.output_text[:200]}...") + + print("\n✓ Workflow completed successfully") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_empty_vector_store_handling(self, **kwargs): + """ + Test how agent handles empty vector store (no files uploaded). + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create EMPTY vector store + vector_store = openai_client.vector_stores.create(name="EmptyStore") + print(f"Empty vector store created (id: {vector_store.id})") + + # Define function tool + error_function = FunctionTool( + name="report_error", + description="Report when data is not available", + parameters={ + "type": "object", + "properties": { + "error_message": { + "type": "string", + "description": "Description of the error", + }, + }, + "required": ["error_message"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="file-search-function-error-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Search for data. If you can't find it, use report_error function to explain what happened.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + error_function, + ], + ), + description="Agent for testing error handling.", + ) + print(f"Agent created (id: {agent.id})") + + # Request analysis of non-existent file + print("\nAsking agent to find non-existent data...") + + response = openai_client.responses.create( + input="Find and analyze the quarterly sales report.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_text = response.output_text + print(f"Response: '{response_text[:200] if response_text else '(empty)'}...'") + + # Verify agent didn't crash + assert response.id is not None, "Agent should return a valid response" + assert len(response.output) >= 0, "Agent should return output items" + + # If there's text, it should be meaningful + if response_text: + assert len(response_text) > 10, "Non-empty response should be meaningful" + + print("\n✓ Agent handled missing data gracefully") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_python_code_file_search(self, **kwargs): + """ + Test searching for Python code files. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create sample Python code + python_code = """# Sample Python code for analysis +def calculate_sum(numbers): + return sum(numbers) + +# Test data +data = [1, 2, 3, 4, 5] +result = calculate_sum(data) +print(f"Sum: {result}") +""" + + # Create vector store and upload + vector_store = openai_client.vector_stores.create(name="CodeStore") + + code_file = BytesIO(python_code.encode("utf-8")) + code_file.name = "sample_code.py" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=code_file, + ) + print(f"Python file uploaded (id: {file.id})") + + # Define function tool + save_function = FunctionTool( + name="save_code_review", + description="Save code review findings", + parameters={ + "type": "object", + "properties": { + "findings": { + "type": "string", + "description": "Code review findings", + }, + }, + "required": ["findings"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="file-search-function-code-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You can search for code files and describe what they do. Save your findings.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + save_function, + ], + ), + description="Agent for testing file search with code.", + ) + print(f"Agent created (id: {agent.id})") + + # Request code analysis + print("\nAsking agent to find and analyze the Python code...") + + response = openai_client.responses.create( + input="Find the Python code file and tell me what the calculate_sum function does.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_text = response.output_text + print(f"Response: {response_text[:300]}...") + + # Verify agent found and analyzed the code + assert len(response_text) > 50, "Expected detailed analysis" + + response_lower = response_text.lower() + assert any( + keyword in response_lower for keyword in ["sum", "calculate", "function", "numbers", "code", "python"] + ), "Expected response to discuss the code" + + print("\n✓ Agent successfully found code file using File Search") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_multi_turn_search_and_save_workflow(self, **kwargs): + """ + Test multi-turn workflow: search documents, ask follow-ups, save findings. + + This tests: + - File Search across multiple turns + - Function calls interspersed with searches + - Context retention across searches and function calls + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create research documents + doc1_content = """Research Paper: Machine Learning in Healthcare + +Abstract: +Machine learning algorithms are revolutionizing healthcare diagnostics. +Recent studies show 95% accuracy in disease detection using neural networks. +Key applications include medical imaging, patient risk prediction, and drug discovery. + +Conclusion: +ML techniques offer promising solutions for early disease detection and personalized treatment. +""" + + doc2_content = """Research Paper: AI Ethics and Governance + +Abstract: +As AI systems become more prevalent, ethical considerations are paramount. +Issues include bias in algorithms, data privacy, and accountability. +Regulatory frameworks are being developed globally to address these concerns. + +Conclusion: +Responsible AI development requires multistakeholder collaboration and transparent governance. +""" + + # Create vector store and upload documents + vector_store = openai_client.vector_stores.create(name="ResearchStore") + print(f"Vector store created: {vector_store.id}") + + from io import BytesIO + + file1 = BytesIO(doc1_content.encode("utf-8")) + file1.name = "ml_healthcare.txt" + file2 = BytesIO(doc2_content.encode("utf-8")) + file2.name = "ai_ethics.txt" + + uploaded1 = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=file1, + ) + uploaded2 = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=file2, + ) + print(f"Documents uploaded: {uploaded1.id}, {uploaded2.id}") + + # Define save function + save_finding = FunctionTool( + name="save_finding", + description="Save research finding", + parameters={ + "type": "object", + "properties": { + "topic": {"type": "string", "description": "Research topic"}, + "finding": {"type": "string", "description": "Key finding"}, + }, + "required": ["topic", "finding"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="research-assistant-multi-turn", + definition=PromptAgentDefinition( + model=model, + instructions="You are a research assistant. Search documents and save important findings.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + save_finding, + ], + ), + description="Research assistant for multi-turn testing.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Search for ML in healthcare + print("\n--- Turn 1: Initial search query ---") + response_1 = openai_client.responses.create( + input="What does the research say about machine learning in healthcare?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "95" in response_1_text or "accuracy" in response_1_text.lower() + + # Turn 2: Follow-up for specifics + print("\n--- Turn 2: Follow-up for details ---") + response_2 = openai_client.responses.create( + input="What specific applications are mentioned?", + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + response_2_lower = response_2_text.lower() + assert any(keyword in response_2_lower for keyword in ["imaging", "drug", "risk", "prediction"]) + + # Turn 3: Save the finding + print("\n--- Turn 3: Save finding ---") + response_3 = openai_client.responses.create( + input="Please save that finding about ML accuracy.", + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + function_called = False + for item in response_3.output: + if item.type == "function_call": + function_called = True + print(f"Function called: {item.name} with args: {item.arguments}") + assert item.name == "save_finding" + + args = json.loads(item.arguments) + assert "topic" in args and "finding" in args + print(f" Topic: {args['topic']}") + print(f" Finding: {args['finding'][:100]}...") + + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"status": "saved", "id": "finding_001"}), + ) + ) + + assert function_called, "Expected save_finding to be called" + + # Send function result + response_3 = openai_client.responses.create( + input=input_list, + previous_response_id=response_3.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response 3: {response_3.output_text[:150]}...") + + # Turn 4: Switch to different topic (AI ethics) + print("\n--- Turn 4: New search topic ---") + response_4 = openai_client.responses.create( + input="Now tell me about AI ethics concerns mentioned in the research.", + previous_response_id=response_3.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_4_text = response_4.output_text + print(f"Response 4: {response_4_text[:200]}...") + response_4_lower = response_4_text.lower() + assert any(keyword in response_4_lower for keyword in ["bias", "privacy", "ethics", "accountability"]) + + print("\n✓ Multi-turn File Search + Function workflow successful!") + print(" - Multiple searches across different documents") + print(" - Function called after context-building searches") + print(" - Topic switching works correctly") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py new file mode 100644 index 000000000000..b24c6d491e22 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_agent_file_search_code_interpreter_function.py @@ -0,0 +1,187 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +""" +Multi-Tool Tests: File Search + Code Interpreter + Function Tool + +Tests various scenarios using an agent with all three tools together. +All tests use the same 3-tool combination but different inputs and workflows. +""" + +import os +import json +import pytest +from io import BytesIO +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + PromptAgentDefinition, + FileSearchTool, + CodeInterpreterTool, + CodeInterpreterToolAuto, + FunctionTool, +) +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestAgentFileSearchCodeInterpreterFunction(TestBase): + """Tests for agents using File Search + Code Interpreter + Function Tool.""" + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_complete_analysis_workflow(self, **kwargs): + """ + Test complete workflow: find data, analyze it, save results. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create data file + txt_content = "Sample data for analysis" + vector_store = openai_client.vector_stores.create(name="ThreeToolStore") + + txt_file = BytesIO(txt_content.encode("utf-8")) + txt_file.name = "data.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=txt_file, + ) + print(f"File uploaded (id: {file.id})") + + # Define function tool + func_tool = FunctionTool( + name="save_result", + description="Save analysis result", + parameters={ + "type": "object", + "properties": { + "result": {"type": "string", "description": "The result"}, + }, + "required": ["result"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent with all three tools + agent = project_client.agents.create_version( + agent_name="three-tool-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Use file search to find data, code interpreter to analyze it, and save_result to save findings.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + func_tool, + ], + ), + description="Agent using File Search, Code Interpreter, and Function Tool.", + ) + print(f"Agent created (id: {agent.id})") + + # Use the agent + response = openai_client.responses.create( + input="Find the data file, analyze it, and save the results.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response received (id: {response.id})") + + assert response.id is not None + print("✓ Three-tool combination works!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_four_tools_combination(self, **kwargs): + """ + Test with 4 tools: File Search + Code Interpreter + 2 Functions. + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create vector store + txt_content = "Test data" + vector_store = openai_client.vector_stores.create(name="FourToolStore") + + txt_file = BytesIO(txt_content.encode("utf-8")) + txt_file.name = "data.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=txt_file, + ) + + # Define two function tools + func_tool_1 = FunctionTool( + name="save_result", + description="Save result", + parameters={ + "type": "object", + "properties": { + "result": {"type": "string", "description": "The result"}, + }, + "required": ["result"], + "additionalProperties": False, + }, + strict=True, + ) + + func_tool_2 = FunctionTool( + name="log_action", + description="Log an action", + parameters={ + "type": "object", + "properties": { + "action": {"type": "string", "description": "Action taken"}, + }, + "required": ["action"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent with 4 tools + agent = project_client.agents.create_version( + agent_name="four-tool-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Use all available tools.", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + CodeInterpreterTool(container=CodeInterpreterToolAuto()), + func_tool_1, + func_tool_2, + ], + ), + description="Agent with 4 tools.", + ) + print(f"Agent with 4 tools created (id: {agent.id})") + + assert agent.id is not None + print("✓ 4 tools works!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py new file mode 100644 index 000000000000..35bd8339821e --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/multitool/test_multitool_with_conversations.py @@ -0,0 +1,200 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +Test agents using multiple tools within conversations. + +This test file demonstrates how to use multiple agent tools (both server-side and client-side) +within the context of conversations, testing conversation state management with multi-tool interactions. +""" + +import json +import pytest +from io import BytesIO +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + FunctionTool, + FileSearchTool, + PromptAgentDefinition, +) +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestMultiToolWithConversations(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_file_search_and_function_with_conversation(self, **kwargs): + """ + Test using multiple tools (FileSearch + Function) within one conversation. + + This tests: + - Mixing FileSearch (server-side) and Function (client-side) tools in same conversation + - Complex multi-turn workflow with different tool types + - Conversation managing state across different tool executions + - Verifying conversation state preserves all tool interactions + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_client(operation_group="agents", **kwargs) + openai_client = project_client.get_openai_client() + + # Create document + doc_content = """Sales Data Q1 2024 + +Product A: $45,000 +Product B: $67,000 +Product C: $32,000 + +Total Revenue: $144,000 +""" + + vector_store = openai_client.vector_stores.create(name="SalesDataStore") + + file = BytesIO(doc_content.encode("utf-8")) + file.name = "sales.txt" + openai_client.vector_stores.files.upload_and_poll(vector_store_id=vector_store.id, file=file) + print(f"Vector store created: {vector_store.id}") + + # Define save function + save_report = FunctionTool( + name="save_report", + description="Save a report summary. Use this when explicitly asked to perform a save operation.", + parameters={ + "type": "object", + "properties": { + "title": {"type": "string"}, + "summary": {"type": "string"}, + }, + "required": ["title", "summary"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent with both tools + agent = project_client.agents.create_version( + agent_name="mixed-tools-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are an analyst. Search data to answer questions and save reports when instructed", + tools=[ + FileSearchTool(vector_store_ids=[vector_store.id]), + save_report, + ], + ), + description="Mixed tools agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Search (server-side tool) + print("\n--- Turn 1: File Search ---") + response_1 = openai_client.responses.create( + input="What was the total revenue in Q1 2024?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response 1: {response_1.output_text[:150]}...") + assert "144,000" in response_1.output_text or "144000" in response_1.output_text + + # Turn 2: Follow-up search + print("\n--- Turn 2: Follow-up search ---") + response_2 = openai_client.responses.create( + input="Which product had the highest sales?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response 2: {response_2.output_text[:150]}...") + assert "Product B" in response_2.output_text or "67,000" in response_2.output_text + + # Turn 3: Save report (client-side tool) + print("\n--- Turn 3: Save report using function ---") + response_3 = openai_client.responses.create( + input="Save a summary report of these Q1 results", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_3.output: + if item.type == "function_call": + print(f"Function called: {item.name}") + args = json.loads(item.arguments) + print(f" Title: {args['title']}") + print(f" Summary: {args['summary'][:100]}...") + + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"status": "saved", "report_id": "Q1_2024"}), + ) + ) + + response_3 = openai_client.responses.create( + input=input_list, + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response 3: {response_3.output_text[:150]}...") + + print("\n✓ Mixed tools with conversation successful!") + print(" - File search (server-side) worked") + print(" - Function call (client-side) worked") + print(" - Both tools used in same conversation") + + # Verify conversation state with multiple tool types + print("\n--- Verifying multi-tool conversation state ---") + all_items = list(openai_client.conversations.items.list(conversation.id)) + print(f"Total conversation items: {len(all_items)}") + + # Count different item types + user_messages = sum(1 for item in all_items if item.type == "message" and item.role == "user") + assistant_messages = sum(1 for item in all_items if item.type == "message" and item.role == "assistant") + function_calls = sum(1 for item in all_items if item.type == "function_call") + function_outputs = sum(1 for item in all_items if item.type == "function_call_output") + + print(f" User messages: {user_messages}") + print(f" Assistant messages: {assistant_messages}") + print(f" Function calls: {function_calls}") + print(f" Function outputs: {function_outputs}") + + # Print item sequence to show tool interleaving + print("\n Conversation item sequence:") + for i, item in enumerate(all_items, 1): + if item.type == "message": + content_preview = str(item.content[0] if item.content else "")[:50] + print(f" {i}. {item.type} ({item.role}): {content_preview}...") + else: + print(f" {i}. {item.type}") + + # Verify we have items from all three turns + assert user_messages >= 3, "Expected at least 3 user messages (three turns)" + assert assistant_messages >= 3, "Expected assistant responses for each turn" + assert function_calls >= 1, "Expected at least 1 function call (save_report)" + assert function_outputs >= 1, "Expected at least 1 function output" + + print("\n✓ Multi-tool conversation state verified") + print(" - Both server-side (FileSearch) and client-side (Function) tools tracked") + print(" - All 3 turns preserved in conversation") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py new file mode 100644 index 000000000000..7d506fc5bc42 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search.py @@ -0,0 +1,210 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + PromptAgentDefinition, + AzureAISearchAgentTool, + AzureAISearchToolResource, + AISearchIndexResource, + AzureAISearchQueryType, +) + + +class TestAgentAISearch(TestBase): + + # Test questions with expected answers + TEST_QUESTIONS = [ + { + "question": "Agent Lightning's unified data interface and MDP formulation are designed to separate task-specific agent design from learning-based policy optimization.", + "answer": True, + }, + { + "question": "LightningRL optimizes multi-call agent trajectories mainly by masking out non-LLM tokens in long trajectories, without decomposing them into transitions.", + "answer": False, + }, + { + "question": "The Training–Agent Disaggregation architecture uses an Agent Lightning Server (with an OpenAI-like API) and a Client so that agents can run their own tool/code logic without being co-located with the GPU training framework.", + "answer": True, + }, + { + "question": "In the text-to-SQL experiment, the authors used LangChain to build a 3-agent workflow on the Spider dataset, but only trained 2 of those agents (the SQL-writing and rewriting agents).", + "answer": True, + }, + { + "question": "The math QA task in the experiments was implemented with LangChain and used a SQL executor as its external tool.", + "answer": False, + }, + ] + + @servicePreparer() + @pytest.mark.skip(reason="Slow sequential sync test - covered by faster parallel async test") + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_ai_search_question_answering(self, **kwargs): + """ + Test agent with Azure AI Search capabilities for question answering. + + NOTE: This test is skipped in favor of the parallel async version which is + significantly faster (~3x) and provides the same coverage. + See test_agent_ai_search_async.py::test_agent_ai_search_question_answering_async_parallel + + This test verifies that an agent can be created with AzureAISearchAgentTool, + use it to search indexed content, and provide accurate answers to questions + based on the search results. + + The test asks 5 true/false questions and validates that at least 4 are + answered correctly by the agent using the search index. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with AI Search) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Get AI Search connection and index from environment + ai_search_connection_id = kwargs.get("azure_ai_projects_tests_ai_search_project_connection_id") + ai_search_index_name = kwargs.get("azure_ai_projects_tests_ai_search_index_name") + + if not ai_search_connection_id: + pytest.skip("AZURE_AI_PROJECTS_TESTS_AI_SEARCH_PROJECT_CONNECTION_ID environment variable not set") + + if not ai_search_index_name: + pytest.skip("AZURE_AI_PROJECTS_TESTS_AI_SEARCH_INDEX_NAME environment variable not set") + + assert isinstance(ai_search_connection_id, str), "ai_search_connection_id must be a string" + assert isinstance(ai_search_index_name, str), "ai_search_index_name must be a string" + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with Azure AI Search tool + agent = project_client.agents.create_version( + agent_name="ai-search-qa-agent", + definition=PromptAgentDefinition( + model=model, + instructions="""You are a helpful assistant that answers true/false questions based on the provided search results. + Always use the Azure AI Search tool to find relevant information before answering. + Respond with only 'True' or 'False' based on what you find in the search results. + If you cannot find clear evidence in the search results, answer 'False'.""", + tools=[ + AzureAISearchAgentTool( + azure_ai_search=AzureAISearchToolResource( + indexes=[ + AISearchIndexResource( + project_connection_id=ai_search_connection_id, + index_name=ai_search_index_name, + query_type=AzureAISearchQueryType.SIMPLE, + ), + ] + ) + ) + ], + ), + description="Agent for testing AI Search question answering.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "ai-search-qa-agent" + assert agent.version is not None + + # Test each question + correct_answers = 0 + total_questions = len(self.TEST_QUESTIONS) + + for i, qa_pair in enumerate(self.TEST_QUESTIONS, 1): + question = qa_pair["question"] + expected_answer = qa_pair["answer"] + + print(f"\n{'='*80}") + print(f"Question {i}/{total_questions}:") + print(f"Q: {question}") + print(f"Expected: {expected_answer}") + + output_text = "" + + stream_response = openai_client.responses.create( + stream=True, + tool_choice="required", + input=f"Answer this question with only 'True' or 'False': {question}", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + for event in stream_response: + if event.type == "response.created": + print(f"Response created with ID: {event.response.id}") + elif event.type == "response.output_text.delta": + pass # Don't print deltas to reduce output + elif event.type == "response.completed": + output_text = event.response.output_text + print(f"Agent's answer: {output_text}") + + # Parse the answer from the output + # Look for "True" or "False" in the response + output_lower = output_text.lower() + agent_answer = None + + # Try to extract boolean answer + if "true" in output_lower and "false" not in output_lower: + agent_answer = True + elif "false" in output_lower and "true" not in output_lower: + agent_answer = False + elif output_lower.strip() in ["true", "false"]: + agent_answer = output_lower.strip() == "true" + else: + # Try to determine based on more complex responses + # Count occurrences + true_count = output_lower.count("true") + false_count = output_lower.count("false") + if true_count > false_count: + agent_answer = True + elif false_count > true_count: + agent_answer = False + + if agent_answer is not None: + is_correct = agent_answer == expected_answer + if is_correct: + correct_answers += 1 + print(f"✓ CORRECT (Agent: {agent_answer}, Expected: {expected_answer})") + else: + print(f"✗ INCORRECT (Agent: {agent_answer}, Expected: {expected_answer})") + else: + print(f"✗ UNABLE TO PARSE ANSWER from: {output_text}") + + # Print summary + print(f"\n{'='*80}") + print(f"SUMMARY: {correct_answers}/{total_questions} questions answered correctly") + print(f"{'='*80}") + + # Verify that at least 4 out of 5 questions were answered correctly + assert correct_answers >= 4, ( + f"Expected at least 4 correct answers out of {total_questions}, " + f"but got {correct_answers}. The agent needs to answer at least 80% correctly." + ) + + print( + f"\n✓ Test passed! Agent answered {correct_answers}/{total_questions} questions correctly (>= 4 required)" + ) + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py new file mode 100644 index 000000000000..1bd914c7d513 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_ai_search_async.py @@ -0,0 +1,238 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import asyncio +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + PromptAgentDefinition, + AzureAISearchAgentTool, + AzureAISearchToolResource, + AISearchIndexResource, + AzureAISearchQueryType, +) + + +class TestAgentAISearchAsync(TestBase): + + # Test questions with expected answers + TEST_QUESTIONS = [ + { + "title": "Unified Data Interface", + "question": "Agent Lightning's unified data interface and MDP formulation are designed to separate task-specific agent design from learning-based policy optimization.", + "answer": True, + }, + { + "title": "LightningRL Optimization", + "question": "LightningRL optimizes multi-call agent trajectories mainly by masking out non-LLM tokens in long trajectories, without decomposing them into transitions.", + "answer": False, + }, + { + "title": "Training-Agent Disaggregation", + "question": "The Training–Agent Disaggregation architecture uses an Agent Lightning Server (with an OpenAI-like API) and a Client so that agents can run their own tool/code logic without being co-located with the GPU training framework.", + "answer": True, + }, + { + "title": "Text-to-SQL Experiment", + "question": "In the text-to-SQL experiment, the authors used LangChain to build a 3-agent workflow on the Spider dataset, but only trained 2 of those agents (the SQL-writing and rewriting agents).", + "answer": True, + }, + { + "title": "Math QA Implementation", + "question": "The math QA task in the experiments was implemented with LangChain and used a SQL executor as its external tool.", + "answer": False, + }, + ] + + async def _ask_question_async( + self, + openai_client, + agent_name: str, + title: str, + question: str, + expected_answer: bool, + question_num: int, + total_questions: int, + ): + """Helper method to ask a single question asynchronously.""" + print(f"\n{'='*80}") + print(f"Q{question_num}/{total_questions}: {title}") + print(f"{question}") + print(f"Expected: {expected_answer}") + + output_text = "" + + stream_response = await openai_client.responses.create( + stream=True, + tool_choice="required", + input=f"Answer this question with only 'True' or 'False': {question}", + extra_body={"agent": {"name": agent_name, "type": "agent_reference"}}, + ) + + async for event in stream_response: + if event.type == "response.created": + print(f"Response ID: {event.response.id}") + elif event.type == "response.completed": + output_text = event.response.output_text + + # Parse the answer from the output + output_lower = output_text.lower() + agent_answer = None + + # Try to extract boolean answer + if "true" in output_lower and "false" not in output_lower: + agent_answer = True + elif "false" in output_lower and "true" not in output_lower: + agent_answer = False + elif output_lower.strip() in ["true", "false"]: + agent_answer = output_lower.strip() == "true" + else: + # Try to determine based on more complex responses + true_count = output_lower.count("true") + false_count = output_lower.count("false") + if true_count > false_count: + agent_answer = True + elif false_count > true_count: + agent_answer = False + + is_correct = False + if agent_answer is not None: + is_correct = agent_answer == expected_answer + if is_correct: + print(f"✓ Q{question_num} ({title}): CORRECT") + else: + print(f"✗ Q{question_num} ({title}): INCORRECT (Agent: {agent_answer}, Expected: {expected_answer})") + else: + print(f"✗ Q{question_num} ({title}): UNABLE TO PARSE ANSWER from: {output_text}") + + return is_correct + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + @pytest.mark.asyncio + async def test_agent_ai_search_question_answering_async_parallel(self, **kwargs): + """ + Test agent with Azure AI Search capabilities for question answering using async (parallel). + + This test verifies that an agent can be created with AzureAISearchAgentTool, + and handle multiple concurrent requests to search indexed content and provide + accurate answers to questions based on the search results. + + The test asks 5 true/false questions IN PARALLEL using asyncio.gather() and + validates that at least 4 are answered correctly by the agent using the search index. + + This should be significantly faster than the sequential version. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses (concurrent) openai_client.responses.create() (with AI Search, parallel) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Setup + project_client = self.create_async_client(operation_group="agents", **kwargs) + + async with project_client: + openai_client = project_client.get_openai_client() + + # Get AI Search connection and index from environment + ai_search_connection_id = kwargs.get("azure_ai_projects_tests_ai_search_project_connection_id") + ai_search_index_name = kwargs.get("azure_ai_projects_tests_ai_search_index_name") + + if not ai_search_connection_id: + pytest.skip("AZURE_AI_PROJECTS_TESTS_AI_SEARCH_PROJECT_CONNECTION_ID environment variable not set") + + if not ai_search_index_name: + pytest.skip("AZURE_AI_PROJECTS_TESTS_AI_SEARCH_INDEX_NAME environment variable not set") + + assert isinstance(ai_search_connection_id, str), "ai_search_connection_id must be a string" + assert isinstance(ai_search_index_name, str), "ai_search_index_name must be a string" + + # Create agent with Azure AI Search tool + agent = await project_client.agents.create_version( + agent_name="ai-search-qa-agent-async-parallel", + definition=PromptAgentDefinition( + model=model, + instructions="""You are a helpful assistant that answers true/false questions based on the provided search results. + Always use the Azure AI Search tool to find relevant information before answering. + Respond with only 'True' or 'False' based on what you find in the search results. + If you cannot find clear evidence in the search results, answer 'False'.""", + tools=[ + AzureAISearchAgentTool( + azure_ai_search=AzureAISearchToolResource( + indexes=[ + AISearchIndexResource( + project_connection_id=ai_search_connection_id, + index_name=ai_search_index_name, + query_type=AzureAISearchQueryType.SIMPLE, + ), + ] + ) + ) + ], + ), + description="Agent for testing AI Search question answering (async parallel).", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "ai-search-qa-agent-async-parallel" + assert agent.version is not None + + # Test all questions IN PARALLEL using asyncio.gather() + total_questions = len(self.TEST_QUESTIONS) + print(f"\nRunning {total_questions} questions in parallel...") + + # Create tasks for all questions + tasks = [] + for i, qa_pair in enumerate(self.TEST_QUESTIONS, 1): + title = qa_pair["title"] + question = qa_pair["question"] + expected_answer = qa_pair["answer"] + + task = self._ask_question_async( + openai_client, agent.name, title, question, expected_answer, i, total_questions + ) + tasks.append(task) + + # Run all tasks in parallel and collect results + results = await asyncio.gather(*tasks) + + # Count correct answers + correct_answers = sum(1 for is_correct in results if is_correct) + + # Print summary + print(f"\n{'='*80}") + print(f"SUMMARY: {correct_answers}/{total_questions} questions answered correctly") + print(f"{'='*80}") + + # Verify that at least 4 out of 5 questions were answered correctly + assert correct_answers >= 4, ( + f"Expected at least 4 correct answers out of {total_questions}, " + f"but got {correct_answers}. The agent needs to answer at least 80% correctly." + ) + + print( + f"\n✓ Test passed! Agent answered {correct_answers}/{total_questions} questions correctly (>= 4 required)" + ) + + # Teardown + await project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py new file mode 100644 index 000000000000..8373963e63e5 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_bing_grounding.py @@ -0,0 +1,220 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + PromptAgentDefinition, + BingGroundingAgentTool, + BingGroundingSearchToolParameters, + BingGroundingSearchConfiguration, +) + + +class TestAgentBingGrounding(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_bing_grounding(self, **kwargs): + """ + Test agent with Bing grounding capabilities. + + This test verifies that an agent can be created with BingGroundingAgentTool, + use it to search the web for current information, and provide responses with + URL citations. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with Bing grounding) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + # Note: This test requires AZURE_AI_PROJECTS_TESTS_BING_PROJECT_CONNECTION_ID environment variable + # to be set with a valid Bing connection ID from the project + bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_project_connection_id") + + if not bing_connection_id: + pytest.skip("AZURE_AI_PROJECTS_TESTS_BING_PROJECT_CONNECTION_ID environment variable not set") + + assert isinstance(bing_connection_id, str), "bing_connection_id must be a string" + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with Bing grounding tool + agent = project_client.agents.create_version( + agent_name="bing-grounding-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant.", + tools=[ + BingGroundingAgentTool( + bing_grounding=BingGroundingSearchToolParameters( + search_configurations=[ + BingGroundingSearchConfiguration(project_connection_id=bing_connection_id) + ] + ) + ) + ], + ), + description="You are a helpful agent.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "bing-grounding-agent" + assert agent.version is not None + + # Test agent with a query that requires current web information + output_text = "" + url_citations = [] + + stream_response = openai_client.responses.create( + stream=True, + tool_choice="required", + input="What is the current weather in Seattle?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + for event in stream_response: + if event.type == "response.created": + print(f"Follow-up response created with ID: {event.response.id}") + assert event.response.id is not None + elif event.type == "response.output_text.delta": + print(f"Delta: {event.delta}") + elif event.type == "response.text.done": + print(f"Follow-up response done!") + elif event.type == "response.output_item.done": + if event.item.type == "message": + item = event.item + if item.content and len(item.content) > 0: + if item.content[-1].type == "output_text": + text_content = item.content[-1] + for annotation in text_content.annotations: + if annotation.type == "url_citation": + print(f"URL Citation: {annotation.url}") + url_citations.append(annotation.url) + elif event.type == "response.completed": + print(f"Follow-up completed!") + print(f"Full response: {event.response.output_text}") + output_text = event.response.output_text + + # Verify that we got a response + assert len(output_text) > 0, "Expected non-empty response text" + + # Verify that we got URL citations (Bing grounding should provide sources) + assert len(url_citations) > 0, "Expected URL citations from Bing grounding" + + # Verify that citations are valid URLs + for url in url_citations: + assert url.startswith("http://") or url.startswith("https://"), f"Invalid URL citation: {url}" + + print(f"Test completed successfully with {len(url_citations)} URL citations") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_bing_grounding_multiple_queries(self, **kwargs): + """ + Test agent with Bing grounding for multiple queries. + + This test verifies that an agent can handle multiple queries using + Bing grounding and provide accurate responses with citations. + """ + + model = self.test_agents_params["model_deployment_name"] + + bing_connection_id = kwargs.get("azure_ai_projects_tests_bing_project_connection_id") + + if not bing_connection_id: + pytest.skip("AZURE_AI_PROJECTS_TESTS_BING_PROJECT_CONNECTION_ID environment variable not set") + + assert isinstance(bing_connection_id, str), "bing_connection_id must be a string" + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with Bing grounding tool + agent = project_client.agents.create_version( + agent_name="bing-grounding-multi-query-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that provides current information.", + tools=[ + BingGroundingAgentTool( + bing_grounding=BingGroundingSearchToolParameters( + search_configurations=[ + BingGroundingSearchConfiguration(project_connection_id=bing_connection_id) + ] + ) + ) + ], + ), + description="Agent for testing multiple Bing grounding queries.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + + # Test with multiple different queries + queries = [ + "What is today's date?", + "What are the latest news about AI?", + ] + + for query in queries: + print(f"\nTesting query: {query}") + output_text = "" + url_citations = [] + + stream_response = openai_client.responses.create( + stream=True, + tool_choice="required", + input=query, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + for event in stream_response: + if event.type == "response.output_item.done": + if event.item.type == "message": + item = event.item + if item.content and len(item.content) > 0: + if item.content[-1].type == "output_text": + text_content = item.content[-1] + for annotation in text_content.annotations: + if annotation.type == "url_citation": + url_citations.append(annotation.url) + elif event.type == "response.completed": + output_text = event.response.output_text + + # Verify that we got a response for each query + assert len(output_text) > 0, f"Expected non-empty response text for query: {query}" + print(f"Response length: {len(output_text)} characters") + print(f"URL citations found: {len(url_citations)}") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py new file mode 100644 index 000000000000..b360d2bd2d32 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_code_interpreter.py @@ -0,0 +1,240 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import os +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + PromptAgentDefinition, + CodeInterpreterTool, + CodeInterpreterToolAuto, +) + + +class TestAgentCodeInterpreter(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_code_interpreter_simple_math(self, **kwargs): + """ + Test agent with Code Interpreter for simple Python code execution. + + This test verifies that an agent can execute simple Python code + without any file uploads or downloads - just pure code execution. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with Code Interpreter) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with code interpreter tool (no files) + agent = project_client.agents.create_version( + agent_name="code-interpreter-simple-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can execute Python code.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[]))], + ), + description="Simple code interpreter agent for basic Python execution.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "code-interpreter-simple-agent" + assert agent.version is not None + + # Ask the agent to execute a complex Python calculation + # Problem: Calculate the sum of cubes from 1 to 50, then add 12!/(8!) + # Expected answer: 1637505 + print("\nAsking agent to calculate: sum of cubes from 1 to 50, plus 12!/(8!)") + + response = openai_client.responses.create( + input="Calculate this using Python: First, find the sum of cubes from 1 to 50 (1³ + 2³ + ... + 50³). Then add 12 factorial divided by 8 factorial (12!/8!). What is the final result?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Get the response text + last_message = response.output[-1] + response_text = "" + + if last_message.type == "message": + for content_item in last_message.content: + if content_item.type == "output_text": + response_text += content_item.text + + print(f"Agent's response: {response_text}") + + # Verify the response contains the correct answer (1637505) + # Note: sum of cubes 1-50 = 1,625,625; 12!/8! = 11,880; total = 1,637,505 + assert ( + "1637505" in response_text or "1,637,505" in response_text + ), f"Expected answer 1637505 to be in response, but got: {response_text}" + + print("✓ Code interpreter successfully executed Python code and returned correct answer") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + @servicePreparer() + @pytest.mark.skip( + reason="Skipped due to known server bug. Enable once https://msdata.visualstudio.com/Vienna/_workitems/edit/4841313 is resolved" + ) + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_code_interpreter_file_generation(self, **kwargs): + """ + Test agent with Code Interpreter for file upload, processing, and download. + + This test verifies that an agent can: + 1. Work with uploaded CSV files + 2. Execute Python code to generate a chart + 3. Return a downloadable file + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /files openai_client.files.create() + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with Code Interpreter + file) + GET /containers/{container_id}/files/{file_id} openai_client.containers.files.content.retrieve() + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + DELETE /files/{file_id} openai_client.files.delete() + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Get the path to the test CSV file + asset_file_path = os.path.abspath( + os.path.join( + os.path.dirname(__file__), "../../../samples/agents/assets/synthetic_500_quarterly_results.csv" + ) + ) + + assert os.path.exists(asset_file_path), f"Test CSV file not found at: {asset_file_path}" + print(f"Using test CSV file: {asset_file_path}") + + # Upload the CSV file + with open(asset_file_path, "rb") as f: + file = openai_client.files.create(purpose="assistants", file=f) + + print(f"File uploaded (id: {file.id})") + assert file.id is not None + + # Create agent with code interpreter tool and the uploaded file + agent = project_client.agents.create_version( + agent_name="code-interpreter-file-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can analyze data and create visualizations.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[file.id]))], + ), + description="Code interpreter agent for file processing and chart generation.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "code-interpreter-file-agent" + assert agent.version is not None + + # Ask the agent to create a chart from the CSV + print("\nAsking agent to create a bar chart...") + + response = openai_client.responses.create( + input="Create a bar chart showing operating profit by sector from the uploaded CSV file. Save it as a PNG file.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Extract file information from response annotations + file_id = "" + filename = "" + container_id = "" + + last_message = response.output[-1] + if last_message.type == "message": + for content_item in last_message.content: + if content_item.type == "output_text": + if content_item.annotations: + for annotation in content_item.annotations: + if annotation.type == "container_file_citation": + file_id = annotation.file_id + filename = annotation.filename + container_id = annotation.container_id + print( + f"Found generated file: {filename} (ID: {file_id}, Container: {container_id})" + ) + break + + # Verify that a file was generated + assert file_id, "Expected a file to be generated but no file ID found in response" + assert filename, "Expected a filename but none found in response" + assert container_id, "Expected a container ID but none found in response" + + print(f"✓ File generated successfully: {filename}") + + # Download the generated file + print(f"Downloading file {filename}...") + file_content = openai_client.containers.files.content.retrieve(file_id=file_id, container_id=container_id) + + # Read the content + content_bytes = file_content.read() + assert len(content_bytes) > 0, "Expected file content but got empty bytes" + + print(f"✓ File downloaded successfully ({len(content_bytes)} bytes)") + + # Verify it's a PNG file (check magic bytes) + if filename.endswith(".png"): + # PNG files start with: 89 50 4E 47 (‰PNG) + assert content_bytes[:4] == b"\x89PNG", "File does not appear to be a valid PNG" + print("✓ File is a valid PNG image") + + # Teardown + print("\nCleaning up...") + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + openai_client.files.delete(file.id) + print("Uploaded file deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py new file mode 100644 index 000000000000..59cde4dd72d2 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search.py @@ -0,0 +1,328 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed----------------------------------------------------------------------------------------- +# cSpell:disable + +import os +import pytest +from io import BytesIO +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool + + +class TestAgentFileSearch(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_file_search(self, **kwargs): + """ + Test agent with File Search tool for document Q&A. + + This test verifies that an agent can: + 1. Upload and index documents into a vector store + 2. Use FileSearchTool to search through uploaded documents + 3. Answer questions based on document content + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /vector_stores openai_client.vector_stores.create() + POST /vector_stores/{id}/files openai_client.vector_stores.files.upload_and_poll() + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with FileSearchTool) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + DELETE /vector_stores/{id} openai_client.vector_stores.delete() + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Get the path to the test file + asset_file_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/product_info.md") + ) + + assert os.path.exists(asset_file_path), f"Test file not found at: {asset_file_path}" + print(f"Using test file: {asset_file_path}") + + # Create vector store for file search + vector_store = openai_client.vector_stores.create(name="ProductInfoStore") + print(f"Vector store created (id: {vector_store.id})") + assert vector_store.id is not None + + # Upload file to vector store + with open(asset_file_path, "rb") as f: + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=f, + ) + + print(f"File uploaded (id: {file.id}, status: {file.status})") + assert file.id is not None + assert file.status == "completed", f"Expected file status 'completed', got '{file.status}'" + + # Create agent with file search tool + agent = project_client.agents.create_version( + agent_name="file-search-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can search through uploaded documents to answer questions.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Agent for testing file search capabilities.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "file-search-agent" + assert agent.version is not None + + # Ask a question about the uploaded document + print("\nAsking agent about the product information...") + + response = openai_client.responses.create( + input="What products are mentioned in the document?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Get the response text + response_text = response.output_text + print(f"\nAgent's response: {response_text[:300]}...") + + # Verify we got a meaningful response + assert len(response_text) > 50, "Expected a substantial response from the agent" + + # The response should mention finding information (indicating file search was used) + # We can't assert exact product names without knowing the file content, + # but we can verify the agent provided an answer + print("\n✓ Agent successfully used file search tool to answer question from uploaded document") + + # Teardown + print("\nCleaning up...") + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + openai_client.vector_stores.delete(vector_store.id) + print("Vector store deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_file_search_unsupported_file_type(self, **kwargs): + """ + Negative test: Verify that unsupported file types are rejected with clear error messages. + + This test validates that: + 1. CSV files (unsupported format) are rejected + 2. The error message clearly indicates the file type is not supported + 3. The error message lists supported file types + + This ensures good developer experience by providing actionable error messages. + """ + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create vector store + vector_store = openai_client.vector_stores.create(name="UnsupportedFileTestStore") + print(f"Vector store created (id: {vector_store.id})") + + # Create CSV file (unsupported format) + csv_content = """product,quarter,revenue +Widget A,Q1,15000 +Widget B,Q1,22000 +Widget A,Q2,18000 +Widget B,Q2,25000""" + + csv_file = BytesIO(csv_content.encode("utf-8")) + csv_file.name = "sales_data.csv" + + # Attempt to upload unsupported file type + print("\nAttempting to upload CSV file (unsupported format)...") + try: + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=csv_file, + ) + # If we get here, the test should fail + openai_client.vector_stores.delete(vector_store.id) + pytest.fail("Expected BadRequestError for CSV file upload, but upload succeeded") + + except Exception as e: + error_message = str(e) + print(f"\n✓ Upload correctly rejected with error: {error_message[:200]}...") + + # Verify error message quality + assert ( + "400" in error_message or "BadRequestError" in type(e).__name__ + ), "Should be a 400 Bad Request error" + + assert ".csv" in error_message.lower(), "Error message should mention the CSV file extension" + + assert ( + "not supported" in error_message.lower() or "unsupported" in error_message.lower() + ), "Error message should clearly state the file type is not supported" + + # Check that supported file types are mentioned (helpful for developers) + error_lower = error_message.lower() + has_supported_list = any(ext in error_lower for ext in [".txt", ".pdf", ".md", ".py"]) + assert has_supported_list, "Error message should list examples of supported file types" + + print("✓ Error message is clear and actionable") + print(" - Mentions unsupported file type (.csv)") + print(" - States it's not supported") + print(" - Lists supported file types") + + # Cleanup + openai_client.vector_stores.delete(vector_store.id) + print("\nVector store deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_file_search_multi_turn_conversation(self, **kwargs): + """ + Test multi-turn conversation with File Search. + + This test verifies that an agent can maintain context across multiple turns + while using File Search to answer follow-up questions. + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create a document with information about products + product_info = """Product Catalog: + +Widget A: +- Price: $150 +- Category: Electronics +- Stock: 50 units +- Rating: 4.5/5 stars + +Widget B: +- Price: $220 +- Category: Electronics +- Stock: 30 units +- Rating: 4.8/5 stars + +Widget C: +- Price: $95 +- Category: Home & Garden +- Stock: 100 units +- Rating: 4.2/5 stars +""" + + # Create vector store and upload document + vector_store = openai_client.vector_stores.create(name="ProductCatalog") + print(f"Vector store created: {vector_store.id}") + + product_file = BytesIO(product_info.encode("utf-8")) + product_file.name = "products.txt" + + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=product_file, + ) + print(f"Product catalog uploaded: {file.id}") + + # Create agent with File Search + agent = project_client.agents.create_version( + agent_name="product-catalog-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a product information assistant. Use file search to answer questions about products.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Agent for multi-turn product queries.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Ask about price + print("\n--- Turn 1: Initial query ---") + response_1 = openai_client.responses.create( + input="What is the price of Widget B?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "$220" in response_1_text or "220" in response_1_text, "Response should mention Widget B's price" + + # Turn 2: Follow-up question (requires context from turn 1) + print("\n--- Turn 2: Follow-up query (testing context retention) ---") + response_2 = openai_client.responses.create( + input="What about its stock level?", + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + assert ( + "30" in response_2_text or "thirty" in response_2_text.lower() + ), "Response should mention Widget B's stock (30 units)" + + # Turn 3: Another follow-up (compare with different product) + print("\n--- Turn 3: Comparison query ---") + response_3 = openai_client.responses.create( + input="How does that compare to Widget A's stock?", + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + assert ( + "50" in response_3_text or "fifty" in response_3_text.lower() + ), "Response should mention Widget A's stock (50 units)" + + # Turn 4: New topic (testing topic switching) + print("\n--- Turn 4: Topic switch ---") + response_4 = openai_client.responses.create( + input="Which widget has the highest rating?", + previous_response_id=response_3.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_4_text = response_4.output_text + print(f"Response 4: {response_4_text[:200]}...") + assert ( + "widget b" in response_4_text.lower() or "4.8" in response_4_text + ), "Response should identify Widget B as highest rated (4.8/5)" + + print("\n✓ Multi-turn conversation successful!") + print(" - Context maintained across turns") + print(" - Follow-up questions handled correctly") + print(" - Topic switching works") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py new file mode 100644 index 000000000000..a207ab32eb9f --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_file_search_stream.py @@ -0,0 +1,137 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import os +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FileSearchTool + + +class TestAgentFileSearchStream(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_file_search_stream(self, **kwargs): + """ + Test agent with File Search tool using streaming responses. + + This test verifies that an agent can: + 1. Upload and index documents into a vector store + 2. Use FileSearchTool with streaming enabled + 3. Stream back responses based on document content + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /vector_stores openai_client.vector_stores.create() + POST /vector_stores/{id}/files openai_client.vector_stores.files.upload_and_poll() + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (stream=True, with FileSearchTool) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + DELETE /vector_stores/{id} openai_client.vector_stores.delete() + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Get the path to the test file + asset_file_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../../samples/agents/assets/product_info.md") + ) + + assert os.path.exists(asset_file_path), f"Test file not found at: {asset_file_path}" + print(f"Using test file: {asset_file_path}") + + # Create vector store for file search + vector_store = openai_client.vector_stores.create(name="ProductInfoStoreStream") + print(f"Vector store created (id: {vector_store.id})") + assert vector_store.id is not None + + # Upload file to vector store + with open(asset_file_path, "rb") as f: + file = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=f, + ) + + print(f"File uploaded (id: {file.id}, status: {file.status})") + assert file.id is not None + assert file.status == "completed", f"Expected file status 'completed', got '{file.status}'" + + # Create agent with file search tool + agent = project_client.agents.create_version( + agent_name="file-search-stream-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can search through uploaded documents to answer questions.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Agent for testing file search with streaming.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "file-search-stream-agent" + assert agent.version is not None + + # Ask a question with streaming enabled + print("\nAsking agent about the product information (streaming)...") + + stream_response = openai_client.responses.create( + stream=True, + input="What products are mentioned in the document? Please provide a brief summary.", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Collect streamed response + response_text = "" + response_id = None + events_received = 0 + + for event in stream_response: + events_received += 1 + + if event.type == "response.output_item.done": + if event.item.type == "message": + for content_item in event.item.content: + if content_item.type == "output_text": + response_text += content_item.text + + elif event.type == "response.completed": + response_id = event.response.id + # Could also use event.response.output_text + + print(f"\nStreaming completed (id: {response_id}, events: {events_received})") + assert response_id is not None, "Expected response ID from stream" + assert events_received > 0, "Expected to receive stream events" + + print(f"Agent's streamed response: {response_text[:300]}...") + + # Verify we got a meaningful response + assert len(response_text) > 50, "Expected a substantial response from the agent" + + print("\n✓ Agent successfully streamed responses using file search tool") + + # Teardown + print("\nCleaning up...") + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + openai_client.vector_stores.delete(vector_store.id) + print("Vector store deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py new file mode 100644 index 000000000000..eaf6d7ec3617 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_function_tool.py @@ -0,0 +1,511 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import json +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, FunctionTool +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestAgentFunctionTool(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_function_tool(self, **kwargs): + """ + Test agent with custom function tool. + + This test verifies that an agent can: + 1. Use a custom function tool defined by the developer + 2. Request function calls when needed + 3. Receive function results and incorporate them into responses + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (triggers function) + POST /openai/responses openai_client.responses.create() (with function result) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Define a function tool for the model to use + func_tool = FunctionTool( + name="get_weather", + description="Get the current weather for a location.", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "A city name like Seattle or London", + }, + }, + "required": ["location"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent with function tool + agent = project_client.agents.create_version( + agent_name="function-tool-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can check the weather. Use the get_weather function when users ask about weather.", + tools=[func_tool], + ), + description="Agent for testing function tool capabilities.", + ) + print( + f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version}, model: {agent.definition['model']})" + ) + assert agent.id is not None + assert agent.name == "function-tool-agent" + assert agent.version is not None + + # Ask a question that should trigger the function call + print("\nAsking agent: What's the weather in Seattle?") + + response = openai_client.responses.create( + input="What's the weather in Seattle?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + + # Check for function calls in the response + function_calls_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "function_call": + function_calls_found += 1 + print(f"Found function call (id: {item.call_id}, name: {item.name})") + + # Parse the arguments + arguments = json.loads(item.arguments) + print(f"Function arguments: {arguments}") + + # Verify the function call is for get_weather + assert item.name == "get_weather", f"Expected function name 'get_weather', got '{item.name}'" + assert "location" in arguments, "Expected 'location' in function arguments" + assert ( + "seattle" in arguments["location"].lower() + ), f"Expected Seattle in location, got {arguments['location']}" + + # Simulate the function execution and provide a result + weather_result = { + "location": arguments["location"], + "temperature": "72°F", + "condition": "Sunny", + "humidity": "45%", + } + + # Add the function result to the input list + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(weather_result), + ) + ) + print(f"✓ Prepared function result: {weather_result}") + + # Verify that at least one function call was made + assert function_calls_found > 0, "Expected at least 1 function call, but found none" + print(f"\n✓ Processed {function_calls_found} function call(s)") + + # Send the function results back to get the final response + print("\nSending function results back to agent...") + + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Final response completed (id: {response.id})") + assert response.id is not None + + # Get the final response text + response_text = response.output_text + print(f"\nAgent's final response: {response_text}") + + # Verify the response incorporates the weather data + assert len(response_text) > 20, "Expected a meaningful response from the agent" + + # Check that the response mentions the weather information we provided + response_lower = response_text.lower() + assert any( + keyword in response_lower for keyword in ["72", "sunny", "weather", "seattle"] + ), f"Expected response to mention weather information, but got: {response_text}" + + print("\n✓ Agent successfully used function tool and incorporated results into response") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_function_tool_multi_turn_with_multiple_calls(self, **kwargs): + """ + Test multi-turn conversation where agent calls functions multiple times. + + This tests: + - Multiple function calls across different turns + - Context retention between turns + - Ability to use previous function results in subsequent queries + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Define multiple function tools + get_weather = FunctionTool( + name="get_weather", + description="Get current weather for a city", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city name", + }, + }, + "required": ["city"], + "additionalProperties": False, + }, + strict=True, + ) + + get_temperature_forecast = FunctionTool( + name="get_temperature_forecast", + description="Get 3-day temperature forecast for a city", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city name", + }, + }, + "required": ["city"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent with multiple functions + agent = project_client.agents.create_version( + agent_name="weather-assistant-multi-turn", + definition=PromptAgentDefinition( + model=model, + instructions="You are a weather assistant. Use available functions to answer weather questions.", + tools=[get_weather, get_temperature_forecast], + ), + description="Weather assistant for multi-turn testing.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Get current weather + print("\n--- Turn 1: Current weather query ---") + response_1 = openai_client.responses.create( + input="What's the weather in New York?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_1.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + assert item.name == "get_weather" + + # Simulate weather API response + weather_data = {"temperature": 68, "condition": "Cloudy", "humidity": 65} + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(weather_data), + ) + ) + + # Get response with function results + response_1 = openai_client.responses.create( + input=input_list, + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "68" in response_1_text or "cloudy" in response_1_text.lower() + + # Turn 2: Follow-up with forecast (requires context) + print("\n--- Turn 2: Follow-up forecast query ---") + response_2 = openai_client.responses.create( + input="What about the forecast for the next few days?", + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle forecast function call + input_list = [] + for item in response_2.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + assert item.name == "get_temperature_forecast" + + # Agent should remember we're talking about New York + args = json.loads(item.arguments) + assert "new york" in args["city"].lower() + + # Simulate forecast API response + forecast_data = { + "city": "New York", + "forecast": [ + {"day": "Tomorrow", "temp": 70}, + {"day": "Day 2", "temp": 72}, + {"day": "Day 3", "temp": 69}, + ], + } + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(forecast_data), + ) + ) + + # Get response with forecast + response_2 = openai_client.responses.create( + input=input_list, + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + assert "70" in response_2_text or "72" in response_2_text + + # Turn 3: Compare with another city + print("\n--- Turn 3: New city query ---") + response_3 = openai_client.responses.create( + input="How does that compare to Seattle's weather?", + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function calls for Seattle (agent might call both weather and forecast) + input_list = [] + for item in response_3.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + args = json.loads(item.arguments) + assert "seattle" in args["city"].lower() + + # Handle based on function name + if item.name == "get_weather": + weather_data = {"temperature": 58, "condition": "Rainy", "humidity": 80} + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(weather_data), + ) + ) + elif item.name == "get_temperature_forecast": + forecast_data = { + "city": "Seattle", + "forecast": [ + {"day": "Tomorrow", "temp": 56}, + {"day": "Day 2", "temp": 59}, + {"day": "Day 3", "temp": 57}, + ], + } + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps(forecast_data), + ) + ) + + # Get final comparison response + response_3 = openai_client.responses.create( + input=input_list, + previous_response_id=response_3.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + # Agent should mention Seattle weather (either 58 for current or comparison) + assert "seattle" in response_3_text.lower() or any(temp in response_3_text for temp in ["58", "56", "59"]) + + print("\n✓ Multi-turn conversation with multiple function calls successful!") + print(" - Multiple functions called across turns") + print(" - Context maintained (agent remembered New York)") + print(" - Comparison between cities works") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_function_tool_context_dependent_followup(self, **kwargs): + """ + Test deeply context-dependent follow-ups (e.g., unit conversion, clarification). + + This tests that the agent truly uses previous response content, not just + remembering parameters from the first query. + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Define function tool + get_temperature = FunctionTool( + name="get_temperature", + description="Get current temperature for a city in Fahrenheit", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city name", + }, + }, + "required": ["city"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="temperature-assistant-context", + definition=PromptAgentDefinition( + model=model, + instructions="You are a temperature assistant. Answer temperature questions.", + tools=[get_temperature], + ), + description="Temperature assistant for context testing.", + ) + print(f"Agent created: {agent.id}") + + # Turn 1: Get temperature in Fahrenheit + print("\n--- Turn 1: Get temperature ---") + response_1 = openai_client.responses.create( + input="What's the temperature in Boston?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_1.output: + if item.type == "function_call": + print(f"Function called: {item.name} with args: {item.arguments}") + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"temperature": 72, "unit": "F"}), + ) + ) + + response_1 = openai_client.responses.create( + input=input_list, + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "72" in response_1_text, "Should mention 72°F" + + # Turn 2: Context-dependent follow-up (convert the previous number) + print("\n--- Turn 2: Context-dependent conversion ---") + response_2 = openai_client.responses.create( + input="What is that in Celsius?", # "that" refers to the 72°F from previous response + previous_response_id=response_1.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + + # Should convert 72°F to ~22°C (without calling the function again) + # The agent should use the previous response's value + response_2_lower = response_2_text.lower() + assert ( + "celsius" in response_2_lower or "°c" in response_2_lower or "c" in response_2_lower + ), "Response should mention Celsius" + assert any( + temp in response_2_text for temp in ["22", "22.2", "22.22", "20", "21", "23"] + ), f"Response should calculate Celsius from 72°F (~22°C), got: {response_2_text}" + + # Turn 3: Another context-dependent follow-up (comparison) + print("\n--- Turn 3: Compare to another value ---") + response_3 = openai_client.responses.create( + input="Is that warmer or colder than 25°C?", # "that" refers to the Celsius value just mentioned + previous_response_id=response_2.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + + # 22°C is colder than 25°C + response_3_lower = response_3_text.lower() + assert ( + "colder" in response_3_lower or "cooler" in response_3_lower or "lower" in response_3_lower + ), f"Response should indicate 22°C is colder than 25°C, got: {response_3_text}" + + print("\n✓ Context-dependent follow-ups successful!") + print(" - Agent converted temperature from previous response") + print(" - Agent compared values from conversation history") + print(" - No unnecessary function calls made") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py new file mode 100644 index 000000000000..7a8c34ed184f --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_image_generation.py @@ -0,0 +1,132 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import os +import base64 +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, ImageGenTool +from azure.core.exceptions import ResourceNotFoundError + + +class TestAgentImageGeneration(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_image_generation(self, **kwargs): + """ + Test agent with Image Generation tool. + + This test verifies that an agent can: + 1. Use ImageGenTool to generate images from text prompts + 2. Return base64-encoded image data + 3. Decode and validate the image format + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with ImageGenTool) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + # Get the image model deployment name from environment variable + image_model_deployment = os.environ.get( + "AZURE_AI_PROJECTS_TESTS_IMAGE_MODEL_DEPLOYMENT_NAME", "gpt-image-1-mini" + ) + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Check if the image model deployment exists in the project + try: + deployment = project_client.deployments.get(image_model_deployment) + print(f"Image model deployment found: {deployment.name}") + except ResourceNotFoundError: + pytest.skip(f"Image generation model '{image_model_deployment}' not available in this project") + except Exception as e: + pytest.skip(f"Unable to verify image model deployment: {e}") + + # Disable retries for faster failure when service returns 500 + openai_client.max_retries = 0 + + # Create agent with image generation tool + agent = project_client.agents.create_version( + agent_name="image-gen-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Generate images based on user prompts", + tools=[ImageGenTool(quality="low", size="1024x1024")], + ), + description="Agent for testing image generation.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "image-gen-agent" + assert agent.version is not None + + # Request image generation + print("\nAsking agent to generate an image of a simple geometric shape...") + + response = openai_client.responses.create( + input="Generate an image of a blue circle on a white background.", + extra_headers={ + "x-ms-oai-image-generation-deployment": image_model_deployment + }, # Required for image generation + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response created (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Extract image data from response + image_data = [output.result for output in response.output if output.type == "image_generation_call"] + + # Verify image was generated + assert len(image_data) > 0, "Expected at least one image to be generated" + assert image_data[0], "Expected image data to be non-empty" + + print(f"✓ Image data received ({len(image_data[0])} base64 characters)") + + # Decode the base64 image + image_bytes = b"" + try: + image_bytes = base64.b64decode(image_data[0]) + assert len(image_bytes) > 0, "Decoded image should have content" + print(f"✓ Image decoded successfully ({len(image_bytes)} bytes)") + except Exception as e: + pytest.fail(f"Failed to decode base64 image data: {e}") + + # Verify it's a PNG image (check magic bytes) + # PNG files start with: 89 50 4E 47 (‰PNG) + assert image_bytes[:4] == b"\x89PNG", "Image does not appear to be a valid PNG" + print("✓ Image is a valid PNG") + + # Verify reasonable image size (should be > 1KB for a 1024x1024 image) + assert len(image_bytes) > 1024, f"Image seems too small ({len(image_bytes)} bytes)" + print(f"✓ Image size is reasonable ({len(image_bytes):,} bytes)") + + print("\n✓ Agent successfully generated and returned a valid image") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py new file mode 100644 index 000000000000..d56d9b97adba --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_mcp.py @@ -0,0 +1,303 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, MCPTool, Tool +from openai.types.responses.response_input_param import McpApprovalResponse, ResponseInputParam + + +class TestAgentMCP(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_mcp_basic(self, **kwargs): + """ + Test agent with MCP (Model Context Protocol) tool for external API access. + + This test verifies that an agent can: + 1. Use an MCP tool to access external resources (GitHub repo) + 2. Request approval for MCP operations + 3. Process approval responses + 4. Complete the task using the MCP tool + + The test uses a public GitHub MCP server that provides access to + Azure REST API specifications. + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + POST /openai/conversations openai_client.conversations.create() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with MCP tool) + POST /openai/responses openai_client.responses.create() (with approval) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create MCP tool that connects to a public GitHub repo via MCP server + mcp_tool = MCPTool( + server_label="api-specs", + server_url="https://gitmcp.io/Azure/azure-rest-api-specs", + require_approval="always", + ) + + tools: list[Tool] = [mcp_tool] + + # Create agent with MCP tool + agent = project_client.agents.create_version( + agent_name="mcp-basic-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful agent that can use MCP tools to assist users. Use the available MCP tools to answer questions and perform tasks.", + tools=tools, + ), + description="Agent for testing basic MCP functionality.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "mcp-basic-agent" + assert agent.version is not None + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Created conversation (id: {conversation.id})") + assert conversation.id is not None + + # Send initial request that will trigger the MCP tool + print("\nAsking agent to summarize Azure REST API specs README...") + + response = openai_client.responses.create( + conversation=conversation.id, + input="Please summarize the Azure REST API specifications Readme", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Process any MCP approval requests + approval_requests_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "mcp_approval_request": + approval_requests_found += 1 + print(f"Found MCP approval request (id: {item.id}, server: {item.server_label})") + + if item.server_label == "api-specs" and item.id: + # Approve the MCP request + input_list.append( + McpApprovalResponse( + type="mcp_approval_response", + approve=True, + approval_request_id=item.id, + ) + ) + print(f"✓ Approved MCP request: {item.id}") + + # Verify that at least one approval request was generated + assert ( + approval_requests_found > 0 + ), f"Expected at least 1 MCP approval request, but found {approval_requests_found}" + + print(f"\n✓ Processed {approval_requests_found} MCP approval request(s)") + + # Send the approval response to continue the agent's work + print("\nSending approval response to continue agent execution...") + + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Final response completed (id: {response.id})") + assert response.id is not None + + # Get the final response text + response_text = response.output_text + print(f"\nAgent's response preview: {response_text[:200]}...") + + # Verify we got a meaningful response + assert len(response_text) > 100, "Expected a substantial response from the agent" + + # Check that the response mentions Azure or REST API (indicating it accessed the repo) + assert any( + keyword in response_text.lower() for keyword in ["azure", "rest", "api", "specification"] + ), f"Expected response to mention Azure/REST API, but got: {response_text[:200]}" + + print("\n✓ Agent successfully used MCP tool to access GitHub repo and complete task") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_mcp_with_project_connection(self, **kwargs): + """ + Test agent with MCP tool using a project connection for authentication. + + This test verifies that an agent can: + 1. Use an MCP tool with a project connection (GitHub PAT) + 2. Access authenticated GitHub API endpoints + 3. Request and process approval for MCP operations + 4. Return personal GitHub profile information + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + POST /openai/conversations openai_client.conversations.create() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with MCP + connection) + POST /openai/responses openai_client.responses.create() (with approval) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Get MCP project connection from environment + mcp_project_connection_id = kwargs.get("azure_ai_projects_tests_mcp_project_connection_id") + + if not mcp_project_connection_id: + pytest.skip("AZURE_AI_PROJECTS_TESTS_MCP_PROJECT_CONNECTION_ID environment variable not set") + + assert isinstance(mcp_project_connection_id, str), "mcp_project_connection_id must be a string" + print(f"Using MCP project connection: {mcp_project_connection_id}") + + # Create MCP tool with project connection for GitHub API access + mcp_tool = MCPTool( + server_label="github-api", + server_url="https://api.githubcopilot.com/mcp", + require_approval="always", + project_connection_id=mcp_project_connection_id, + ) + + tools: list[Tool] = [mcp_tool] + + # Create agent with MCP tool + agent = project_client.agents.create_version( + agent_name="mcp-connection-agent", + definition=PromptAgentDefinition( + model=model, + instructions="Use MCP tools as needed to access GitHub information.", + tools=tools, + ), + description="Agent for testing MCP with project connection.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "mcp-connection-agent" + assert agent.version is not None + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Created conversation (id: {conversation.id})") + assert conversation.id is not None + + # Send initial request that will trigger the MCP tool with authentication + print("\nAsking agent to get GitHub profile username...") + + response = openai_client.responses.create( + conversation=conversation.id, + input="What is my username in Github profile?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Initial response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Process any MCP approval requests + approval_requests_found = 0 + input_list: ResponseInputParam = [] + + for item in response.output: + if item.type == "mcp_approval_request": + approval_requests_found += 1 + print(f"Found MCP approval request (id: {item.id}, server: {item.server_label})") + + if item.server_label == "github-api" and item.id: + # Approve the MCP request + input_list.append( + McpApprovalResponse( + type="mcp_approval_response", + approve=True, + approval_request_id=item.id, + ) + ) + print(f"✓ Approved MCP request: {item.id}") + + # Verify that at least one approval request was generated + assert ( + approval_requests_found > 0 + ), f"Expected at least 1 MCP approval request, but found {approval_requests_found}" + + print(f"\n✓ Processed {approval_requests_found} MCP approval request(s)") + + # Send the approval response to continue the agent's work + print("\nSending approval response to continue agent execution...") + + response = openai_client.responses.create( + input=input_list, + previous_response_id=response.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Final response completed (id: {response.id})") + assert response.id is not None + + # Get the final response text + response_text = response.output_text + print(f"\nAgent's response: {response_text}") + + # Verify we got a meaningful response with a GitHub username + assert len(response_text) > 5, "Expected a response with a GitHub username" + + # The response should contain some indication of a username or GitHub profile info + # We can't assert the exact username, but we can verify it's not an error + assert ( + "error" not in response_text.lower() or "username" in response_text.lower() + ), f"Expected response to contain GitHub profile info, but got: {response_text}" + + print("\n✓ Agent successfully used MCP tool with project connection to access authenticated GitHub API") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py new file mode 100644 index 000000000000..bad368125fdc --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_tools_with_conversations.py @@ -0,0 +1,468 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + +""" +Test agents using tools within conversations. + +This test file demonstrates how to use various agent tools (both server-side and client-side) +within the context of conversations, testing conversation state management with tool interactions. +""" + +import json +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import ( + FunctionTool, + FileSearchTool, + CodeInterpreterTool, + CodeInterpreterToolAuto, + PromptAgentDefinition, +) +from openai.types.responses.response_input_param import FunctionCallOutput, ResponseInputParam + + +class TestAgentToolsWithConversations(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_function_tool_with_conversation(self, **kwargs): + """ + Test using FunctionTool within a conversation. + + This tests: + - Creating a conversation + - Multiple turns with function calls + - Conversation state preservation across function calls + - Using conversation_id parameter + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Define a calculator function + calculator = FunctionTool( + name="calculator", + description="Perform basic arithmetic operations", + parameters={ + "type": "object", + "properties": { + "operation": { + "type": "string", + "enum": ["add", "subtract", "multiply", "divide"], + "description": "The operation to perform", + }, + "a": {"type": "number", "description": "First number"}, + "b": {"type": "number", "description": "Second number"}, + }, + "required": ["operation", "a", "b"], + "additionalProperties": False, + }, + strict=True, + ) + + # Create agent + agent = project_client.agents.create_version( + agent_name="calculator-agent-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are a calculator assistant. Use the calculator function to perform operations.", + tools=[calculator], + ), + description="Calculator agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Add two numbers + print("\n--- Turn 1: Addition ---") + response_1 = openai_client.responses.create( + input="What is 15 plus 27?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list: ResponseInputParam = [] + for item in response_1.output: + if item.type == "function_call": + print(f"Function called: {item.name} with {item.arguments}") + args = json.loads(item.arguments) + + # Execute calculator + result = { + "add": args["a"] + args["b"], + "subtract": args["a"] - args["b"], + "multiply": args["a"] * args["b"], + "divide": args["a"] / args["b"] if args["b"] != 0 else "Error: Division by zero", + }[args["operation"]] + + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"result": result}), + ) + ) + + response_1 = openai_client.responses.create( + input=input_list, + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response 1: {response_1.output_text[:100]}...") + assert "42" in response_1.output_text + + # Turn 2: Follow-up using previous result (tests conversation memory) + print("\n--- Turn 2: Follow-up using conversation context ---") + response_2 = openai_client.responses.create( + input="Now multiply that result by 2", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + # Handle function call + input_list = [] + for item in response_2.output: + if item.type == "function_call": + print(f"Function called: {item.name} with {item.arguments}") + args = json.loads(item.arguments) + + # Should be multiplying 42 by 2 + assert args["operation"] == "multiply" + assert args["a"] == 42 or args["b"] == 42 + + result = args["a"] * args["b"] + input_list.append( + FunctionCallOutput( + type="function_call_output", + call_id=item.call_id, + output=json.dumps({"result": result}), + ) + ) + + response_2 = openai_client.responses.create( + input=input_list, + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + print(f"Response 2: {response_2.output_text[:100]}...") + assert "84" in response_2.output_text + + print("\n✓ Function tool with conversation successful!") + print(" - Conversation preserved state across function calls") + print(" - Agent remembered previous result (42)") + + # Verify conversation state by reading items + print("\n--- Verifying conversation state ---") + all_items = list(openai_client.conversations.items.list(conversation.id)) + print(f"Total conversation items: {len(all_items)}") + + # Count different item types + user_messages = sum(1 for item in all_items if item.type == "message" and item.role == "user") + assistant_messages = sum(1 for item in all_items if item.type == "message" and item.role == "assistant") + function_calls = sum(1 for item in all_items if item.type == "function_call") + function_outputs = sum(1 for item in all_items if item.type == "function_call_output") + + print(f" User messages: {user_messages}") + print(f" Assistant messages: {assistant_messages}") + print(f" Function calls: {function_calls}") + print(f" Function outputs: {function_outputs}") + + # Verify we have expected items + assert user_messages >= 2, "Expected at least 2 user messages (two turns)" + assert function_calls >= 2, "Expected at least 2 function calls (one per turn)" + assert function_outputs >= 2, "Expected at least 2 function outputs" + print("✓ Conversation state verified - all items preserved") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_file_search_with_conversation(self, **kwargs): + """ + Test using FileSearchTool within a conversation. + + This tests: + - Server-side tool execution within conversation + - Multiple search queries in same conversation + - Conversation context retention + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create documents with related information + doc_content = """Product Catalog + +Widget A: +- Price: $100 +- Stock: 50 units +- Rating: 4.5/5 +- Category: Electronics + +Widget B: +- Price: $220 +- Stock: 30 units +- Rating: 4.8/5 +- Category: Electronics + +Widget C: +- Price: $75 +- Stock: 100 units +- Rating: 4.2/5 +- Category: Home Goods +""" + + # Create vector store and upload document + vector_store = openai_client.vector_stores.create(name="ConversationTestStore") + print(f"Vector store created: {vector_store.id}") + + from io import BytesIO + + file = BytesIO(doc_content.encode("utf-8")) + file.name = "products.txt" + + uploaded = openai_client.vector_stores.files.upload_and_poll( + vector_store_id=vector_store.id, + file=file, + ) + print(f"Document uploaded: {uploaded.id}") + + # Create agent with file search + agent = project_client.agents.create_version( + agent_name="search-agent-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are a product search assistant. Answer questions about products.", + tools=[FileSearchTool(vector_store_ids=[vector_store.id])], + ), + description="Search agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Search for highest rated + print("\n--- Turn 1: Search query ---") + response_1 = openai_client.responses.create( + input="Which widget has the highest rating?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:150]}...") + assert "Widget B" in response_1_text or "4.8" in response_1_text + + # Turn 2: Follow-up about that specific product (tests context retention) + print("\n--- Turn 2: Contextual follow-up ---") + response_2 = openai_client.responses.create( + input="What is its price?", # "its" refers to Widget B from previous turn + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:150]}...") + assert "220" in response_2_text + + # Turn 3: New search in same conversation + print("\n--- Turn 3: New search in same conversation ---") + response_3 = openai_client.responses.create( + input="Which widget is in the Home Goods category?", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:150]}...") + assert "Widget C" in response_3_text + + print("\n✓ File search with conversation successful!") + print(" - Multiple searches in same conversation") + print(" - Context preserved (agent remembered Widget B)") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + openai_client.vector_stores.delete(vector_store.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_code_interpreter_with_conversation(self, **kwargs): + """ + Test using CodeInterpreterTool within a conversation. + + This tests: + - Server-side code execution within conversation + - Multiple code executions in same conversation + - Variables/state persistence across turns + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with code interpreter + agent = project_client.agents.create_version( + agent_name="code-agent-conversation", + definition=PromptAgentDefinition( + model=model, + instructions="You are a data analysis assistant. Use Python to perform calculations.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[]))], + ), + description="Code interpreter agent for conversation testing.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Turn 1: Calculate average + print("\n--- Turn 1: Calculate average ---") + response_1 = openai_client.responses.create( + input="Calculate the average of these numbers: 10, 20, 30, 40, 50", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + assert "30" in response_1_text + + # Turn 2: Follow-up calculation (tests conversation context) + print("\n--- Turn 2: Follow-up calculation ---") + response_2 = openai_client.responses.create( + input="Now calculate the standard deviation of those same numbers", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_2_text = response_2.output_text + print(f"Response 2: {response_2_text[:200]}...") + # Standard deviation should be approximately 14.14 or similar + assert any(num in response_2_text for num in ["14", "15", "standard"]) + + # Turn 3: Another operation in same conversation + print("\n--- Turn 3: New calculation ---") + response_3 = openai_client.responses.create( + input="Create a list of squares from 1 to 5", + conversation=conversation.id, + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_3_text = response_3.output_text + print(f"Response 3: {response_3_text[:200]}...") + assert "1" in response_3_text and "4" in response_3_text and "25" in response_3_text + + print("\n✓ Code interpreter with conversation successful!") + print(" - Multiple code executions in conversation") + print(" - Context preserved across calculations") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + print("Cleanup completed") + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_code_interpreter_with_file_in_conversation(self, **kwargs): + """ + Test using CodeInterpreterTool with file upload within a conversation. + + This test reproduces the 500 error seen in the sample when using + code interpreter with uploaded files in conversations. + + This tests: + - Uploading a real file (not BytesIO) for code interpreter + - Using code interpreter with files in conversation + - Server-side code execution with file access and chart generation + """ + + model = self.test_agents_params["model_deployment_name"] + import os + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Use the same CSV file as the sample + asset_file_path = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "../../../samples/agents/assets/synthetic_500_quarterly_results.csv", + ) + ) + + # Upload file using open() with rb mode, just like the sample + with open(asset_file_path, "rb") as f: + uploaded_file = openai_client.files.create(file=f, purpose="assistants") + print(f"File uploaded: {uploaded_file.id}") + + # Create agent with code interpreter - matching sample exactly + agent = project_client.agents.create_version( + agent_name="agent-code-interpreter-with-file-pbatum1", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant.", + tools=[CodeInterpreterTool(container=CodeInterpreterToolAuto(file_ids=[uploaded_file.id]))], + ), + description="Code interpreter agent for data analysis and visualization.", + ) + print(f"Agent created: {agent.id}") + + # Create conversation + conversation = openai_client.conversations.create() + print(f"Conversation created: {conversation.id}") + + # Use the same prompt as the sample - requesting chart generation + print("\n--- Turn 1: Create bar chart ---") + response_1 = openai_client.responses.create( + conversation=conversation.id, + input="Could you please create bar chart in TRANSPORTATION sector for the operating profit from the uploaded csv file and provide file to me?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + response_1_text = response_1.output_text + print(f"Response 1: {response_1_text[:200]}...") + + print("\n✓ Code interpreter with file in conversation successful!") + + # Cleanup + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + openai_client.conversations.delete(conversation_id=conversation.id) + openai_client.files.delete(uploaded_file.id) + print("Cleanup completed") diff --git a/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py new file mode 100644 index 000000000000..30e896825e64 --- /dev/null +++ b/sdk/ai/azure-ai-projects/tests/agents/tools/test_agent_web_search.py @@ -0,0 +1,99 @@ +# pylint: disable=too-many-lines,line-too-long,useless-suppression +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +# cSpell:disable + +import pytest +from test_base import TestBase, servicePreparer +from devtools_testutils import is_live_and_not_recording +from azure.ai.projects.models import PromptAgentDefinition, WebSearchPreviewTool, ApproximateLocation + + +class TestAgentWebSearch(TestBase): + + @servicePreparer() + @pytest.mark.skipif( + condition=(not is_live_and_not_recording()), + reason="Skipped because we cannot record network calls with OpenAI client", + ) + def test_agent_web_search(self, **kwargs): + """ + Test agent with Web Search tool for real-time information. + + This test verifies that an agent can: + 1. Use WebSearchPreviewTool to search the web + 2. Get current/real-time information + 3. Provide answers based on web search results + + Routes used in this test: + + Action REST API Route Client Method + ------+---------------------------------------------+----------------------------------- + # Setup: + POST /agents/{agent_name}/versions project_client.agents.create_version() + + # Test focus: + POST /openai/responses openai_client.responses.create() (with WebSearchPreviewTool) + + # Teardown: + DELETE /agents/{agent_name}/versions/{agent_version} project_client.agents.delete_version() + """ + + model = self.test_agents_params["model_deployment_name"] + + with ( + self.create_client(operation_group="agents", **kwargs) as project_client, + project_client.get_openai_client() as openai_client, + ): + # Create agent with web search tool + agent = project_client.agents.create_version( + agent_name="web-search-agent", + definition=PromptAgentDefinition( + model=model, + instructions="You are a helpful assistant that can search the web for current information.", + tools=[ + WebSearchPreviewTool( + user_location=ApproximateLocation(country="US", city="Seattle", region="Washington") + ) + ], + ), + description="Agent for testing web search capabilities.", + ) + print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})") + assert agent.id is not None + assert agent.name == "web-search-agent" + assert agent.version is not None + + # Ask a question that requires web search for current information + print("\nAsking agent about current weather...") + + response = openai_client.responses.create( + input="What is the current weather in Seattle?", + extra_body={"agent": {"name": agent.name, "type": "agent_reference"}}, + ) + + print(f"Response completed (id: {response.id})") + assert response.id is not None + assert response.output is not None + assert len(response.output) > 0 + + # Get the response text + response_text = response.output_text + print(f"\nAgent's response: {response_text[:300]}...") + + # Verify we got a meaningful response + assert len(response_text) > 30, "Expected a substantial response from the agent" + + # The response should mention weather-related terms or Seattle + response_lower = response_text.lower() + assert any( + keyword in response_lower for keyword in ["weather", "temperature", "seattle", "forecast"] + ), f"Expected response to contain weather information, but got: {response_text[:200]}" + + print("\n✓ Agent successfully used web search tool to get current information") + + # Teardown + project_client.agents.delete_version(agent_name=agent.name, agent_version=agent.version) + print("Agent deleted") diff --git a/sdk/ai/azure-ai-projects/tests/test_base.py b/sdk/ai/azure-ai-projects/tests/test_base.py index c371560cb41b..5260e44c38d6 100644 --- a/sdk/ai/azure-ai-projects/tests/test_base.py +++ b/sdk/ai/azure-ai-projects/tests/test_base.py @@ -62,6 +62,11 @@ azure_ai_projects_tests_tracing_project_endpoint="https://sanitized-account-name.services.ai.azure.com/api/projects/sanitized-project-name", azure_ai_projects_tests_container_app_resource_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/00000/providers/Microsoft.App/containerApps/00000", azure_ai_projects_tests_container_ingress_subdomain_suffix="00000", + azure_ai_projects_tests_bing_project_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-bing-connection", + azure_ai_projects_tests_ai_search_project_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-ai-search-connection", + azure_ai_projects_tests_ai_search_index_name="sanitized-index-name", + azure_ai_projects_tests_mcp_project_connection_id="/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/sanitized-resource-group/providers/Microsoft.CognitiveServices/accounts/sanitized-account/projects/sanitized-project/connections/sanitized-mcp-connection", + azure_ai_projects_tests_image_model_deployment_name="gpt-image-1-mini", )