From fdeb1c583d2adf4070a6f8c6f86d09828e629af9 Mon Sep 17 00:00:00 2001 From: Taylor Date: Fri, 22 Aug 2025 11:12:25 +1000 Subject: [PATCH 01/11] Update coverage report generation command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix command to generate HTML report for coverage using `diff-cover` ๐Ÿ› ๏ธ --- CONTRIBUTING.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d0b2e83c97..a320933a62 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,14 +12,15 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. -- [Submitting a Pull Request (PR)](#submitting-a-pull-request-pr) -- [Setting up the development environment](#setting-up-the-development-environment) -- [Running unit tests](#running-unit-tests) -- [Running E2E tests](#running-e2e-tests) -- [Code style](#code-style) -- [Adding new features](#adding-new-features) - - [Adding new azd environment variables](#adding-new-azd-environment-variables) - - [Adding new UI strings](#adding-new-ui-strings) +- [Contributing](#contributing) + - [Submitting a Pull Request (PR)](#submitting-a-pull-request-pr) + - [Setting up the development environment](#setting-up-the-development-environment) + - [Running unit tests](#running-unit-tests) + - [Running E2E tests](#running-e2e-tests) + - [Code style](#code-style) + - [Adding new features](#adding-new-features) + - [Adding new azd environment variables](#adding-new-azd-environment-variables) + - [Adding new UI strings](#adding-new-ui-strings) ## Submitting a Pull Request (PR) @@ -73,7 +74,7 @@ Once tests are passing, generate a coverage report to make sure your changes are ```shell pytest --cov --cov-report=xml && \ -diff-cover coverage.xml --format html:coverage_report.html && \ +diff-cover coverage.xml --html-report coverage_report.html && \ open coverage_report.html ``` From a999085619ab40f7abb82265217583e50b93e341 Mon Sep 17 00:00:00 2001 From: Taylor Date: Fri, 22 Aug 2025 11:13:22 +1000 Subject: [PATCH 02/11] Add support for agentic reference hydration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Introduce ENABLE_AGENTIC_REF_HYDRATION environment variable to control reference hydration behaviour ๐ŸŒฑ - Update Approach classes to accept hydrate_references parameter for managing reference hydration logic ๐Ÿ”ง - Modify document retrieval logic to hydrate references when enabled, improving data completeness ๐Ÿ“„ --- app/backend/app.py | 3 + app/backend/approaches/approach.py | 120 ++++++++++++++---- .../approaches/chatreadretrieveread.py | 2 + app/backend/approaches/retrievethenread.py | 2 + 4 files changed, 104 insertions(+), 23 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index 8466dd7498..f067d82bf2 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -488,6 +488,7 @@ async def setup_clients(): USE_CHAT_HISTORY_BROWSER = os.getenv("USE_CHAT_HISTORY_BROWSER", "").lower() == "true" USE_CHAT_HISTORY_COSMOS = os.getenv("USE_CHAT_HISTORY_COSMOS", "").lower() == "true" USE_AGENTIC_RETRIEVAL = os.getenv("USE_AGENTIC_RETRIEVAL", "").lower() == "true" + ENABLE_AGENTIC_REF_HYDRATION = os.getenv("ENABLE_AGENTIC_REF_HYDRATION", "").lower() == "true" # WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None @@ -707,6 +708,7 @@ async def setup_clients(): query_speller=AZURE_SEARCH_QUERY_SPELLER, prompt_manager=prompt_manager, reasoning_effort=OPENAI_REASONING_EFFORT, + hydrate_references=ENABLE_AGENTIC_REF_HYDRATION, ) # ChatReadRetrieveReadApproach is used by /chat for multi-turn conversation @@ -730,6 +732,7 @@ async def setup_clients(): query_speller=AZURE_SEARCH_QUERY_SPELLER, prompt_manager=prompt_manager, reasoning_effort=OPENAI_REASONING_EFFORT, + hydrate_references=ENABLE_AGENTIC_REF_HYDRATION, ) if USE_GPT4V: diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index 6248f4ff2c..6d984c4b68 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -161,6 +161,7 @@ def __init__( vision_token_provider: Callable[[], Awaitable[str]], prompt_manager: PromptManager, reasoning_effort: Optional[str] = None, + hydrate_references: bool = False, ): self.search_client = search_client self.openai_client = openai_client @@ -176,6 +177,7 @@ def __init__( self.vision_token_provider = vision_token_provider self.prompt_manager = prompt_manager self.reasoning_effort = reasoning_effort + self.hydrate_references = hydrate_references self.include_token_usage = True def build_filter(self, overrides: dict[str, Any], auth_claims: dict[str, Any]) -> Optional[str]: @@ -229,7 +231,7 @@ async def search( vector_queries=search_vectors, ) - documents = [] + documents: list[Document] = [] async for page in results.by_page(): async for document in page: documents.append( @@ -291,40 +293,112 @@ async def run_agentic_retrieval( ) ) - # STEP 2: Generate a contextual and content specific answer using the search results and chat history + # Map activity id -> agent's internal search query activities = response.activity - activity_mapping = ( + activity_mapping: dict[int, str] = ( { - activity.id: activity.query.search if activity.query else "" + activity.id: activity.query.search for activity in activities - if isinstance(activity, KnowledgeAgentSearchActivityRecord) + if ( + isinstance(activity, KnowledgeAgentSearchActivityRecord) + and activity.query + and activity.query.search is not None + ) } if activities else {} ) - results = [] - if response and response.references: - if results_merge_strategy == "interleaved": - # Use interleaved reference order - references = sorted(response.references, key=lambda reference: int(reference.id)) - else: - # Default to descending strategy - references = response.references - for reference in references: - if isinstance(reference, KnowledgeAgentAzureSearchDocReference) and reference.source_data: - results.append( + # No refs? we're done + if not (response and response.references): + return response, [] + + # Extract references + refs = [r for r in response.references if isinstance(r, KnowledgeAgentAzureSearchDocReference)] + + documents: list[Document] = [] + + if self.hydrate_references: + # Hydrate references to get full documents + documents = await self.hydrate_agent_references( + references=refs, + top=top, + ) + else: + # Create documents from reference source data + for ref in refs: + if ref.source_data: + documents.append( Document( - id=reference.doc_key, - content=reference.source_data["content"], - sourcepage=reference.source_data["sourcepage"], - search_agent_query=activity_mapping[reference.activity_source], + id=ref.doc_key, + content=ref.source_data.get("content"), + sourcepage=ref.source_data.get("sourcepage"), ) ) - if top and len(results) == top: - break + if top and len(documents) >= top: + break + + # Build mappings for agent queries and sorting + ref_to_activity: dict[str, int] = {} + doc_to_ref_id: dict[str, str] = {} + for ref in refs: + if ref.doc_key: + ref_to_activity[ref.doc_key] = ref.activity_source + doc_to_ref_id[ref.doc_key] = ref.id + + # Inject agent search queries into all documents + for doc in documents: + if doc.id and doc.id in ref_to_activity: + activity_id = ref_to_activity[doc.id] + doc.search_agent_query = activity_mapping.get(activity_id, "") + + # Apply sorting strategy to the documents + if results_merge_strategy == "interleaved": # Use interleaved reference order + documents = sorted( + documents, + key=lambda d: int(doc_to_ref_id.get(d.id, 0)) if d.id and doc_to_ref_id.get(d.id) else 0, + ) + # else: Default - preserve original order + + return response, documents + + async def hydrate_agent_references( + self, + references: list[KnowledgeAgentAzureSearchDocReference], + top: Optional[int], + ) -> list[Document]: + doc_keys: set[str] = set() + + for ref in references: + if not ref.doc_key: + continue + doc_keys.add(ref.doc_key) + if top and len(doc_keys) >= top: + break + + if not doc_keys: + return [] + + # Build search filter only on unique doc IDs + id_csv = ",".join(doc_keys) + id_filter = f"search.in(id, '{id_csv}', ',')" + + # Fetch full documents + hydrated_docs: list[Document] = await self.search( + top=len(doc_keys), + query_text=None, + filter=id_filter, + vectors=[], + use_text_search=False, + use_vector_search=False, + use_semantic_ranker=False, + use_semantic_captions=False, + minimum_search_score=None, + minimum_reranker_score=None, + use_query_rewriting=False, + ) - return response, results + return hydrated_docs def get_sources_content( self, results: list[Document], use_semantic_captions: bool, use_image_citation: bool diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 55e09c46a6..eb320b0e32 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -47,6 +47,7 @@ def __init__( query_speller: str, prompt_manager: PromptManager, reasoning_effort: Optional[str] = None, + hydrate_references: bool = False, ): self.search_client = search_client self.search_index_name = search_index_name @@ -70,6 +71,7 @@ def __init__( self.query_rewrite_tools = self.prompt_manager.load_tools("chat_query_rewrite_tools.json") self.answer_prompt = self.prompt_manager.load_prompt("chat_answer_question.prompty") self.reasoning_effort = reasoning_effort + self.hydrate_references = hydrate_references self.include_token_usage = True async def run_until_final_call( diff --git a/app/backend/approaches/retrievethenread.py b/app/backend/approaches/retrievethenread.py index d59f903b0e..f937252e73 100644 --- a/app/backend/approaches/retrievethenread.py +++ b/app/backend/approaches/retrievethenread.py @@ -40,6 +40,7 @@ def __init__( query_speller: str, prompt_manager: PromptManager, reasoning_effort: Optional[str] = None, + hydrate_references: bool = False, ): self.search_client = search_client self.search_index_name = search_index_name @@ -63,6 +64,7 @@ def __init__( self.answer_prompt = self.prompt_manager.load_prompt("ask_answer_question.prompty") self.reasoning_effort = reasoning_effort self.include_token_usage = True + self.hydrate_references = hydrate_references async def run( self, From 226b47890c94d1ad33e308e9515bf4a02729ca55 Mon Sep 17 00:00:00 2001 From: Taylor Date: Fri, 22 Aug 2025 11:13:45 +1000 Subject: [PATCH 03/11] Enhance agentic retrieval with optional field hydration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - โœจ Add support for enabling extra field hydration in agentic retrieval - ๐Ÿ”ง Update infrastructure to include new parameter for hydration - ๐Ÿ“ Modify documentation to reflect changes in usage instructions --- docs/agentic_retrieval.md | 18 ++++++++++++++---- infra/main.bicep | 2 ++ infra/main.parameters.json | 3 +++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/agentic_retrieval.md b/docs/agentic_retrieval.md index 0f8f8e69a9..1774a950b0 100644 --- a/docs/agentic_retrieval.md +++ b/docs/agentic_retrieval.md @@ -34,21 +34,31 @@ See the agentic retrieval documentation. azd env set AZURE_OPENAI_SEARCHAGENT_MODEL_VERSION 2025-04-14 ``` -3. **Update the infrastructure and application:** +3. **(Optional) Enable extra field hydration** + + By default, agentic retrieval only returns fields included in the semantic configuration. + + You can enable this optional feature below, to include all fields from the search index in the result. + + ```shell + azd env set ENABLE_AGENTIC_REF_HYDRATION true + ``` + +4. **Update the infrastructure and application:** Execute `azd up` to provision the infrastructure changes (only the new model, if you ran `up` previously) and deploy the application code with the updated environment variables. -4. **Try out the feature:** +5. **Try out the feature:** Open the web app and start a new chat. Agentic retrieval will be used to find all sources. -5. **Experiment with max subqueries:** +6. **Experiment with max subqueries:** Select the developer options in the web app and change max subqueries to any value between 1 and 20. This controls the maximum amount of subqueries that can be created in the query plan. ![Max subqueries screenshot](./images/max-subqueries.png) -6. **Review the query plan** +7. **Review the query plan** Agentic retrieval use additional billed tokens behind the scenes for the planning process. To see the token usage, select the lightbulb icon on a chat answer. This will open the "Thought process" tab, which shows the amount of tokens used by and the queries produced by the planning process diff --git a/infra/main.bicep b/infra/main.bicep index d74047f8bc..ab64cb8cdc 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -41,6 +41,7 @@ param storageSkuName string // Set in main.parameters.json param defaultReasoningEffort string // Set in main.parameters.json param useAgenticRetrieval bool // Set in main.parameters.json +param enableAgenticRefHydration bool // Set in main.parameters.json param userStorageAccountName string = '' param userStorageContainerName string = 'user-content' @@ -424,6 +425,7 @@ var appEnvVariables = { USE_SPEECH_OUTPUT_BROWSER: useSpeechOutputBrowser USE_SPEECH_OUTPUT_AZURE: useSpeechOutputAzure USE_AGENTIC_RETRIEVAL: useAgenticRetrieval + ENABLE_AGENTIC_REF_HYDRATION: enableAgenticRefHydration // Chat history settings USE_CHAT_HISTORY_BROWSER: useChatHistoryBrowser USE_CHAT_HISTORY_COSMOS: useChatHistoryCosmos diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 5202fdcde0..482019a037 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -355,6 +355,9 @@ }, "useAgenticRetrieval": { "value": "${USE_AGENTIC_RETRIEVAL=false}" + }, + "enableAgenticRefHydration": { + "value": "${ENABLE_AGENTIC_REF_HYDRATION=false}" } } } From 2f0e45eb8108ea253fe2e8d621fa7a12d675066e Mon Sep 17 00:00:00 2001 From: Taylor Date: Fri, 22 Aug 2025 11:14:10 +1000 Subject: [PATCH 04/11] Add tests for hydration support for agentic retrieval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ๐ŸŽ‰ Introduce ENABLE_AGENTIC_REF_HYDRATION environment variable for configuration - ๐Ÿงช Implement mock search results for hydration testing in agentic retrieval - ๐Ÿ” Create tests for agentic retrieval with and without hydration enabled - ๐Ÿ“œ Ensure hydrated results include additional fields from search results --- tests/conftest.py | 2 + tests/mocks.py | 66 +++ tests/test_approach_agentic_retrieval.py | 586 +++++++++++++++++++++++ tests/test_chatapproach.py | 86 ++++ 4 files changed, 740 insertions(+) create mode 100644 tests/test_approach_agentic_retrieval.py diff --git a/tests/conftest.py b/tests/conftest.py index 2405dad8e9..fe93991058 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -379,6 +379,7 @@ def mock_blob_container_client(monkeypatch): "AZURE_OPENAI_SEARCHAGENT_MODEL": "gpt-4.1-mini", "AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT": "gpt-4.1-mini", "USE_AGENTIC_RETRIEVAL": "true", + "ENABLE_AGENTIC_REF_HYDRATION": "true" } ] @@ -392,6 +393,7 @@ def mock_blob_container_client(monkeypatch): "AZURE_OPENAI_SEARCHAGENT_MODEL": "gpt-4.1-mini", "AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT": "gpt-4.1-mini", "USE_AGENTIC_RETRIEVAL": "true", + "ENABLE_AGENTIC_REF_HYDRATION": "true", "AZURE_USE_AUTHENTICATION": "true", "AZURE_SERVER_APP_ID": "SERVER_APP", "AZURE_SERVER_APP_SECRET": "SECRET", diff --git a/tests/mocks.py b/tests/mocks.py index fae3022f94..5b8d449a21 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -126,6 +126,72 @@ def __init__(self, search_text, vector_queries: Optional[list[VectorQuery]]): } ] ] + elif search_text == "hydrated": + # Mock search results for hydration testing with complete data + self.data = [ + [ + { + "sourcepage": "Benefit_Options-2.pdf", + "sourcefile": "Benefit_Options.pdf", + "content": "There is a whistleblower policy.", + "embedding": [], + "category": "benefits", + "id": "Benefit_Options-2.pdf", + "@search.score": 0.03279569745063782, + "@search.reranker_score": 3.4577205181121826, + "@search.highlights": None, + "@search.captions": [MockCaption("Caption: A whistleblower policy.")], + }, + ] + ] + elif search_text == "hydrated_multi": + # Mock search results for multiple document hydration + self.data = [ + [ + { + "id": "doc1", + "content": "Hydrated content 1", + "sourcepage": "page1.pdf", + "sourcefile": "file1.pdf", + "category": "category1", + "@search.score": 0.9, + "@search.reranker_score": 3.5, + "@search.highlights": None, + "@search.captions": [], + }, + { + "id": "doc2", + "content": "Hydrated content 2", + "sourcepage": "page2.pdf", + "sourcefile": "file2.pdf", + "category": "category2", + "@search.score": 0.8, + "@search.reranker_score": 3.2, + "@search.highlights": None, + "@search.captions": [], + }, + ] + ] + elif search_text == "hydrated_single": + # Mock search results for single document hydration + self.data = [ + [ + { + "id": "doc1", + "content": "Hydrated content 1", + "sourcepage": "page1.pdf", + "sourcefile": "file1.pdf", + "category": "category1", + "@search.score": 0.9, + "@search.reranker_score": 3.5, + "@search.highlights": None, + "@search.captions": [], + }, + ] + ] + elif search_text == "hydrated_empty": + # Mock search results for empty hydration + self.data = [[]] else: self.data = [ [ diff --git a/tests/test_approach_agentic_retrieval.py b/tests/test_approach_agentic_retrieval.py new file mode 100644 index 0000000000..1f099d557e --- /dev/null +++ b/tests/test_approach_agentic_retrieval.py @@ -0,0 +1,586 @@ +import pytest +from azure.core.credentials import AzureKeyCredential +from azure.search.documents.agent.aio import KnowledgeAgentRetrievalClient +from azure.search.documents.agent.models import ( + KnowledgeAgentAzureSearchDocReference, + KnowledgeAgentMessage, + KnowledgeAgentMessageTextContent, + KnowledgeAgentModelQueryPlanningActivityRecord, + KnowledgeAgentRetrievalResponse, + KnowledgeAgentSearchActivityRecord, + KnowledgeAgentSearchActivityRecordQuery, +) +from azure.search.documents.aio import SearchClient + +from approaches.approach import Approach, Document +from approaches.promptmanager import PromptyManager +from core.authentication import AuthenticationHelper + +from .mocks import ( + MOCK_EMBEDDING_DIMENSIONS, + MOCK_EMBEDDING_MODEL_NAME, + MockAsyncSearchResultsIterator, + MockAzureCredential, +) + + +class MockApproach(Approach): + """Concrete implementation of abstract Approach for testing""" + + async def run(self, messages, session_state=None, context={}): + pass + + async def run_stream(self, messages, session_state=None, context={}): + pass + + +@pytest.fixture +def approach(): + """Create a test approach instance""" + return MockApproach( + search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), + openai_client=None, + auth_helper=AuthenticationHelper( + search_index=None, + use_authentication=False, + server_app_id=None, + server_app_secret=None, + client_app_id=None, + tenant_id=None, + ), + query_language="en-us", + query_speller="lexicon", + embedding_deployment="embeddings", + embedding_model=MOCK_EMBEDDING_MODEL_NAME, + embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, + embedding_field="embedding", + openai_host="", + vision_endpoint="", + vision_token_provider=lambda: MockAzureCredential().get_token(""), + prompt_manager=PromptyManager(), + hydrate_references=False, + ) + + +@pytest.fixture +def hydrating_approach(): + """Create a test approach instance with hydration enabled""" + return MockApproach( + search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), + openai_client=None, + auth_helper=AuthenticationHelper( + search_index=None, + use_authentication=False, + server_app_id=None, + server_app_secret=None, + client_app_id=None, + tenant_id=None, + ), + query_language="en-us", + query_speller="lexicon", + embedding_deployment="embeddings", + embedding_model=MOCK_EMBEDDING_MODEL_NAME, + embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, + embedding_field="embedding", + openai_host="", + vision_endpoint="", + vision_token_provider=lambda: MockAzureCredential().get_token(""), + prompt_manager=PromptyManager(), + hydrate_references=True, + ) + + +def mock_retrieval_response_with_sorting(): + """Mock response with multiple references for testing sorting""" + return KnowledgeAgentRetrievalResponse( + response=[ + KnowledgeAgentMessage( + role="assistant", + content=[KnowledgeAgentMessageTextContent(text="Test response")], + ) + ], + activity=[ + KnowledgeAgentSearchActivityRecord( + id=1, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="first query"), + count=10, + elapsed_ms=50, + ), + KnowledgeAgentSearchActivityRecord( + id=2, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="second query"), + count=10, + elapsed_ms=50, + ), + ], + references=[ + KnowledgeAgentAzureSearchDocReference( + id="2", # Higher ID for testing interleaved sorting + activity_source=2, + doc_key="doc2", + source_data={"content": "Content 2", "sourcepage": "page2.pdf"}, + ), + KnowledgeAgentAzureSearchDocReference( + id="1", # Lower ID for testing interleaved sorting + activity_source=1, + doc_key="doc1", + source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, + ), + ], + ) + + +def mock_retrieval_response_with_duplicates(): + """Mock response with duplicate doc_keys for testing deduplication""" + return KnowledgeAgentRetrievalResponse( + response=[ + KnowledgeAgentMessage( + role="assistant", + content=[KnowledgeAgentMessageTextContent(text="Test response")], + ) + ], + activity=[ + KnowledgeAgentSearchActivityRecord( + id=1, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="query for doc1"), + count=10, + elapsed_ms=50, + ), + KnowledgeAgentSearchActivityRecord( + id=2, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="another query for doc1"), + count=10, + elapsed_ms=50, + ), + ], + references=[ + KnowledgeAgentAzureSearchDocReference( + id="1", + activity_source=1, + doc_key="doc1", # Same doc_key + source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, + ), + KnowledgeAgentAzureSearchDocReference( + id="2", + activity_source=2, + doc_key="doc1", # Duplicate doc_key + source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, + ), + KnowledgeAgentAzureSearchDocReference( + id="3", + activity_source=1, + doc_key="doc2", # Different doc_key + source_data={"content": "Content 2", "sourcepage": "page2.pdf"}, + ), + ], + ) + + +async def mock_search_for_hydration(*args, **kwargs): + """Mock search that returns documents matching the filter""" + filter_param = kwargs.get("filter", "") + + # Create documents based on filter - use search_text to distinguish different calls + search_text = "" + if "doc1" in filter_param and "doc2" in filter_param: + search_text = "hydrated_multi" + elif "doc1" in filter_param: + search_text = "hydrated_single" + else: + search_text = "hydrated_empty" + + return MockAsyncSearchResultsIterator(search_text, None) + + +@pytest.mark.asyncio +async def test_agentic_retrieval_non_hydrated_default_sort(approach, monkeypatch): + """Test non-hydrated path with default sorting (preserve original order)""" + + async def mock_retrieval(*args, **kwargs): + return mock_retrieval_response_with_sorting() + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index", + results_merge_strategy=None # Default sorting + ) + + assert len(results) == 2 + # Default sorting preserves original order (doc2, doc1) + assert results[0].id == "doc2" + assert results[0].content == "Content 2" + assert results[0].search_agent_query == "second query" + + assert results[1].id == "doc1" + assert results[1].content == "Content 1" + assert results[1].search_agent_query == "first query" + + +@pytest.mark.asyncio +async def test_agentic_retrieval_non_hydrated_interleaved_sort(approach, monkeypatch): + """Test non-hydrated path with interleaved sorting""" + + async def mock_retrieval(*args, **kwargs): + return mock_retrieval_response_with_sorting() + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index", + results_merge_strategy="interleaved" + ) + + assert len(results) == 2 + # Interleaved sorting orders by reference ID (1, 2) + assert results[0].id == "doc1" # ref.id = "1" + assert results[0].content == "Content 1" + assert results[0].search_agent_query == "first query" + + assert results[1].id == "doc2" # ref.id = "2" + assert results[1].content == "Content 2" + assert results[1].search_agent_query == "second query" + + +@pytest.mark.asyncio +async def test_agentic_retrieval_hydrated_with_sorting(hydrating_approach, monkeypatch): + """Test hydrated path with sorting""" + + async def mock_retrieval(*args, **kwargs): + return mock_retrieval_response_with_sorting() + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(SearchClient, "search", mock_search_for_hydration) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await hydrating_approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index", + results_merge_strategy="interleaved" + ) + + assert len(results) == 2 + # Should have hydrated content, not source_data content + assert results[0].content == "Hydrated content 1" + assert results[1].content == "Hydrated content 2" + # Should still have agent queries injected + assert results[0].search_agent_query == "first query" + assert results[1].search_agent_query == "second query" + + +@pytest.mark.asyncio +async def test_hydrate_agent_references_deduplication(hydrating_approach, monkeypatch): + """Test that hydrate_agent_references deduplicates doc_keys""" + + async def mock_retrieval(*args, **kwargs): + return mock_retrieval_response_with_duplicates() + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(SearchClient, "search", mock_search_for_hydration) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await hydrating_approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index" + ) + + # Should only get 2 unique documents despite 3 references (doc1 appears twice) + assert len(results) == 2 + doc_ids = [doc.id for doc in results] + assert "doc1" in doc_ids + assert "doc2" in doc_ids + + +@pytest.mark.asyncio +async def test_agentic_retrieval_no_references(approach, monkeypatch): + """Test behavior when agent returns no references""" + + async def mock_retrieval(*args, **kwargs): + return KnowledgeAgentRetrievalResponse( + response=[KnowledgeAgentMessage(role="assistant", content=[])], + activity=[], + references=[] + ) + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index" + ) + + assert len(results) == 0 + + +@pytest.mark.asyncio +async def test_activity_mapping_injection(approach, monkeypatch): + """Test that search_agent_query is properly injected from activity mapping""" + + async def mock_retrieval(*args, **kwargs): + return mock_retrieval_response_with_sorting() + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index" + ) + + # Verify that search_agent_query is correctly mapped from activity + assert len(results) == 2 + + # Find each document and verify its query + doc1 = next(doc for doc in results if doc.id == "doc1") + doc2 = next(doc for doc in results if doc.id == "doc2") + + assert doc1.search_agent_query == "first query" # From activity_source=1 + assert doc2.search_agent_query == "second query" # From activity_source=2 + + +def mock_retrieval_response_with_missing_doc_key(): + """Mock response with missing doc_key to test continue condition""" + return KnowledgeAgentRetrievalResponse( + response=[ + KnowledgeAgentMessage( + role="assistant", + content=[KnowledgeAgentMessageTextContent(text="Test response")], + ) + ], + activity=[ + KnowledgeAgentSearchActivityRecord( + id=1, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="query"), + count=10, + elapsed_ms=50, + ), + ], + references=[ + KnowledgeAgentAzureSearchDocReference( + id="1", + activity_source=1, + doc_key=None, # Missing doc_key + source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, + ), + KnowledgeAgentAzureSearchDocReference( + id="2", + activity_source=1, + doc_key="", # Empty doc_key + source_data={"content": "Content 2", "sourcepage": "page2.pdf"}, + ), + KnowledgeAgentAzureSearchDocReference( + id="3", + activity_source=1, + doc_key="doc3", # Valid doc_key + source_data={"content": "Content 3", "sourcepage": "page3.pdf"}, + ), + ], + ) + + +def mock_retrieval_response_with_top_limit(): + """Mock response with many references to test top limit during document building""" + references = [] + for i in range(15): # More than any reasonable top limit + references.append( + KnowledgeAgentAzureSearchDocReference( + id=str(i), + activity_source=1, + doc_key=f"doc{i}", + source_data={"content": f"Content {i}", "sourcepage": f"page{i}.pdf"}, + ) + ) + + return KnowledgeAgentRetrievalResponse( + response=[ + KnowledgeAgentMessage( + role="assistant", + content=[KnowledgeAgentMessageTextContent(text="Test response")], + ) + ], + activity=[ + KnowledgeAgentSearchActivityRecord( + id=1, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="query"), + count=10, + elapsed_ms=50, + ), + ], + references=references, + ) + + +@pytest.mark.asyncio +async def test_hydrate_agent_references_missing_doc_keys(hydrating_approach, monkeypatch): + """Test that hydrate_agent_references handles missing/empty doc_keys correctly""" + + async def mock_retrieval(*args, **kwargs): + return mock_retrieval_response_with_missing_doc_key() + + # Mock search to return single document for doc3 + async def mock_search_single(*args, **kwargs): + return MockAsyncSearchResultsIterator("hydrated_single", None) + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(SearchClient, "search", mock_search_single) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await hydrating_approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index" + ) + + # Should only get doc3 since doc_key was missing/empty for others + assert len(results) == 1 + assert results[0].id == "doc1" # From mock search result + assert results[0].content == "Hydrated content 1" + + +@pytest.mark.asyncio +async def test_hydrate_agent_references_empty_doc_keys(hydrating_approach, monkeypatch): + """Test that hydrate_agent_references handles case with no valid doc_keys""" + + async def mock_retrieval_no_valid_keys(*args, **kwargs): + return KnowledgeAgentRetrievalResponse( + response=[KnowledgeAgentMessage(role="assistant", content=[])], + activity=[], + references=[ + KnowledgeAgentAzureSearchDocReference( + id="1", + activity_source=1, + doc_key=None, # No valid doc_key + source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, + ), + ] + ) + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_no_valid_keys) + # No need to mock search since it should never be called + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await hydrating_approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index" + ) + + # Should get empty results since no valid doc_keys + assert len(results) == 0 + + +@pytest.mark.asyncio +async def test_hydrate_agent_references_search_returns_empty(hydrating_approach, monkeypatch): + """Test that hydrate_agent_references handles case where search returns no results""" + + async def mock_retrieval_valid_keys(*args, **kwargs): + return KnowledgeAgentRetrievalResponse( + response=[KnowledgeAgentMessage(role="assistant", content=[])], + activity=[], + references=[ + KnowledgeAgentAzureSearchDocReference( + id="1", + activity_source=1, + doc_key="nonexistent_doc", # Valid doc_key but document doesn't exist + source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, + ), + ] + ) + + # Mock search to return empty results (no documents found) + async def mock_search_returns_empty(*args, **kwargs): + return MockAsyncSearchResultsIterator("hydrated_empty", None) + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_valid_keys) + monkeypatch.setattr(SearchClient, "search", mock_search_returns_empty) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await hydrating_approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index" + ) + + # When hydration is enabled but returns empty results, we should get empty list + # rather than falling back to source_data (this is the expected behavior) + assert len(results) == 0 + + +@pytest.mark.asyncio +async def test_agentic_retrieval_with_top_limit_during_building(approach, monkeypatch): + """Test that document building respects top limit and breaks early""" + + async def mock_retrieval(*args, **kwargs): + return mock_retrieval_response_with_top_limit() + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index", + top=5 # Limit to 5 documents + ) + + # Should get exactly 5 documents due to top limit during building + assert len(results) == 5 + for i, result in enumerate(results): + assert result.id == f"doc{i}" + assert result.content == f"Content {i}" + + +@pytest.mark.asyncio +async def test_hydrate_agent_references_with_top_limit_during_collection(hydrating_approach, monkeypatch): + """Test that hydration respects top limit when collecting doc_keys""" + + async def mock_retrieval(*args, **kwargs): + return mock_retrieval_response_with_top_limit() + + # Mock search to return multi results (more than our top limit) + async def mock_search_multi(*args, **kwargs): + return MockAsyncSearchResultsIterator("hydrated_multi", None) + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(SearchClient, "search", mock_search_multi) + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + _, results = await hydrating_approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index", + top=2 # Limit to 2 documents + ) + + # Should get exactly 2 documents due to top limit during doc_keys collection + assert len(results) == 2 + assert results[0].content == "Hydrated content 1" + assert results[1].content == "Hydrated content 2" diff --git a/tests/test_chatapproach.py b/tests/test_chatapproach.py index 70c5ace4c1..e55257b323 100644 --- a/tests/test_chatapproach.py +++ b/tests/test_chatapproach.py @@ -49,6 +49,32 @@ def chat_approach(): ) +@pytest.fixture +def chat_approach_with_hydration(): + return ChatReadRetrieveReadApproach( + search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), + search_index_name=None, + agent_model=None, + agent_deployment=None, + agent_client=None, + auth_helper=None, + openai_client=None, + chatgpt_model="gpt-4.1-mini", + chatgpt_deployment="chat", + embedding_deployment="embeddings", + embedding_model=MOCK_EMBEDDING_MODEL_NAME, + embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, + embedding_field="embedding3", + sourcepage_field="", + content_field="", + query_language="en-us", + query_speller="lexicon", + prompt_manager=PromptyManager(), + hydrate_references=True, + ) + + + def test_get_search_query(chat_approach): payload = """ { @@ -300,3 +326,63 @@ async def test_agent_retrieval_results(monkeypatch): assert results[0].content == "There is a whistleblower policy." assert results[0].sourcepage == "Benefit_Options-2.pdf" assert results[0].search_agent_query == "whistleblower query" + + +@pytest.mark.asyncio +async def test_agentic_retrieval_without_hydration(chat_approach, monkeypatch): + """Test agentic retrieval without hydration""" + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + + _, results = await chat_approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="" + ) + + assert len(results) == 1 + assert results[0].id == "Benefit_Options-2.pdf" + # Content should be from source_data since no hydration + assert results[0].content == "There is a whistleblower policy." + assert results[0].sourcepage == "Benefit_Options-2.pdf" + assert results[0].search_agent_query == "whistleblower query" + # These fields should NOT be present without hydration + assert not hasattr(results[0], 'sourcefile') or results[0].sourcefile is None + assert not hasattr(results[0], 'category') or results[0].category is None + assert not hasattr(results[0], 'score') or results[0].score is None + + +async def mock_search_with_hydration(*args, **kwargs): + """Mock search client that returns data with sourcefile and category for hydration testing""" + return MockAsyncSearchResultsIterator("hydrated", None) + + +@pytest.mark.asyncio +async def test_agentic_retrieval_with_hydration(chat_approach_with_hydration, monkeypatch): + """Test agentic retrieval with hydration enabled""" + + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) + + # Mock the agent retrieval and search client + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(SearchClient, "search", mock_search_with_hydration) + + _, results = await chat_approach_with_hydration.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="" + ) + + assert len(results) == 1 + assert results[0].id == "Benefit_Options-2.pdf" + # Content should be from hydrated search, not source_data + assert results[0].content == "There is a whistleblower policy." + assert results[0].sourcepage == "Benefit_Options-2.pdf" + assert results[0].search_agent_query == "whistleblower query" + # These fields should be present from hydration (from search results) + assert results[0].sourcefile == "Benefit_Options.pdf" + assert results[0].category == "benefits" + assert results[0].score == 0.03279569745063782 + assert results[0].reranker_score == 3.4577205181121826 From 5dc2b4f7bc151d2195a1637a0bee94b6b1417b1c Mon Sep 17 00:00:00 2001 From: Taylor Date: Sun, 24 Aug 2025 06:50:49 +1000 Subject: [PATCH 05/11] Ran ruff and black on new tests --- tests/test_approach_agentic_retrieval.py | 199 ++++++++++------------- 1 file changed, 86 insertions(+), 113 deletions(-) diff --git a/tests/test_approach_agentic_retrieval.py b/tests/test_approach_agentic_retrieval.py index 1f099d557e..5e7e9dbeb2 100644 --- a/tests/test_approach_agentic_retrieval.py +++ b/tests/test_approach_agentic_retrieval.py @@ -5,14 +5,13 @@ KnowledgeAgentAzureSearchDocReference, KnowledgeAgentMessage, KnowledgeAgentMessageTextContent, - KnowledgeAgentModelQueryPlanningActivityRecord, KnowledgeAgentRetrievalResponse, KnowledgeAgentSearchActivityRecord, KnowledgeAgentSearchActivityRecordQuery, ) from azure.search.documents.aio import SearchClient -from approaches.approach import Approach, Document +from approaches.approach import Approach from approaches.promptmanager import PromptyManager from core.authentication import AuthenticationHelper @@ -109,7 +108,7 @@ def mock_retrieval_response_with_sorting(): ), KnowledgeAgentSearchActivityRecord( id=2, - target_index="index", + target_index="index", query=KnowledgeAgentSearchActivityRecordQuery(search="second query"), count=10, elapsed_ms=50, @@ -165,7 +164,7 @@ def mock_retrieval_response_with_duplicates(): source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, ), KnowledgeAgentAzureSearchDocReference( - id="2", + id="2", activity_source=2, doc_key="doc1", # Duplicate doc_key source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, @@ -183,7 +182,7 @@ def mock_retrieval_response_with_duplicates(): async def mock_search_for_hydration(*args, **kwargs): """Mock search that returns documents matching the filter""" filter_param = kwargs.get("filter", "") - + # Create documents based on filter - use search_text to distinguish different calls search_text = "" if "doc1" in filter_param and "doc2" in filter_param: @@ -192,35 +191,35 @@ async def mock_search_for_hydration(*args, **kwargs): search_text = "hydrated_single" else: search_text = "hydrated_empty" - + return MockAsyncSearchResultsIterator(search_text, None) @pytest.mark.asyncio async def test_agentic_retrieval_non_hydrated_default_sort(approach, monkeypatch): """Test non-hydrated path with default sorting (preserve original order)""" - + async def mock_retrieval(*args, **kwargs): return mock_retrieval_response_with_sorting() - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await approach.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index", - results_merge_strategy=None # Default sorting + results_merge_strategy=None, # Default sorting ) - + assert len(results) == 2 # Default sorting preserves original order (doc2, doc1) assert results[0].id == "doc2" assert results[0].content == "Content 2" assert results[0].search_agent_query == "second query" - - assert results[1].id == "doc1" + + assert results[1].id == "doc1" assert results[1].content == "Content 1" assert results[1].search_agent_query == "first query" @@ -228,28 +227,25 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio async def test_agentic_retrieval_non_hydrated_interleaved_sort(approach, monkeypatch): """Test non-hydrated path with interleaved sorting""" - + async def mock_retrieval(*args, **kwargs): return mock_retrieval_response_with_sorting() - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="test-index", - results_merge_strategy="interleaved" + messages=[], agent_client=agent_client, search_index_name="test-index", results_merge_strategy="interleaved" ) - + assert len(results) == 2 # Interleaved sorting orders by reference ID (1, 2) assert results[0].id == "doc1" # ref.id = "1" assert results[0].content == "Content 1" assert results[0].search_agent_query == "first query" - - assert results[1].id == "doc2" # ref.id = "2" + + assert results[1].id == "doc2" # ref.id = "2" assert results[1].content == "Content 2" assert results[1].search_agent_query == "second query" @@ -257,22 +253,19 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio async def test_agentic_retrieval_hydrated_with_sorting(hydrating_approach, monkeypatch): """Test hydrated path with sorting""" - + async def mock_retrieval(*args, **kwargs): return mock_retrieval_response_with_sorting() - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) monkeypatch.setattr(SearchClient, "search", mock_search_for_hydration) - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await hydrating_approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="test-index", - results_merge_strategy="interleaved" + messages=[], agent_client=agent_client, search_index_name="test-index", results_merge_strategy="interleaved" ) - + assert len(results) == 2 # Should have hydrated content, not source_data content assert results[0].content == "Hydrated content 1" @@ -285,21 +278,19 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio async def test_hydrate_agent_references_deduplication(hydrating_approach, monkeypatch): """Test that hydrate_agent_references deduplicates doc_keys""" - + async def mock_retrieval(*args, **kwargs): return mock_retrieval_response_with_duplicates() - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) monkeypatch.setattr(SearchClient, "search", mock_search_for_hydration) - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await hydrating_approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="test-index" + messages=[], agent_client=agent_client, search_index_name="test-index" ) - + # Should only get 2 unique documents despite 3 references (doc1 appears twice) assert len(results) == 2 doc_ids = [doc.id for doc in results] @@ -310,51 +301,45 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio async def test_agentic_retrieval_no_references(approach, monkeypatch): """Test behavior when agent returns no references""" - + async def mock_retrieval(*args, **kwargs): return KnowledgeAgentRetrievalResponse( - response=[KnowledgeAgentMessage(role="assistant", content=[])], - activity=[], - references=[] + response=[KnowledgeAgentMessage(role="assistant", content=[])], activity=[], references=[] ) - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="test-index" + messages=[], agent_client=agent_client, search_index_name="test-index" ) - + assert len(results) == 0 @pytest.mark.asyncio async def test_activity_mapping_injection(approach, monkeypatch): """Test that search_agent_query is properly injected from activity mapping""" - + async def mock_retrieval(*args, **kwargs): return mock_retrieval_response_with_sorting() - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="test-index" + messages=[], agent_client=agent_client, search_index_name="test-index" ) - + # Verify that search_agent_query is correctly mapped from activity assert len(results) == 2 - + # Find each document and verify its query doc1 = next(doc for doc in results if doc.id == "doc1") doc2 = next(doc for doc in results if doc.id == "doc2") - + assert doc1.search_agent_query == "first query" # From activity_source=1 assert doc2.search_agent_query == "second query" # From activity_source=2 @@ -412,7 +397,7 @@ def mock_retrieval_response_with_top_limit(): source_data={"content": f"Content {i}", "sourcepage": f"page{i}.pdf"}, ) ) - + return KnowledgeAgentRetrievalResponse( response=[ KnowledgeAgentMessage( @@ -436,25 +421,23 @@ def mock_retrieval_response_with_top_limit(): @pytest.mark.asyncio async def test_hydrate_agent_references_missing_doc_keys(hydrating_approach, monkeypatch): """Test that hydrate_agent_references handles missing/empty doc_keys correctly""" - + async def mock_retrieval(*args, **kwargs): return mock_retrieval_response_with_missing_doc_key() - + # Mock search to return single document for doc3 async def mock_search_single(*args, **kwargs): return MockAsyncSearchResultsIterator("hydrated_single", None) - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) monkeypatch.setattr(SearchClient, "search", mock_search_single) - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await hydrating_approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="test-index" + messages=[], agent_client=agent_client, search_index_name="test-index" ) - + # Should only get doc3 since doc_key was missing/empty for others assert len(results) == 1 assert results[0].id == "doc1" # From mock search result @@ -464,7 +447,7 @@ async def mock_search_single(*args, **kwargs): @pytest.mark.asyncio async def test_hydrate_agent_references_empty_doc_keys(hydrating_approach, monkeypatch): """Test that hydrate_agent_references handles case with no valid doc_keys""" - + async def mock_retrieval_no_valid_keys(*args, **kwargs): return KnowledgeAgentRetrievalResponse( response=[KnowledgeAgentMessage(role="assistant", content=[])], @@ -476,57 +459,53 @@ async def mock_retrieval_no_valid_keys(*args, **kwargs): doc_key=None, # No valid doc_key source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, ), - ] + ], ) - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_no_valid_keys) # No need to mock search since it should never be called - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await hydrating_approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="test-index" + messages=[], agent_client=agent_client, search_index_name="test-index" ) - + # Should get empty results since no valid doc_keys assert len(results) == 0 -@pytest.mark.asyncio +@pytest.mark.asyncio async def test_hydrate_agent_references_search_returns_empty(hydrating_approach, monkeypatch): """Test that hydrate_agent_references handles case where search returns no results""" - + async def mock_retrieval_valid_keys(*args, **kwargs): return KnowledgeAgentRetrievalResponse( response=[KnowledgeAgentMessage(role="assistant", content=[])], activity=[], references=[ KnowledgeAgentAzureSearchDocReference( - id="1", + id="1", activity_source=1, doc_key="nonexistent_doc", # Valid doc_key but document doesn't exist source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, ), - ] + ], ) - + # Mock search to return empty results (no documents found) async def mock_search_returns_empty(*args, **kwargs): return MockAsyncSearchResultsIterator("hydrated_empty", None) - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_valid_keys) monkeypatch.setattr(SearchClient, "search", mock_search_returns_empty) - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await hydrating_approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="test-index" + messages=[], agent_client=agent_client, search_index_name="test-index" ) - + # When hydration is enabled but returns empty results, we should get empty list # rather than falling back to source_data (this is the expected behavior) assert len(results) == 0 @@ -535,21 +514,18 @@ async def mock_search_returns_empty(*args, **kwargs): @pytest.mark.asyncio async def test_agentic_retrieval_with_top_limit_during_building(approach, monkeypatch): """Test that document building respects top limit and breaks early""" - + async def mock_retrieval(*args, **kwargs): return mock_retrieval_response_with_top_limit() - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="test-index", - top=5 # Limit to 5 documents + messages=[], agent_client=agent_client, search_index_name="test-index", top=5 # Limit to 5 documents ) - + # Should get exactly 5 documents due to top limit during building assert len(results) == 5 for i, result in enumerate(results): @@ -560,26 +536,23 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio async def test_hydrate_agent_references_with_top_limit_during_collection(hydrating_approach, monkeypatch): """Test that hydration respects top limit when collecting doc_keys""" - + async def mock_retrieval(*args, **kwargs): return mock_retrieval_response_with_top_limit() - + # Mock search to return multi results (more than our top limit) async def mock_search_multi(*args, **kwargs): return MockAsyncSearchResultsIterator("hydrated_multi", None) - + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) monkeypatch.setattr(SearchClient, "search", mock_search_multi) - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - + _, results = await hydrating_approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="test-index", - top=2 # Limit to 2 documents + messages=[], agent_client=agent_client, search_index_name="test-index", top=2 # Limit to 2 documents ) - + # Should get exactly 2 documents due to top limit during doc_keys collection assert len(results) == 2 assert results[0].content == "Hydrated content 1" From 422e5c3495304533793780604cf03d5b1d1747dd Mon Sep 17 00:00:00 2001 From: Taylor Date: Mon, 25 Aug 2025 07:31:17 +1000 Subject: [PATCH 06/11] Ran black on changed files --- tests/conftest.py | 2 +- tests/mocks.py | 2 +- tests/test_chatapproach.py | 21 +++++++-------------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fe93991058..4c0175ec1b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -379,7 +379,7 @@ def mock_blob_container_client(monkeypatch): "AZURE_OPENAI_SEARCHAGENT_MODEL": "gpt-4.1-mini", "AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT": "gpt-4.1-mini", "USE_AGENTIC_RETRIEVAL": "true", - "ENABLE_AGENTIC_REF_HYDRATION": "true" + "ENABLE_AGENTIC_REF_HYDRATION": "true", } ] diff --git a/tests/mocks.py b/tests/mocks.py index 5b8d449a21..f440db67c2 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -160,7 +160,7 @@ def __init__(self, search_text, vector_queries: Optional[list[VectorQuery]]): "@search.captions": [], }, { - "id": "doc2", + "id": "doc2", "content": "Hydrated content 2", "sourcepage": "page2.pdf", "sourcefile": "file2.pdf", diff --git a/tests/test_chatapproach.py b/tests/test_chatapproach.py index e55257b323..d53e89456b 100644 --- a/tests/test_chatapproach.py +++ b/tests/test_chatapproach.py @@ -74,7 +74,6 @@ def chat_approach_with_hydration(): ) - def test_get_search_query(chat_approach): payload = """ { @@ -331,16 +330,12 @@ async def test_agent_retrieval_results(monkeypatch): @pytest.mark.asyncio async def test_agentic_retrieval_without_hydration(chat_approach, monkeypatch): """Test agentic retrieval without hydration""" - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - _, results = await chat_approach.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="" - ) + _, results = await chat_approach.run_agentic_retrieval(messages=[], agent_client=agent_client, search_index_name="") assert len(results) == 1 assert results[0].id == "Benefit_Options-2.pdf" @@ -349,9 +344,9 @@ async def test_agentic_retrieval_without_hydration(chat_approach, monkeypatch): assert results[0].sourcepage == "Benefit_Options-2.pdf" assert results[0].search_agent_query == "whistleblower query" # These fields should NOT be present without hydration - assert not hasattr(results[0], 'sourcefile') or results[0].sourcefile is None - assert not hasattr(results[0], 'category') or results[0].category is None - assert not hasattr(results[0], 'score') or results[0].score is None + assert not hasattr(results[0], "sourcefile") or results[0].sourcefile is None + assert not hasattr(results[0], "category") or results[0].category is None + assert not hasattr(results[0], "score") or results[0].score is None async def mock_search_with_hydration(*args, **kwargs): @@ -362,7 +357,7 @@ async def mock_search_with_hydration(*args, **kwargs): @pytest.mark.asyncio async def test_agentic_retrieval_with_hydration(chat_approach_with_hydration, monkeypatch): """Test agentic retrieval with hydration enabled""" - + agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) # Mock the agent retrieval and search client @@ -370,9 +365,7 @@ async def test_agentic_retrieval_with_hydration(chat_approach_with_hydration, mo monkeypatch.setattr(SearchClient, "search", mock_search_with_hydration) _, results = await chat_approach_with_hydration.run_agentic_retrieval( - messages=[], - agent_client=agent_client, - search_index_name="" + messages=[], agent_client=agent_client, search_index_name="" ) assert len(results) == 1 From 8faa4191715fd9b65abb3730d375f4d6663d41e6 Mon Sep 17 00:00:00 2001 From: Taylor Date: Tue, 26 Aug 2025 06:28:25 +1000 Subject: [PATCH 07/11] Update test snapshots --- .../agent_client0/result.json | 18 ++++++++++++------ .../agent_auth_client0/result.json | 18 ++++++++++++------ .../agent_client0/result.json | 18 ++++++++++++------ .../agent_auth_client0/result.json | 18 ++++++++++++------ 4 files changed, 48 insertions(+), 24 deletions(-) diff --git a/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json index 86019f5e31..65174dfc73 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json @@ -26,16 +26,22 @@ { "description": [ { - "captions": [], + "captions": [ + { + "additional_properties": {}, + "highlights": [], + "text": "Caption: A whistleblower policy." + } + ], "category": null, "content": "There is a whistleblower policy.", "groups": null, - "id": "Benefit_Options-2.pdf", + "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "oids": null, - "reranker_score": null, - "score": null, - "search_agent_query": "whistleblower query", - "sourcefile": null, + "reranker_score": 3.4577205181121826, + "score": 0.03279569745063782, + "search_agent_query": null, + "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } ], diff --git a/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json index d997437fed..d3945c8ae2 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json @@ -26,16 +26,22 @@ { "description": [ { - "captions": [], + "captions": [ + { + "additional_properties": {}, + "highlights": [], + "text": "Caption: A whistleblower policy." + } + ], "category": null, "content": "There is a whistleblower policy.", "groups": null, - "id": "Benefit_Options-2.pdf", + "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "oids": null, - "reranker_score": null, - "score": null, - "search_agent_query": "whistleblower query", - "sourcefile": null, + "reranker_score": 3.4577205181121826, + "score": 0.03279569745063782, + "search_agent_query": null, + "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } ], diff --git a/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json b/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json index c249658ac0..5851ac66fc 100644 --- a/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json +++ b/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json @@ -26,16 +26,22 @@ { "description": [ { - "captions": [], + "captions": [ + { + "additional_properties": {}, + "highlights": [], + "text": "Caption: A whistleblower policy." + } + ], "category": null, "content": "There is a whistleblower policy.", "groups": null, - "id": "Benefit_Options-2.pdf", + "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "oids": null, - "reranker_score": null, - "score": null, - "search_agent_query": "whistleblower query", - "sourcefile": null, + "reranker_score": 3.4577205181121826, + "score": 0.03279569745063782, + "search_agent_query": null, + "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } ], diff --git a/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json b/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json index a14482013a..1f5ab672cb 100644 --- a/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json +++ b/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json @@ -26,16 +26,22 @@ { "description": [ { - "captions": [], + "captions": [ + { + "additional_properties": {}, + "highlights": [], + "text": "Caption: A whistleblower policy." + } + ], "category": null, "content": "There is a whistleblower policy.", "groups": null, - "id": "Benefit_Options-2.pdf", + "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "oids": null, - "reranker_score": null, - "score": null, - "search_agent_query": "whistleblower query", - "sourcefile": null, + "reranker_score": 3.4577205181121826, + "score": 0.03279569745063782, + "search_agent_query": null, + "sourcefile": "Benefit_Options.pdf", "sourcepage": "Benefit_Options-2.pdf" } ], From b0a20d85ad43860dd3e2073082ccbb3bb2b8dd95 Mon Sep 17 00:00:00 2001 From: Taylor Date: Mon, 1 Sep 2025 10:58:22 +1000 Subject: [PATCH 08/11] Working on tests --- tests/conftest.py | 38 ++- tests/mocks.py | 183 +++++++++++++- .../agent_client0/result.json | 2 +- .../agent_auth_client0/result.json | 2 +- .../agent_client0/result.json | 2 +- .../agent_auth_client0/result.json | 2 +- ...retrieval.py => test_agentic_retrieval.py} | 234 +++--------------- tests/test_chatapproach.py | 186 +------------- 8 files changed, 257 insertions(+), 392 deletions(-) rename tests/{test_approach_agentic_retrieval.py => test_agentic_retrieval.py} (63%) diff --git a/tests/conftest.py b/tests/conftest.py index ca8d5e9a84..34be10826d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ import msal import pytest import pytest_asyncio +from azure.core.credentials import AzureKeyCredential from azure.search.documents.agent.aio import KnowledgeAgentRetrievalClient from azure.search.documents.aio import SearchClient from azure.search.documents.indexes.aio import SearchIndexClient @@ -1102,7 +1103,7 @@ def mock_user_directory_client(monkeypatch): @pytest.fixture def chat_approach(): return ChatReadRetrieveReadApproach( - search_client=None, + search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), search_index_name=None, agent_model=None, agent_deployment=None, @@ -1131,3 +1132,38 @@ def chat_approach(): credential=MockAzureCredential(), ), ) + + +@pytest.fixture +def chat_approach_with_hydration(): + return ChatReadRetrieveReadApproach( + search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), + search_index_name=None, + agent_model=None, + agent_deployment=None, + agent_client=None, + auth_helper=None, + openai_client=None, + chatgpt_model="gpt-4.1-mini", + chatgpt_deployment="chat", + embedding_deployment="embeddings", + embedding_model=MOCK_EMBEDDING_MODEL_NAME, + embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, + embedding_field="embedding3", + sourcepage_field="", + content_field="", + query_language="en-us", + query_speller="lexicon", + prompt_manager=PromptyManager(), + hydrate_references=True, + user_blob_manager=AdlsBlobManager( + endpoint="https://test-userstorage-account.dfs.core.windows.net", + container="test-userstorage-container", + credential=MockAzureCredential(), + ), + global_blob_manager=BlobManager( # on normal Azure storage + endpoint="https://test-globalstorage-account.blob.core.windows.net", + container="test-globalstorage-container", + credential=MockAzureCredential(), + ), + ) diff --git a/tests/mocks.py b/tests/mocks.py index 728ea01a3f..16bfd31120 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -269,7 +269,6 @@ def __init__(self, search_text, vector_queries: Optional[list[VectorQuery]]): ] ] elif search_text == "hydrated": - # Mock search results for hydration testing with complete data self.data = [ [ { @@ -287,7 +286,6 @@ def __init__(self, search_text, vector_queries: Optional[list[VectorQuery]]): ] ] elif search_text == "hydrated_multi": - # Mock search results for multiple document hydration self.data = [ [ { @@ -315,7 +313,6 @@ def __init__(self, search_text, vector_queries: Optional[list[VectorQuery]]): ] ] elif search_text == "hydrated_single": - # Mock search results for single document hydration self.data = [ [ { @@ -458,6 +455,186 @@ def mock_retrieval_response(): ) +def mock_retrieval_response_with_sorting(): + """Mock response with multiple references for testing sorting""" + return KnowledgeAgentRetrievalResponse( + response=[ + KnowledgeAgentMessage( + role="assistant", + content=[KnowledgeAgentMessageTextContent(text="Test response")], + ) + ], + activity=[ + KnowledgeAgentSearchActivityRecord( + id=1, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="first query"), + count=10, + elapsed_ms=50, + ), + KnowledgeAgentSearchActivityRecord( + id=2, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="second query"), + count=10, + elapsed_ms=50, + ), + ], + references=[ + KnowledgeAgentAzureSearchDocReference( + id="2", # Higher ID for testing interleaved sorting + activity_source=2, + doc_key="doc2", + source_data={"content": "Content 2", "sourcepage": "page2.pdf"}, + ), + KnowledgeAgentAzureSearchDocReference( + id="1", # Lower ID for testing interleaved sorting + activity_source=1, + doc_key="doc1", + source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, + ), + ], + ) + + +def mock_retrieval_response_with_duplicates(): + """Mock response with duplicate doc_keys for testing deduplication""" + return KnowledgeAgentRetrievalResponse( + response=[ + KnowledgeAgentMessage( + role="assistant", + content=[KnowledgeAgentMessageTextContent(text="Test response")], + ) + ], + activity=[ + KnowledgeAgentSearchActivityRecord( + id=1, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="query for doc1"), + count=10, + elapsed_ms=50, + ), + KnowledgeAgentSearchActivityRecord( + id=2, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="another query for doc1"), + count=10, + elapsed_ms=50, + ), + ], + references=[ + KnowledgeAgentAzureSearchDocReference( + id="1", + activity_source=1, + doc_key="doc1", # Same doc_key + source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, + ), + KnowledgeAgentAzureSearchDocReference( + id="2", + activity_source=2, + doc_key="doc1", # Duplicate doc_key + source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, + ), + KnowledgeAgentAzureSearchDocReference( + id="3", + activity_source=1, + doc_key="doc2", # Different doc_key + source_data={"content": "Content 2", "sourcepage": "page2.pdf"}, + ), + ], + ) + + +async def mock_search_for_hydration(*args, **kwargs): + """Mock search that returns documents matching the filter""" + filter_param = kwargs.get("filter", "") + + # Create documents based on filter - use search_text to distinguish different calls + search_text = "" + if "doc1" in filter_param and "doc2" in filter_param: + search_text = "hydrated_multi" + elif "doc1" in filter_param: + search_text = "hydrated_single" + else: + search_text = "hydrated_empty" + + return MockAsyncSearchResultsIterator(search_text, None) + + +def mock_retrieval_response_with_missing_doc_key(): + """Mock response with missing doc_key to test continue condition""" + return KnowledgeAgentRetrievalResponse( + response=[ + KnowledgeAgentMessage( + role="assistant", + content=[KnowledgeAgentMessageTextContent(text="Test response")], + ) + ], + activity=[ + KnowledgeAgentSearchActivityRecord( + id=1, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="query"), + count=10, + elapsed_ms=50, + ), + ], + references=[ + KnowledgeAgentAzureSearchDocReference( + id="1", + activity_source=1, + doc_key=None, # Missing doc_key + source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, + ), + KnowledgeAgentAzureSearchDocReference( + id="2", + activity_source=1, + doc_key="", # Empty doc_key + source_data={"content": "Content 2", "sourcepage": "page2.pdf"}, + ), + KnowledgeAgentAzureSearchDocReference( + id="3", + activity_source=1, + doc_key="doc3", # Valid doc_key + source_data={"content": "Content 3", "sourcepage": "page3.pdf"}, + ), + ], + ) + + +def mock_retrieval_response_with_top_limit(): + """Mock response with many references to test top limit during document building""" + references = [] + for i in range(15): # More than any reasonable top limit + references.append( + KnowledgeAgentAzureSearchDocReference( + id=str(i), + activity_source=1, + doc_key=f"doc{i}", + source_data={"content": f"Content {i}", "sourcepage": f"page{i}.pdf"}, + ) + ) + + return KnowledgeAgentRetrievalResponse( + response=[ + KnowledgeAgentMessage( + role="assistant", + content=[KnowledgeAgentMessageTextContent(text="Test response")], + ) + ], + activity=[ + KnowledgeAgentSearchActivityRecord( + id=1, + target_index="index", + query=KnowledgeAgentSearchActivityRecordQuery(search="query"), + count=10, + elapsed_ms=50, + ), + ], + references=references, + ) + + class MockAudio: def __init__(self, audio_data): self.audio_data = audio_data diff --git a/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json index c9586d028b..1fd69cb588 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text_agent/agent_client0/result.json @@ -38,7 +38,7 @@ "category": null, "content": "There is a whistleblower policy.", "groups": null, - "id": "Benefit_Options-2.pdf", + "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "images": null, "oids": null, "reranker_score": 3.4577205181121826, diff --git a/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json b/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json index 9e735a1b7c..3193a65e9d 100644 --- a/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json +++ b/tests/snapshots/test_app/test_ask_rtr_text_agent_filter/agent_auth_client0/result.json @@ -38,7 +38,7 @@ "category": null, "content": "There is a whistleblower policy.", "groups": null, - "id": "Benefit_Options-2.pdf", + "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "images": null, "oids": null, "reranker_score": 3.4577205181121826, diff --git a/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json b/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json index 8d1cfa43eb..cbd6e457c1 100644 --- a/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json +++ b/tests/snapshots/test_app/test_chat_text_agent/agent_client0/result.json @@ -39,7 +39,7 @@ "category": null, "content": "There is a whistleblower policy.", "groups": null, - "id": "Benefit_Options-2.pdf", + "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "images": null, "oids": null, "reranker_score": 3.4577205181121826, diff --git a/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json b/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json index ffef4524d6..448297c32e 100644 --- a/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json +++ b/tests/snapshots/test_app/test_chat_text_filter_agent/agent_auth_client0/result.json @@ -39,7 +39,7 @@ "category": null, "content": "There is a whistleblower policy.", "groups": null, - "id": "Benefit_Options-2.pdf", + "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "images": null, "oids": null, "reranker_score": 3.4577205181121826, diff --git a/tests/test_approach_agentic_retrieval.py b/tests/test_agentic_retrieval.py similarity index 63% rename from tests/test_approach_agentic_retrieval.py rename to tests/test_agentic_retrieval.py index 5e7e9dbeb2..6dcfe92a25 100644 --- a/tests/test_approach_agentic_retrieval.py +++ b/tests/test_agentic_retrieval.py @@ -11,179 +11,23 @@ ) from azure.search.documents.aio import SearchClient -from approaches.approach import Approach -from approaches.promptmanager import PromptyManager -from core.authentication import AuthenticationHelper - from .mocks import ( - MOCK_EMBEDDING_DIMENSIONS, - MOCK_EMBEDDING_MODEL_NAME, MockAsyncSearchResultsIterator, - MockAzureCredential, + mock_retrieval_response, + mock_retrieval_response_with_duplicates, + mock_retrieval_response_with_sorting, ) -class MockApproach(Approach): - """Concrete implementation of abstract Approach for testing""" - - async def run(self, messages, session_state=None, context={}): - pass - - async def run_stream(self, messages, session_state=None, context={}): - pass - - -@pytest.fixture -def approach(): - """Create a test approach instance""" - return MockApproach( - search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), - openai_client=None, - auth_helper=AuthenticationHelper( - search_index=None, - use_authentication=False, - server_app_id=None, - server_app_secret=None, - client_app_id=None, - tenant_id=None, - ), - query_language="en-us", - query_speller="lexicon", - embedding_deployment="embeddings", - embedding_model=MOCK_EMBEDDING_MODEL_NAME, - embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, - embedding_field="embedding", - openai_host="", - vision_endpoint="", - vision_token_provider=lambda: MockAzureCredential().get_token(""), - prompt_manager=PromptyManager(), - hydrate_references=False, - ) - - -@pytest.fixture -def hydrating_approach(): - """Create a test approach instance with hydration enabled""" - return MockApproach( - search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), - openai_client=None, - auth_helper=AuthenticationHelper( - search_index=None, - use_authentication=False, - server_app_id=None, - server_app_secret=None, - client_app_id=None, - tenant_id=None, - ), - query_language="en-us", - query_speller="lexicon", - embedding_deployment="embeddings", - embedding_model=MOCK_EMBEDDING_MODEL_NAME, - embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, - embedding_field="embedding", - openai_host="", - vision_endpoint="", - vision_token_provider=lambda: MockAzureCredential().get_token(""), - prompt_manager=PromptyManager(), - hydrate_references=True, - ) -def mock_retrieval_response_with_sorting(): - """Mock response with multiple references for testing sorting""" - return KnowledgeAgentRetrievalResponse( - response=[ - KnowledgeAgentMessage( - role="assistant", - content=[KnowledgeAgentMessageTextContent(text="Test response")], - ) - ], - activity=[ - KnowledgeAgentSearchActivityRecord( - id=1, - target_index="index", - query=KnowledgeAgentSearchActivityRecordQuery(search="first query"), - count=10, - elapsed_ms=50, - ), - KnowledgeAgentSearchActivityRecord( - id=2, - target_index="index", - query=KnowledgeAgentSearchActivityRecordQuery(search="second query"), - count=10, - elapsed_ms=50, - ), - ], - references=[ - KnowledgeAgentAzureSearchDocReference( - id="2", # Higher ID for testing interleaved sorting - activity_source=2, - doc_key="doc2", - source_data={"content": "Content 2", "sourcepage": "page2.pdf"}, - ), - KnowledgeAgentAzureSearchDocReference( - id="1", # Lower ID for testing interleaved sorting - activity_source=1, - doc_key="doc1", - source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, - ), - ], - ) - - -def mock_retrieval_response_with_duplicates(): - """Mock response with duplicate doc_keys for testing deduplication""" - return KnowledgeAgentRetrievalResponse( - response=[ - KnowledgeAgentMessage( - role="assistant", - content=[KnowledgeAgentMessageTextContent(text="Test response")], - ) - ], - activity=[ - KnowledgeAgentSearchActivityRecord( - id=1, - target_index="index", - query=KnowledgeAgentSearchActivityRecordQuery(search="query for doc1"), - count=10, - elapsed_ms=50, - ), - KnowledgeAgentSearchActivityRecord( - id=2, - target_index="index", - query=KnowledgeAgentSearchActivityRecordQuery(search="another query for doc1"), - count=10, - elapsed_ms=50, - ), - ], - references=[ - KnowledgeAgentAzureSearchDocReference( - id="1", - activity_source=1, - doc_key="doc1", # Same doc_key - source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, - ), - KnowledgeAgentAzureSearchDocReference( - id="2", - activity_source=2, - doc_key="doc1", # Duplicate doc_key - source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, - ), - KnowledgeAgentAzureSearchDocReference( - id="3", - activity_source=1, - doc_key="doc2", # Different doc_key - source_data={"content": "Content 2", "sourcepage": "page2.pdf"}, - ), - ], - ) +async def mock_search(*args, **kwargs): + return MockAsyncSearchResultsIterator(kwargs.get("search_text"), kwargs.get("vector_queries")) async def mock_search_for_hydration(*args, **kwargs): - """Mock search that returns documents matching the filter""" filter_param = kwargs.get("filter", "") - # Create documents based on filter - use search_text to distinguish different calls search_text = "" if "doc1" in filter_param and "doc2" in filter_param: search_text = "hydrated_multi" @@ -192,21 +36,20 @@ async def mock_search_for_hydration(*args, **kwargs): else: search_text = "hydrated_empty" - return MockAsyncSearchResultsIterator(search_text, None) + kwargs["search_text"] = search_text + + return mock_search(*args, **kwargs) @pytest.mark.asyncio -async def test_agentic_retrieval_non_hydrated_default_sort(approach, monkeypatch): +async def test_agentic_retrieval_non_hydrated_default_sort(chat_approach, monkeypatch): """Test non-hydrated path with default sorting (preserve original order)""" - async def mock_retrieval(*args, **kwargs): - return mock_retrieval_response_with_sorting() - - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_response_with_sorting) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await approach.run_agentic_retrieval( + _, results = await chat_approach.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index", @@ -225,17 +68,14 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio -async def test_agentic_retrieval_non_hydrated_interleaved_sort(approach, monkeypatch): +async def test_agentic_retrieval_non_hydrated_interleaved_sort(chat_approach, monkeypatch): """Test non-hydrated path with interleaved sorting""" - async def mock_retrieval(*args, **kwargs): - return mock_retrieval_response_with_sorting() - - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_response_with_sorting) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await approach.run_agentic_retrieval( + _, results = await chat_approach.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index", results_merge_strategy="interleaved" ) @@ -251,18 +91,15 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio -async def test_agentic_retrieval_hydrated_with_sorting(hydrating_approach, monkeypatch): +async def test_agentic_retrieval_hydrated_with_sorting(chat_approach_with_hydration, monkeypatch): """Test hydrated path with sorting""" - async def mock_retrieval(*args, **kwargs): - return mock_retrieval_response_with_sorting() - - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_response_with_sorting) monkeypatch.setattr(SearchClient, "search", mock_search_for_hydration) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await hydrating_approach.run_agentic_retrieval( + _, results = await chat_approach_with_hydration.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index", results_merge_strategy="interleaved" ) @@ -276,18 +113,15 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio -async def test_hydrate_agent_references_deduplication(hydrating_approach, monkeypatch): +async def test_hydrate_agent_references_deduplication(chat_approach_with_hydration, monkeypatch): """Test that hydrate_agent_references deduplicates doc_keys""" - async def mock_retrieval(*args, **kwargs): - return mock_retrieval_response_with_duplicates() - - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_response_with_duplicates) monkeypatch.setattr(SearchClient, "search", mock_search_for_hydration) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await hydrating_approach.run_agentic_retrieval( + _, results = await chat_approach_with_hydration.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index" ) @@ -299,7 +133,7 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio -async def test_agentic_retrieval_no_references(approach, monkeypatch): +async def test_agentic_retrieval_no_references(chat_approach, monkeypatch): """Test behavior when agent returns no references""" async def mock_retrieval(*args, **kwargs): @@ -311,7 +145,7 @@ async def mock_retrieval(*args, **kwargs): agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await approach.run_agentic_retrieval( + _, results = await chat_approach.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index" ) @@ -319,7 +153,7 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio -async def test_activity_mapping_injection(approach, monkeypatch): +async def test_activity_mapping_injection(chat_approach, monkeypatch): """Test that search_agent_query is properly injected from activity mapping""" async def mock_retrieval(*args, **kwargs): @@ -329,7 +163,7 @@ async def mock_retrieval(*args, **kwargs): agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await approach.run_agentic_retrieval( + _, results = await chat_approach.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index" ) @@ -419,7 +253,7 @@ def mock_retrieval_response_with_top_limit(): @pytest.mark.asyncio -async def test_hydrate_agent_references_missing_doc_keys(hydrating_approach, monkeypatch): +async def test_hydrate_agent_references_missing_doc_keys(chat_approach_with_hydration, monkeypatch): """Test that hydrate_agent_references handles missing/empty doc_keys correctly""" async def mock_retrieval(*args, **kwargs): @@ -434,7 +268,7 @@ async def mock_search_single(*args, **kwargs): agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await hydrating_approach.run_agentic_retrieval( + _, results = await chat_approach_with_hydration.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index" ) @@ -445,7 +279,7 @@ async def mock_search_single(*args, **kwargs): @pytest.mark.asyncio -async def test_hydrate_agent_references_empty_doc_keys(hydrating_approach, monkeypatch): +async def test_hydrate_agent_references_empty_doc_keys(chat_approach_with_hydration, monkeypatch): """Test that hydrate_agent_references handles case with no valid doc_keys""" async def mock_retrieval_no_valid_keys(*args, **kwargs): @@ -467,7 +301,7 @@ async def mock_retrieval_no_valid_keys(*args, **kwargs): agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await hydrating_approach.run_agentic_retrieval( + _, results = await chat_approach_with_hydration.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index" ) @@ -476,7 +310,7 @@ async def mock_retrieval_no_valid_keys(*args, **kwargs): @pytest.mark.asyncio -async def test_hydrate_agent_references_search_returns_empty(hydrating_approach, monkeypatch): +async def test_hydrate_agent_references_search_returns_empty(chat_approach_with_hydration, monkeypatch): """Test that hydrate_agent_references handles case where search returns no results""" async def mock_retrieval_valid_keys(*args, **kwargs): @@ -502,7 +336,7 @@ async def mock_search_returns_empty(*args, **kwargs): agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await hydrating_approach.run_agentic_retrieval( + _, results = await chat_approach_with_hydration.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index" ) @@ -512,7 +346,7 @@ async def mock_search_returns_empty(*args, **kwargs): @pytest.mark.asyncio -async def test_agentic_retrieval_with_top_limit_during_building(approach, monkeypatch): +async def test_agentic_retrieval_with_top_limit_during_building(chat_approach_with_hydration, monkeypatch): """Test that document building respects top limit and breaks early""" async def mock_retrieval(*args, **kwargs): @@ -522,7 +356,7 @@ async def mock_retrieval(*args, **kwargs): agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await approach.run_agentic_retrieval( + _, results = await chat_approach_with_hydration.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index", top=5 # Limit to 5 documents ) @@ -534,7 +368,7 @@ async def mock_retrieval(*args, **kwargs): @pytest.mark.asyncio -async def test_hydrate_agent_references_with_top_limit_during_collection(hydrating_approach, monkeypatch): +async def test_hydrate_agent_references_with_top_limit_during_collection(chat_approach_with_hydration, monkeypatch): """Test that hydration respects top limit when collecting doc_keys""" async def mock_retrieval(*args, **kwargs): @@ -549,7 +383,7 @@ async def mock_search_multi(*args, **kwargs): agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await hydrating_approach.run_agentic_retrieval( + _, results = await chat_approach_with_hydration.run_agentic_retrieval( messages=[], agent_client=agent_client, search_index_name="test-index", top=2 # Limit to 2 documents ) diff --git a/tests/test_chatapproach.py b/tests/test_chatapproach.py index cb09b04ec9..aa6145f273 100644 --- a/tests/test_chatapproach.py +++ b/tests/test_chatapproach.py @@ -2,7 +2,6 @@ import pytest from azure.core.credentials import AzureKeyCredential -from azure.search.documents.agent.aio import KnowledgeAgentRetrievalClient from azure.search.documents.aio import SearchClient from azure.search.documents.models import VectorizedQuery from openai.types.chat import ChatCompletion @@ -27,55 +26,6 @@ async def mock_retrieval(*args, **kwargs): return mock_retrieval_response() -@pytest.fixture -def chat_approach(): - return ChatReadRetrieveReadApproach( - search_client=None, - search_index_name=None, - agent_model=None, - agent_deployment=None, - agent_client=None, - auth_helper=None, - openai_client=None, - chatgpt_model="gpt-4.1-mini", - chatgpt_deployment="chat", - embedding_deployment="embeddings", - embedding_model=MOCK_EMBEDDING_MODEL_NAME, - embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, - embedding_field="embedding3", - sourcepage_field="", - content_field="", - query_language="en-us", - query_speller="lexicon", - prompt_manager=PromptyManager(), - ) - - -@pytest.fixture -def chat_approach_with_hydration(): - return ChatReadRetrieveReadApproach( - search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), - search_index_name=None, - agent_model=None, - agent_deployment=None, - agent_client=None, - auth_helper=None, - openai_client=None, - chatgpt_model="gpt-4.1-mini", - chatgpt_deployment="chat", - embedding_deployment="embeddings", - embedding_model=MOCK_EMBEDDING_MODEL_NAME, - embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, - embedding_field="embedding3", - sourcepage_field="", - content_field="", - query_language="en-us", - query_speller="lexicon", - prompt_manager=PromptyManager(), - hydrate_references=True, - ) - - def test_get_search_query(chat_approach): payload = """ { @@ -202,30 +152,8 @@ def test_extract_followup_questions_no_pre_content(chat_approach): ], ) async def test_search_results_filtering_by_scores( - monkeypatch, minimum_search_score, minimum_reranker_score, expected_result_count + chat_approach, monkeypatch, minimum_search_score, minimum_reranker_score, expected_result_count ): - - chat_approach = ChatReadRetrieveReadApproach( - search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), - search_index_name=None, - agent_model=None, - agent_deployment=None, - agent_client=None, - auth_helper=None, - openai_client=None, - chatgpt_model="gpt-4.1-mini", - chatgpt_deployment="chat", - embedding_deployment="embeddings", - embedding_model=MOCK_EMBEDDING_MODEL_NAME, - embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, - embedding_field="embedding3", - sourcepage_field="", - content_field="", - query_language="en-us", - query_speller="lexicon", - prompt_manager=PromptyManager(), - ) - monkeypatch.setattr(SearchClient, "search", mock_search) filtered_results = await chat_approach.search( @@ -247,27 +175,7 @@ async def test_search_results_filtering_by_scores( @pytest.mark.asyncio -async def test_search_results_query_rewriting(monkeypatch): - chat_approach = ChatReadRetrieveReadApproach( - search_client=SearchClient(endpoint="", index_name="", credential=AzureKeyCredential("")), - search_index_name=None, - agent_model=None, - agent_deployment=None, - agent_client=None, - auth_helper=None, - openai_client=None, - chatgpt_model="gpt-35-turbo", - chatgpt_deployment="chat", - embedding_deployment="embeddings", - embedding_model=MOCK_EMBEDDING_MODEL_NAME, - embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, - embedding_field="embedding3", - sourcepage_field="", - content_field="", - query_language="en-us", - query_speller="lexicon", - prompt_manager=PromptyManager(), - ) +async def test_search_results_query_rewriting(chat_approach, monkeypatch): query_rewrites = None @@ -293,96 +201,6 @@ async def validate_qr_and_mock_search(*args, **kwargs): assert query_rewrites == "generative" -@pytest.mark.asyncio -async def test_agent_retrieval_results(monkeypatch): - chat_approach = ChatReadRetrieveReadApproach( - search_client=None, - search_index_name=None, - agent_model=None, - agent_deployment=None, - agent_client=None, - auth_helper=None, - openai_client=None, - chatgpt_model="gpt-35-turbo", - chatgpt_deployment="chat", - embedding_deployment="embeddings", - embedding_model=MOCK_EMBEDDING_MODEL_NAME, - embedding_dimensions=MOCK_EMBEDDING_DIMENSIONS, - embedding_field="embedding3", - sourcepage_field="", - content_field="", - query_language="en-us", - query_speller="lexicon", - prompt_manager=PromptyManager(), - ) - - agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - - _, results = await chat_approach.run_agentic_retrieval(messages=[], agent_client=agent_client, search_index_name="") - - assert len(results) == 1 - assert results[0].id == "Benefit_Options-2.pdf" - assert results[0].content == "There is a whistleblower policy." - assert results[0].sourcepage == "Benefit_Options-2.pdf" - assert results[0].search_agent_query == "whistleblower query" - - -@pytest.mark.asyncio -async def test_agentic_retrieval_without_hydration(chat_approach, monkeypatch): - """Test agentic retrieval without hydration""" - - agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - - _, results = await chat_approach.run_agentic_retrieval(messages=[], agent_client=agent_client, search_index_name="") - - assert len(results) == 1 - assert results[0].id == "Benefit_Options-2.pdf" - # Content should be from source_data since no hydration - assert results[0].content == "There is a whistleblower policy." - assert results[0].sourcepage == "Benefit_Options-2.pdf" - assert results[0].search_agent_query == "whistleblower query" - # These fields should NOT be present without hydration - assert not hasattr(results[0], "sourcefile") or results[0].sourcefile is None - assert not hasattr(results[0], "category") or results[0].category is None - assert not hasattr(results[0], "score") or results[0].score is None - - -async def mock_search_with_hydration(*args, **kwargs): - """Mock search client that returns data with sourcefile and category for hydration testing""" - return MockAsyncSearchResultsIterator("hydrated", None) - - -@pytest.mark.asyncio -async def test_agentic_retrieval_with_hydration(chat_approach_with_hydration, monkeypatch): - """Test agentic retrieval with hydration enabled""" - - agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - - # Mock the agent retrieval and search client - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - monkeypatch.setattr(SearchClient, "search", mock_search_with_hydration) - - _, results = await chat_approach_with_hydration.run_agentic_retrieval( - messages=[], agent_client=agent_client, search_index_name="" - ) - - assert len(results) == 1 - assert results[0].id == "Benefit_Options-2.pdf" - # Content should be from hydrated search, not source_data - assert results[0].content == "There is a whistleblower policy." - assert results[0].sourcepage == "Benefit_Options-2.pdf" - assert results[0].search_agent_query == "whistleblower query" - # These fields should be present from hydration (from search results) - assert results[0].sourcefile == "Benefit_Options.pdf" - assert results[0].category == "benefits" - assert results[0].score == 0.03279569745063782 - assert results[0].reranker_score == 3.4577205181121826 - - @pytest.mark.asyncio async def test_compute_multimodal_embedding(monkeypatch, chat_approach): # Create a mock for the ImageEmbeddings.create_embedding_for_text method From eab9a3e6a456be01feab6039f7327f0f15fe9cc6 Mon Sep 17 00:00:00 2001 From: Taylor Date: Mon, 1 Sep 2025 15:57:40 +1000 Subject: [PATCH 09/11] Refactor mock retrieval functions for improved flexibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ๐ŸŽจ Introduce `create_mock_retrieve` to parameterise mock retrieval responses. - ๐Ÿ”„ Remove redundant mock search functions to streamline code. - ๐Ÿงช Update tests to use the new mock retrieval function for various scenarios. - ๐Ÿงน Clean up unused mock functions to enhance maintainability. --- tests/conftest.py | 44 +++++-- tests/mocks.py | 16 --- tests/test_agentic_retrieval.py | 195 +++++++++----------------------- 3 files changed, 91 insertions(+), 164 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 34be10826d..b9aeba9344 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,10 @@ MockResponse, MockTransport, mock_retrieval_response, + mock_retrieval_response_with_duplicates, + mock_retrieval_response_with_missing_doc_key, + mock_retrieval_response_with_sorting, + mock_retrieval_response_with_top_limit, mock_speak_text_cancelled, mock_speak_text_failed, mock_speak_text_success, @@ -68,13 +72,37 @@ async def mock_search(self, *args, **kwargs): return MockAsyncSearchResultsIterator(kwargs.get("search_text"), kwargs.get("vector_queries")) -async def mock_retrieve(self, *args, **kwargs): - retrieval_request = kwargs.get("retrieval_request") - assert retrieval_request is not None - assert retrieval_request.target_index_params is not None - assert len(retrieval_request.target_index_params) == 1 - self.filter = retrieval_request.target_index_params[0].filter_add_on - return mock_retrieval_response() +def create_mock_retrieve(response_type="default"): + """Create a mock_retrieve function that returns different response types. + + Args: + response_type: Type of response to return. Options: + - "default": mock_retrieval_response() + - "sorting": mock_retrieval_response_with_sorting() + - "duplicates": mock_retrieval_response_with_duplicates() + - "missing_doc_key": mock_retrieval_response_with_missing_doc_key() + - "top_limit": mock_retrieval_response_with_top_limit() + """ + + async def mock_retrieve_parameterized(self, *args, **kwargs): + retrieval_request = kwargs.get("retrieval_request") + assert retrieval_request is not None + assert retrieval_request.target_index_params is not None + assert len(retrieval_request.target_index_params) == 1 + self.filter = retrieval_request.target_index_params[0].filter_add_on + + if response_type == "sorting": + return mock_retrieval_response_with_sorting() + elif response_type == "duplicates": + return mock_retrieval_response_with_duplicates() + elif response_type == "missing_doc_key": + return mock_retrieval_response_with_missing_doc_key() + elif response_type == "top_limit": + return mock_retrieval_response_with_top_limit() + else: # default + return mock_retrieval_response() + + return mock_retrieve_parameterized @pytest.fixture @@ -281,7 +309,7 @@ async def mock_get_index(*args, **kwargs): @pytest.fixture def mock_acs_agent(monkeypatch): - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieve) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", create_mock_retrieve()) async def mock_get_agent(*args, **kwargs): return MockAgent diff --git a/tests/mocks.py b/tests/mocks.py index 16bfd31120..de84fa470e 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -545,22 +545,6 @@ def mock_retrieval_response_with_duplicates(): ) -async def mock_search_for_hydration(*args, **kwargs): - """Mock search that returns documents matching the filter""" - filter_param = kwargs.get("filter", "") - - # Create documents based on filter - use search_text to distinguish different calls - search_text = "" - if "doc1" in filter_param and "doc2" in filter_param: - search_text = "hydrated_multi" - elif "doc1" in filter_param: - search_text = "hydrated_single" - else: - search_text = "hydrated_empty" - - return MockAsyncSearchResultsIterator(search_text, None) - - def mock_retrieval_response_with_missing_doc_key(): """Mock response with missing doc_key to test continue condition""" return KnowledgeAgentRetrievalResponse( diff --git a/tests/test_agentic_retrieval.py b/tests/test_agentic_retrieval.py index 6dcfe92a25..656a3fccbc 100644 --- a/tests/test_agentic_retrieval.py +++ b/tests/test_agentic_retrieval.py @@ -4,48 +4,21 @@ from azure.search.documents.agent.models import ( KnowledgeAgentAzureSearchDocReference, KnowledgeAgentMessage, - KnowledgeAgentMessageTextContent, KnowledgeAgentRetrievalResponse, - KnowledgeAgentSearchActivityRecord, - KnowledgeAgentSearchActivityRecordQuery, ) from azure.search.documents.aio import SearchClient +from .conftest import create_mock_retrieve from .mocks import ( MockAsyncSearchResultsIterator, - mock_retrieval_response, - mock_retrieval_response_with_duplicates, - mock_retrieval_response_with_sorting, ) - - -async def mock_search(*args, **kwargs): - return MockAsyncSearchResultsIterator(kwargs.get("search_text"), kwargs.get("vector_queries")) - - -async def mock_search_for_hydration(*args, **kwargs): - filter_param = kwargs.get("filter", "") - - search_text = "" - if "doc1" in filter_param and "doc2" in filter_param: - search_text = "hydrated_multi" - elif "doc1" in filter_param: - search_text = "hydrated_single" - else: - search_text = "hydrated_empty" - - kwargs["search_text"] = search_text - - return mock_search(*args, **kwargs) - - @pytest.mark.asyncio async def test_agentic_retrieval_non_hydrated_default_sort(chat_approach, monkeypatch): """Test non-hydrated path with default sorting (preserve original order)""" - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_response_with_sorting) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", create_mock_retrieve("sorting")) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) @@ -71,12 +44,15 @@ async def test_agentic_retrieval_non_hydrated_default_sort(chat_approach, monkey async def test_agentic_retrieval_non_hydrated_interleaved_sort(chat_approach, monkeypatch): """Test non-hydrated path with interleaved sorting""" - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_response_with_sorting) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", create_mock_retrieve("sorting")) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) _, results = await chat_approach.run_agentic_retrieval( - messages=[], agent_client=agent_client, search_index_name="test-index", results_merge_strategy="interleaved" + messages=[], + agent_client=agent_client, + search_index_name="test-index", + results_merge_strategy="interleaved", ) assert len(results) == 2 @@ -94,13 +70,21 @@ async def test_agentic_retrieval_non_hydrated_interleaved_sort(chat_approach, mo async def test_agentic_retrieval_hydrated_with_sorting(chat_approach_with_hydration, monkeypatch): """Test hydrated path with sorting""" - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_response_with_sorting) - monkeypatch.setattr(SearchClient, "search", mock_search_for_hydration) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", create_mock_retrieve("sorting")) + + async def mock_search(self, *args, **kwargs): + # For hydration, we expect a filter like "search.in(id, 'doc1,doc2', ',')" + return MockAsyncSearchResultsIterator("hydrated_multi", None) + + monkeypatch.setattr(SearchClient, "search", mock_search) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) _, results = await chat_approach_with_hydration.run_agentic_retrieval( - messages=[], agent_client=agent_client, search_index_name="test-index", results_merge_strategy="interleaved" + messages=[], + agent_client=agent_client, + search_index_name="test-index", + results_merge_strategy="interleaved", ) assert len(results) == 2 @@ -116,8 +100,13 @@ async def test_agentic_retrieval_hydrated_with_sorting(chat_approach_with_hydrat async def test_hydrate_agent_references_deduplication(chat_approach_with_hydration, monkeypatch): """Test that hydrate_agent_references deduplicates doc_keys""" - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_response_with_duplicates) - monkeypatch.setattr(SearchClient, "search", mock_search_for_hydration) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", create_mock_retrieve("duplicates")) + + async def mock_search(self, *args, **kwargs): + # For deduplication test, we expect doc1 and doc2 to be in the filter + return MockAsyncSearchResultsIterator("hydrated_multi", None) + + monkeypatch.setattr(SearchClient, "search", mock_search) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) @@ -138,7 +127,9 @@ async def test_agentic_retrieval_no_references(chat_approach, monkeypatch): async def mock_retrieval(*args, **kwargs): return KnowledgeAgentRetrievalResponse( - response=[KnowledgeAgentMessage(role="assistant", content=[])], activity=[], references=[] + response=[KnowledgeAgentMessage(role="assistant", content=[])], + activity=[], + references=[], ) monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) @@ -156,10 +147,7 @@ async def mock_retrieval(*args, **kwargs): async def test_activity_mapping_injection(chat_approach, monkeypatch): """Test that search_agent_query is properly injected from activity mapping""" - async def mock_retrieval(*args, **kwargs): - return mock_retrieval_response_with_sorting() - - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", create_mock_retrieve("sorting")) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) @@ -178,93 +166,20 @@ async def mock_retrieval(*args, **kwargs): assert doc2.search_agent_query == "second query" # From activity_source=2 -def mock_retrieval_response_with_missing_doc_key(): - """Mock response with missing doc_key to test continue condition""" - return KnowledgeAgentRetrievalResponse( - response=[ - KnowledgeAgentMessage( - role="assistant", - content=[KnowledgeAgentMessageTextContent(text="Test response")], - ) - ], - activity=[ - KnowledgeAgentSearchActivityRecord( - id=1, - target_index="index", - query=KnowledgeAgentSearchActivityRecordQuery(search="query"), - count=10, - elapsed_ms=50, - ), - ], - references=[ - KnowledgeAgentAzureSearchDocReference( - id="1", - activity_source=1, - doc_key=None, # Missing doc_key - source_data={"content": "Content 1", "sourcepage": "page1.pdf"}, - ), - KnowledgeAgentAzureSearchDocReference( - id="2", - activity_source=1, - doc_key="", # Empty doc_key - source_data={"content": "Content 2", "sourcepage": "page2.pdf"}, - ), - KnowledgeAgentAzureSearchDocReference( - id="3", - activity_source=1, - doc_key="doc3", # Valid doc_key - source_data={"content": "Content 3", "sourcepage": "page3.pdf"}, - ), - ], - ) - - -def mock_retrieval_response_with_top_limit(): - """Mock response with many references to test top limit during document building""" - references = [] - for i in range(15): # More than any reasonable top limit - references.append( - KnowledgeAgentAzureSearchDocReference( - id=str(i), - activity_source=1, - doc_key=f"doc{i}", - source_data={"content": f"Content {i}", "sourcepage": f"page{i}.pdf"}, - ) - ) - - return KnowledgeAgentRetrievalResponse( - response=[ - KnowledgeAgentMessage( - role="assistant", - content=[KnowledgeAgentMessageTextContent(text="Test response")], - ) - ], - activity=[ - KnowledgeAgentSearchActivityRecord( - id=1, - target_index="index", - query=KnowledgeAgentSearchActivityRecordQuery(search="query"), - count=10, - elapsed_ms=50, - ), - ], - references=references, - ) - - @pytest.mark.asyncio async def test_hydrate_agent_references_missing_doc_keys(chat_approach_with_hydration, monkeypatch): """Test that hydrate_agent_references handles missing/empty doc_keys correctly""" - async def mock_retrieval(*args, **kwargs): - return mock_retrieval_response_with_missing_doc_key() + monkeypatch.setattr( + KnowledgeAgentRetrievalClient, + "retrieve", + create_mock_retrieve("missing_doc_key"), + ) - # Mock search to return single document for doc3 - async def mock_search_single(*args, **kwargs): + async def mock_search(self, *args, **kwargs): return MockAsyncSearchResultsIterator("hydrated_single", None) - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - monkeypatch.setattr(SearchClient, "search", mock_search_single) + monkeypatch.setattr(SearchClient, "search", mock_search) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) @@ -327,12 +242,12 @@ async def mock_retrieval_valid_keys(*args, **kwargs): ], ) - # Mock search to return empty results (no documents found) - async def mock_search_returns_empty(*args, **kwargs): + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_valid_keys) + + async def mock_search(self, *args, **kwargs): return MockAsyncSearchResultsIterator("hydrated_empty", None) - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval_valid_keys) - monkeypatch.setattr(SearchClient, "search", mock_search_returns_empty) + monkeypatch.setattr(SearchClient, "search", mock_search) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) @@ -346,18 +261,18 @@ async def mock_search_returns_empty(*args, **kwargs): @pytest.mark.asyncio -async def test_agentic_retrieval_with_top_limit_during_building(chat_approach_with_hydration, monkeypatch): - """Test that document building respects top limit and breaks early""" +async def test_agentic_retrieval_with_top_limit_during_building(chat_approach, monkeypatch): + """Test that document building respects top limit and breaks early (non-hydrated path)""" - async def mock_retrieval(*args, **kwargs): - return mock_retrieval_response_with_top_limit() - - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", create_mock_retrieve("top_limit")) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) - _, results = await chat_approach_with_hydration.run_agentic_retrieval( - messages=[], agent_client=agent_client, search_index_name="test-index", top=5 # Limit to 5 documents + _, results = await chat_approach.run_agentic_retrieval( + messages=[], + agent_client=agent_client, + search_index_name="test-index", + top=5, # Limit to 5 documents ) # Should get exactly 5 documents due to top limit during building @@ -371,20 +286,20 @@ async def mock_retrieval(*args, **kwargs): async def test_hydrate_agent_references_with_top_limit_during_collection(chat_approach_with_hydration, monkeypatch): """Test that hydration respects top limit when collecting doc_keys""" - async def mock_retrieval(*args, **kwargs): - return mock_retrieval_response_with_top_limit() + monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", create_mock_retrieve("top_limit")) - # Mock search to return multi results (more than our top limit) - async def mock_search_multi(*args, **kwargs): + async def mock_search(self, *args, **kwargs): return MockAsyncSearchResultsIterator("hydrated_multi", None) - monkeypatch.setattr(KnowledgeAgentRetrievalClient, "retrieve", mock_retrieval) - monkeypatch.setattr(SearchClient, "search", mock_search_multi) + monkeypatch.setattr(SearchClient, "search", mock_search) agent_client = KnowledgeAgentRetrievalClient(endpoint="", agent_name="", credential=AzureKeyCredential("")) _, results = await chat_approach_with_hydration.run_agentic_retrieval( - messages=[], agent_client=agent_client, search_index_name="test-index", top=2 # Limit to 2 documents + messages=[], + agent_client=agent_client, + search_index_name="test-index", + top=2, # Limit to 2 documents ) # Should get exactly 2 documents due to top limit during doc_keys collection From a9b183f5406b5cdf1931c45f5cbcb52d4232b92b Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 2 Sep 2025 23:35:52 -0700 Subject: [PATCH 10/11] Rename env var to match API parameter --- app/backend/app.py | 6 +++--- docs/agentic_retrieval.md | 4 +++- infra/main.bicep | 4 ++-- infra/main.parameters.json | 4 ++-- tests/conftest.py | 4 ++-- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index 38d3dbb007..d391e8b779 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -471,7 +471,7 @@ async def setup_clients(): USE_CHAT_HISTORY_BROWSER = os.getenv("USE_CHAT_HISTORY_BROWSER", "").lower() == "true" USE_CHAT_HISTORY_COSMOS = os.getenv("USE_CHAT_HISTORY_COSMOS", "").lower() == "true" USE_AGENTIC_RETRIEVAL = os.getenv("USE_AGENTIC_RETRIEVAL", "").lower() == "true" - ENABLE_AGENTIC_REF_HYDRATION = os.getenv("ENABLE_AGENTIC_REF_HYDRATION", "").lower() == "true" + ENABLE_AGENTIC_RETRIEVAL_SOURCE_DATA = os.getenv("ENABLE_AGENTIC_RETRIEVAL_SOURCE_DATA", "").lower() == "true" # WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None @@ -690,7 +690,7 @@ async def setup_clients(): query_speller=AZURE_SEARCH_QUERY_SPELLER, prompt_manager=prompt_manager, reasoning_effort=OPENAI_REASONING_EFFORT, - hydrate_references=ENABLE_AGENTIC_REF_HYDRATION, + hydrate_references=ENABLE_AGENTIC_RETRIEVAL_SOURCE_DATA, multimodal_enabled=USE_MULTIMODAL, image_embeddings_client=image_embeddings_client, global_blob_manager=global_blob_manager, @@ -718,7 +718,7 @@ async def setup_clients(): query_speller=AZURE_SEARCH_QUERY_SPELLER, prompt_manager=prompt_manager, reasoning_effort=OPENAI_REASONING_EFFORT, - hydrate_references=ENABLE_AGENTIC_REF_HYDRATION, + hydrate_references=ENABLE_AGENTIC_RETRIEVAL_SOURCE_DATA, multimodal_enabled=USE_MULTIMODAL, image_embeddings_client=image_embeddings_client, global_blob_manager=global_blob_manager, diff --git a/docs/agentic_retrieval.md b/docs/agentic_retrieval.md index 1774a950b0..baa55994c1 100644 --- a/docs/agentic_retrieval.md +++ b/docs/agentic_retrieval.md @@ -39,9 +39,11 @@ See the agentic retrieval documentation. By default, agentic retrieval only returns fields included in the semantic configuration. You can enable this optional feature below, to include all fields from the search index in the result. + โš ๏ธ This feature is currently only compatible with indexes set up with integrated vectorization, + or indexes that otherwise have an "id" field marked as filterable. ```shell - azd env set ENABLE_AGENTIC_REF_HYDRATION true + azd env set ENABLE_AGENTIC_RETRIEVAL_SOURCE_DATA true ``` 4. **Update the infrastructure and application:** diff --git a/infra/main.bicep b/infra/main.bicep index d130230773..6964b9dc75 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -41,7 +41,7 @@ param storageSkuName string // Set in main.parameters.json param defaultReasoningEffort string // Set in main.parameters.json param useAgenticRetrieval bool // Set in main.parameters.json -param enableAgenticRefHydration bool // Set in main.parameters.json +param enableAgenticRetrievalSourceData bool // Set in main.parameters.json param userStorageAccountName string = '' param userStorageContainerName string = 'user-content' @@ -424,7 +424,7 @@ var appEnvVariables = { USE_SPEECH_OUTPUT_BROWSER: useSpeechOutputBrowser USE_SPEECH_OUTPUT_AZURE: useSpeechOutputAzure USE_AGENTIC_RETRIEVAL: useAgenticRetrieval - ENABLE_AGENTIC_REF_HYDRATION: enableAgenticRefHydration + ENABLE_AGENTIC_RETRIEVAL_SOURCE_DATA: enableAgenticRetrievalSourceData // Chat history settings USE_CHAT_HISTORY_BROWSER: useChatHistoryBrowser USE_CHAT_HISTORY_COSMOS: useChatHistoryCosmos diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 46c94b6e42..dd047dc56f 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -338,8 +338,8 @@ "useAgenticRetrieval": { "value": "${USE_AGENTIC_RETRIEVAL=false}" }, - "enableAgenticRefHydration": { - "value": "${ENABLE_AGENTIC_REF_HYDRATION=false}" + "enableAgenticRetrievalSourceData": { + "value": "${ENABLE_AGENTIC_RETRIEVAL_SOURCE_DATA=false}" }, "ragSearchTextEmbeddings": { "value": "${RAG_SEARCH_TEXT_EMBEDDINGS=true}" diff --git a/tests/conftest.py b/tests/conftest.py index b9aeba9344..89c1c66711 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -447,7 +447,7 @@ async def mock_exists(*args, **kwargs): "AZURE_OPENAI_SEARCHAGENT_MODEL": "gpt-4.1-mini", "AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT": "gpt-4.1-mini", "USE_AGENTIC_RETRIEVAL": "true", - "ENABLE_AGENTIC_REF_HYDRATION": "true", + "ENABLE_AGENTIC_RETRIEVAL_SOURCE_DATA": "true", } ] @@ -461,7 +461,7 @@ async def mock_exists(*args, **kwargs): "AZURE_OPENAI_SEARCHAGENT_MODEL": "gpt-4.1-mini", "AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT": "gpt-4.1-mini", "USE_AGENTIC_RETRIEVAL": "true", - "ENABLE_AGENTIC_REF_HYDRATION": "true", + "ENABLE_AGENTIC_RETRIEVAL_SOURCE_DATA": "true", "AZURE_USE_AUTHENTICATION": "true", "AZURE_SERVER_APP_ID": "SERVER_APP", "AZURE_SERVER_APP_SECRET": "SECRET", From 4f8749807417bb68e291ee21f34d428799f7093b Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 2 Sep 2025 23:42:45 -0700 Subject: [PATCH 11/11] Revert CONTRIBUTING.md TOC change as unneeded --- CONTRIBUTING.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a320933a62..4ac1ea82ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,15 +12,14 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. -- [Contributing](#contributing) - - [Submitting a Pull Request (PR)](#submitting-a-pull-request-pr) - - [Setting up the development environment](#setting-up-the-development-environment) - - [Running unit tests](#running-unit-tests) - - [Running E2E tests](#running-e2e-tests) - - [Code style](#code-style) - - [Adding new features](#adding-new-features) - - [Adding new azd environment variables](#adding-new-azd-environment-variables) - - [Adding new UI strings](#adding-new-ui-strings) +- [Submitting a Pull Request (PR)](#submitting-a-pull-request-pr) +- [Setting up the development environment](#setting-up-the-development-environment) +- [Running unit tests](#running-unit-tests) +- [Running E2E tests](#running-e2e-tests) +- [Code style](#code-style) +- [Adding new features](#adding-new-features) + - [Adding new azd environment variables](#adding-new-azd-environment-variables) + - [Adding new UI strings](#adding-new-ui-strings) ## Submitting a Pull Request (PR)