Skip to content

Commit 212d3cf

Browse files
committed
add e2e test
1 parent a984dc1 commit 212d3cf

File tree

3 files changed

+175
-1
lines changed

3 files changed

+175
-1
lines changed

tests/conftest.py

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
MockBlobClient,
4040
MockResponse,
4141
mock_computervision_response,
42+
mock_retrieval_response,
4243
mock_speak_text_cancelled,
4344
mock_speak_text_failed,
4445
mock_speak_text_success,
@@ -60,7 +61,7 @@ async def mock_search(self, *args, **kwargs):
6061

6162

6263
async def mock_retrieve(self, *args, **kwargs):
63-
return
64+
return mock_retrieval_response()
6465

6566

6667
@pytest.fixture
@@ -363,6 +364,19 @@ def mock_blob_container_client(monkeypatch):
363364
},
364365
]
365366

367+
agent_envs = [
368+
{
369+
"OPENAI_HOST": "azure",
370+
"AZURE_OPENAI_SERVICE": "test-openai-service",
371+
"AZURE_OPENAI_CHATGPT_MODEL": "gpt-4o-mini",
372+
"AZURE_OPENAI_CHATGPT_DEPLOYMENT": "gpt-4o-mini",
373+
"AZURE_OPENAI_EMB_DEPLOYMENT": "test-ada",
374+
"AZURE_OPENAI_SEARCHAGENT_MODEL": "gpt-4o-mini",
375+
"AZURE_OPENAI_SEARCHAGENT_DEPLOYMENT": "gpt-4o-mini",
376+
"USE_AGENTIC_RETRIEVAL": "true",
377+
}
378+
]
379+
366380

367381
@pytest.fixture(params=envs, ids=["client0", "client1"])
368382
def mock_env(monkeypatch, request):
@@ -413,6 +427,28 @@ def mock_reasoning_env(monkeypatch, request):
413427
mock_default_azure_credential.return_value = MockAzureCredential()
414428
yield
415429

430+
@pytest.fixture(params=agent_envs, ids=["agent_client0"])
431+
def mock_agent_env(monkeypatch, request):
432+
with mock.patch.dict(os.environ, clear=True):
433+
monkeypatch.setenv("AZURE_STORAGE_ACCOUNT", "test-storage-account")
434+
monkeypatch.setenv("AZURE_STORAGE_CONTAINER", "test-storage-container")
435+
monkeypatch.setenv("AZURE_STORAGE_RESOURCE_GROUP", "test-storage-rg")
436+
monkeypatch.setenv("AZURE_SUBSCRIPTION_ID", "test-storage-subid")
437+
monkeypatch.setenv("ENABLE_LANGUAGE_PICKER", "true")
438+
monkeypatch.setenv("USE_SPEECH_INPUT_BROWSER", "true")
439+
monkeypatch.setenv("USE_SPEECH_OUTPUT_AZURE", "true")
440+
monkeypatch.setenv("AZURE_SEARCH_INDEX", "test-search-index")
441+
monkeypatch.setenv("AZURE_SEARCH_AGENT", "test-search-agent")
442+
monkeypatch.setenv("AZURE_SEARCH_SERVICE", "test-search-service")
443+
monkeypatch.setenv("AZURE_SPEECH_SERVICE_ID", "test-id")
444+
monkeypatch.setenv("AZURE_SPEECH_SERVICE_LOCATION", "eastus")
445+
monkeypatch.setenv("ALLOWED_ORIGIN", "https://frontend.com")
446+
for key, value in request.param.items():
447+
monkeypatch.setenv(key, value)
448+
449+
with mock.patch("app.AzureDeveloperCliCredential") as mock_default_azure_credential:
450+
mock_default_azure_credential.return_value = MockAzureCredential()
451+
yield
416452

417453
@pytest_asyncio.fixture(scope="function")
418454
async def client(
@@ -451,6 +487,25 @@ async def reasoning_client(
451487
mock_openai_embedding(test_app.app.config[app.CONFIG_OPENAI_CLIENT])
452488
yield test_app.test_client()
453489

490+
@pytest_asyncio.fixture(scope="function")
491+
async def agent_client(
492+
monkeypatch,
493+
mock_agent_env,
494+
mock_openai_chatcompletion,
495+
mock_openai_embedding,
496+
mock_acs_search,
497+
mock_acs_agent,
498+
mock_blob_container_client,
499+
mock_azurehttp_calls,
500+
):
501+
quart_app = app.create_app()
502+
503+
async with quart_app.test_app() as test_app:
504+
test_app.app.config.update({"TESTING": True})
505+
mock_openai_chatcompletion(test_app.app.config[app.CONFIG_OPENAI_CLIENT])
506+
mock_openai_embedding(test_app.app.config[app.CONFIG_OPENAI_CLIENT])
507+
yield test_app.test_client()
508+
454509

455510
@pytest_asyncio.fixture(scope="function")
456511
async def client_with_expiring_token(
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
{
2+
"context": {
3+
"data_points": {
4+
"images": null,
5+
"text": [
6+
"Benefit_Options-2.pdf: There is a whistleblower policy."
7+
]
8+
},
9+
"followup_questions": null,
10+
"thoughts": [
11+
{
12+
"description": [
13+
{
14+
"content": "What is the capital of France?",
15+
"role": "user"
16+
}
17+
],
18+
"props": {
19+
"filter": null,
20+
"max_docs_for_reranker": 150,
21+
"reranker_threshold": 0
22+
},
23+
"title": "Use agentic retrieval"
24+
},
25+
{
26+
"description": [
27+
{
28+
"captions": [],
29+
"category": null,
30+
"content": "There is a whistleblower policy.",
31+
"groups": null,
32+
"id": "Benefit_Options-2.pdf",
33+
"oids": null,
34+
"reranker_score": null,
35+
"score": null,
36+
"search_agent_query": "whistleblower query",
37+
"sourcefile": null,
38+
"sourcepage": "Benefit_Options-2.pdf"
39+
}
40+
],
41+
"props": {
42+
"deployment": "gpt-4o-mini",
43+
"model": "gpt-4o-mini",
44+
"query_plan": [
45+
{
46+
"elapsed_ms": 200,
47+
"id": 0,
48+
"input_tokens": 10,
49+
"output_tokens": 20,
50+
"type": "ModelQueryPlanning"
51+
},
52+
{
53+
"count": 10,
54+
"elapsed_ms": 50,
55+
"id": 1,
56+
"query": {
57+
"search": "whistleblower query"
58+
},
59+
"target_index": "index",
60+
"type": "AzureSearchQuery"
61+
}
62+
]
63+
},
64+
"title": "Agentic retrieval results (top 3)"
65+
},
66+
{
67+
"description": [
68+
{
69+
"content": "You are an intelligent assistant helping Contoso Inc employees with their healthcare plan questions and employee handbook questions.\nUse 'you' to refer to the individual asking the questions even if they ask with 'I'.\nAnswer the following question using only the data provided in the sources below.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response.\nIf you cannot answer using the sources below, say you don't know. Use below example to answer",
70+
"role": "system"
71+
},
72+
{
73+
"content": "What is the deductible for the employee plan for a visit to Overlake in Bellevue?\n\nSources:\ninfo1.txt: deductibles depend on whether you are in-network or out-of-network. In-network deductibles are $500 for employee and $1000 for family. Out-of-network deductibles are $1000 for employee and $2000 for family.\ninfo2.pdf: Overlake is in-network for the employee plan.\ninfo3.pdf: Overlake is the name of the area that includes a park and ride near Bellevue.\ninfo4.pdf: In-network institutions include Overlake, Swedish and others in the region.",
74+
"role": "user"
75+
},
76+
{
77+
"content": "In-network deductibles are $500 for employee and $1000 for family [info1.txt] and Overlake is in-network for the employee plan [info2.pdf][info4.pdf].",
78+
"role": "assistant"
79+
},
80+
{
81+
"content": "What is the capital of France?\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy.",
82+
"role": "user"
83+
}
84+
],
85+
"props": {
86+
"deployment": "gpt-4o-mini",
87+
"model": "gpt-4o-mini",
88+
"token_usage": {
89+
"completion_tokens": 896,
90+
"prompt_tokens": 23,
91+
"reasoning_tokens": 0,
92+
"total_tokens": 919
93+
}
94+
},
95+
"title": "Prompt to generate answer"
96+
}
97+
]
98+
},
99+
"message": {
100+
"content": "The capital of France is Paris. [Benefit_Options-2.pdf].",
101+
"role": "assistant"
102+
},
103+
"session_state": null
104+
}

tests/test_app.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,21 @@ async def test_ask_rtr_text(client, snapshot):
168168
result = await response.get_json()
169169
snapshot.assert_match(json.dumps(result, indent=4), "result.json")
170170

171+
@pytest.mark.asyncio
172+
async def test_ask_rtr_text_agent(agent_client, snapshot):
173+
response = await agent_client.post(
174+
"/ask",
175+
json={
176+
"messages": [{"content": "What is the capital of France?", "role": "user"}],
177+
"context": {
178+
"overrides": {"retrieval_mode": "text", "use_agentic_retrieval": True},
179+
},
180+
},
181+
)
182+
assert response.status_code == 200
183+
result = await response.get_json()
184+
snapshot.assert_match(json.dumps(result, indent=4), "result.json")
185+
171186

172187
@pytest.mark.asyncio
173188
async def test_ask_rtr_text_filter(auth_client, snapshot):

0 commit comments

Comments
 (0)