Skip to content

Commit 39bf02c

Browse files
mattgotteinermattgotteiner
committed
adding tests
1 parent 8347150 commit 39bf02c

File tree

5 files changed

+357
-4
lines changed

5 files changed

+357
-4
lines changed

‎tests/conftest.py‎

Lines changed: 89 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import json
22
import os
3-
from typing import IO
3+
from typing import IO, Dict
44
from unittest import mock
55

66
import aiohttp
@@ -14,7 +14,7 @@
1414
from azure.search.documents.indexes.aio import SearchIndexClient
1515
from azure.search.documents.indexes.models import SearchField, SearchIndex
1616
from azure.storage.blob.aio import ContainerClient
17-
from openai.types import CreateEmbeddingResponse, Embedding
17+
from openai.types import CompletionUsage, CreateEmbeddingResponse, Embedding
1818
from openai.types.chat import ChatCompletion, ChatCompletionChunk
1919
from openai.types.chat.chat_completion import (
2020
ChatCompletionMessage,
@@ -107,10 +107,23 @@ def patch(openai_client):
107107

108108
@pytest.fixture
109109
def mock_openai_chatcompletion(monkeypatch):
110+
reasoning = os.getenv("TEST_ENABLE_REASONING") is not None
111+
completion_usage: Dict[str, any] = {
112+
"completion_tokens": 896,
113+
"prompt_tokens": 23,
114+
"total_tokens": 919,
115+
"completion_tokens_details": {
116+
"accepted_prediction_tokens": 0,
117+
"audio_tokens": 0,
118+
"reasoning_tokens": 384 if reasoning else 0,
119+
"rejected_prediction_tokens": 0,
120+
},
121+
}
122+
110123
class AsyncChatCompletionIterator:
111124
def __init__(self, answer: str):
112125
chunk_id = "test-id"
113-
model = "gpt-4o-mini"
126+
model = "gpt-4o-mini" if not reasoning else "o3-mini"
114127
self.responses = [
115128
{"object": "chat.completion.chunk", "choices": [], "id": chunk_id, "model": model, "created": 1},
116129
{
@@ -170,6 +183,17 @@ def __init__(self, answer: str):
170183
}
171184
)
172185

186+
self.responses.append(
187+
{
188+
"object": "chat.completion.chunk",
189+
"choices": [],
190+
"id": chunk_id,
191+
"model": model,
192+
"created": 1,
193+
"usage": completion_usage,
194+
}
195+
)
196+
173197
def __aiter__(self):
174198
return self
175199

@@ -208,6 +232,7 @@ async def mock_acreate(*args, **kwargs):
208232
id="test-123",
209233
created=0,
210234
model="test-model",
235+
usage=CompletionUsage.model_validate(completion_usage),
211236
)
212237

213238
def patch(openai_client):
@@ -292,6 +317,24 @@ def mock_blob_container_client(monkeypatch):
292317
},
293318
]
294319

320+
reasoning_envs = [
321+
{
322+
"OPENAI_HOST": "azure",
323+
"AZURE_OPENAI_SERVICE": "test-openai-service",
324+
"AZURE_OPENAI_CHATGPT_MODEL": "o3-mini",
325+
"AZURE_OPENAI_CHATGPT_DEPLOYMENT": "o3-mini",
326+
"AZURE_OPENAI_EMB_DEPLOYMENT": "test-ada",
327+
},
328+
{
329+
"OPENAI_HOST": "azure",
330+
"AZURE_OPENAI_SERVICE": "test-openai-service",
331+
"AZURE_OPENAI_CHATGPT_MODEL": "o3-mini",
332+
"AZURE_OPENAI_CHATGPT_DEPLOYMENT": "o3-mini",
333+
"AZURE_OPENAI_EMB_DEPLOYMENT": "test-ada",
334+
"AZURE_OPENAI_REASONING_EFFORT": "low",
335+
},
336+
]
337+
295338

296339
@pytest.fixture(params=envs, ids=["client0", "client1"])
297340
def mock_env(monkeypatch, request):
@@ -319,6 +362,30 @@ def mock_env(monkeypatch, request):
319362
yield
320363

321364

365+
@pytest.fixture(params=reasoning_envs, ids=["reasoning_client0", "reasoning_client1"])
366+
def mock_reasoning_env(monkeypatch, request):
367+
with mock.patch.dict(os.environ, clear=True):
368+
monkeypatch.setenv("AZURE_STORAGE_ACCOUNT", "test-storage-account")
369+
monkeypatch.setenv("AZURE_STORAGE_CONTAINER", "test-storage-container")
370+
monkeypatch.setenv("AZURE_STORAGE_RESOURCE_GROUP", "test-storage-rg")
371+
monkeypatch.setenv("AZURE_SUBSCRIPTION_ID", "test-storage-subid")
372+
monkeypatch.setenv("ENABLE_LANGUAGE_PICKER", "true")
373+
monkeypatch.setenv("USE_SPEECH_INPUT_BROWSER", "true")
374+
monkeypatch.setenv("USE_SPEECH_OUTPUT_AZURE", "true")
375+
monkeypatch.setenv("AZURE_SEARCH_INDEX", "test-search-index")
376+
monkeypatch.setenv("AZURE_SEARCH_SERVICE", "test-search-service")
377+
monkeypatch.setenv("AZURE_SPEECH_SERVICE_ID", "test-id")
378+
monkeypatch.setenv("AZURE_SPEECH_SERVICE_LOCATION", "eastus")
379+
monkeypatch.setenv("ALLOWED_ORIGIN", "https://frontend.com")
380+
monkeypatch.setenv("TEST_ENABLE_REASONING", "true")
381+
for key, value in request.param.items():
382+
monkeypatch.setenv(key, value)
383+
384+
with mock.patch("app.AzureDeveloperCliCredential") as mock_default_azure_credential:
385+
mock_default_azure_credential.return_value = MockAzureCredential()
386+
yield
387+
388+
322389
@pytest_asyncio.fixture(scope="function")
323390
async def client(
324391
monkeypatch,
@@ -338,6 +405,25 @@ async def client(
338405
yield test_app.test_client()
339406

340407

408+
@pytest_asyncio.fixture(scope="function")
409+
async def reasoning_client(
410+
monkeypatch,
411+
mock_reasoning_env,
412+
mock_openai_chatcompletion,
413+
mock_openai_embedding,
414+
mock_acs_search,
415+
mock_blob_container_client,
416+
mock_azurehttp_calls,
417+
):
418+
quart_app = app.create_app()
419+
420+
async with quart_app.test_app() as test_app:
421+
test_app.app.config.update({"TESTING": True})
422+
mock_openai_chatcompletion(test_app.app.config[app.CONFIG_OPENAI_CLIENT])
423+
mock_openai_embedding(test_app.app.config[app.CONFIG_OPENAI_CLIENT])
424+
yield test_app.test_client()
425+
426+
341427
@pytest_asyncio.fixture(scope="function")
342428
async def client_with_expiring_token(
343429
monkeypatch,

‎tests/snapshots/test_app/test_chat_text_reasoning/reasoning_client0/result.json‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
{
2+
"context": {
3+
"data_points": {
4+
"images": null,
5+
"text": [
6+
"Benefit_Options-2.pdf: There is a whistleblower policy."
7+
]
8+
},
9+
"followup_questions": null,
10+
"thoughts": [
11+
{
12+
"description": [
13+
{
14+
"content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0.",
15+
"role": "system"
16+
},
17+
{
18+
"content": "How did crypto do last year?",
19+
"role": "user"
20+
},
21+
{
22+
"content": "Summarize Cryptocurrency Market Dynamics from last year",
23+
"role": "assistant"
24+
},
25+
{
26+
"content": "What are my health plans?",
27+
"role": "user"
28+
},
29+
{
30+
"content": "Show available health plans",
31+
"role": "assistant"
32+
},
33+
{
34+
"content": "Generate search query for: What is the capital of France?",
35+
"role": "user"
36+
}
37+
],
38+
"props": {
39+
"deployment": "o3-mini",
40+
"model": "o3-mini",
41+
"reasoning_effort": "low",
42+
"token_usage": {
43+
"completion_tokens": 896,
44+
"prompt_tokens": 23,
45+
"reasoning_tokens": 384,
46+
"total_tokens": 919
47+
}
48+
},
49+
"title": "Prompt to generate search query"
50+
},
51+
{
52+
"description": "capital of France",
53+
"props": {
54+
"filter": null,
55+
"top": 3,
56+
"use_query_rewriting": false,
57+
"use_semantic_captions": false,
58+
"use_semantic_ranker": false,
59+
"use_text_search": true,
60+
"use_vector_search": false
61+
},
62+
"title": "Search using generated search query"
63+
},
64+
{
65+
"description": [
66+
{
67+
"captions": [
68+
{
69+
"additional_properties": {},
70+
"highlights": [],
71+
"text": "Caption: A whistleblower policy."
72+
}
73+
],
74+
"category": null,
75+
"content": "There is a whistleblower policy.",
76+
"embedding": null,
77+
"groups": null,
78+
"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2",
79+
"imageEmbedding": null,
80+
"oids": null,
81+
"reranker_score": 3.4577205181121826,
82+
"score": 0.03279569745063782,
83+
"sourcefile": "Benefit_Options.pdf",
84+
"sourcepage": "Benefit_Options-2.pdf"
85+
}
86+
],
87+
"props": null,
88+
"title": "Search results"
89+
},
90+
{
91+
"description": [
92+
{
93+
"content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].",
94+
"role": "system"
95+
},
96+
{
97+
"content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy.",
98+
"role": "user"
99+
}
100+
],
101+
"props": {
102+
"deployment": "o3-mini",
103+
"model": "o3-mini",
104+
"reasoning_effort": null,
105+
"token_usage": {
106+
"completion_tokens": 896,
107+
"prompt_tokens": 23,
108+
"reasoning_tokens": 384,
109+
"total_tokens": 919
110+
}
111+
},
112+
"title": "Prompt to generate answer"
113+
}
114+
]
115+
},
116+
"message": {
117+
"content": "The capital of France is Paris. [Benefit_Options-2.pdf].",
118+
"role": "assistant"
119+
},
120+
"session_state": null
121+
}

‎tests/snapshots/test_app/test_chat_text_reasoning/reasoning_client1/result.json‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
{
2+
"context": {
3+
"data_points": {
4+
"images": null,
5+
"text": [
6+
"Benefit_Options-2.pdf: There is a whistleblower policy."
7+
]
8+
},
9+
"followup_questions": null,
10+
"thoughts": [
11+
{
12+
"description": [
13+
{
14+
"content": "Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base.\nYou have access to Azure AI Search index with 100's of documents.\nGenerate a search query based on the conversation and the new question.\nDo not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms.\nDo not include any text inside [] or <<>> in the search query terms.\nDo not include any special characters like '+'.\nIf the question is not in English, translate the question to English before generating the search query.\nIf you cannot generate a search query, return just the number 0.",
15+
"role": "system"
16+
},
17+
{
18+
"content": "How did crypto do last year?",
19+
"role": "user"
20+
},
21+
{
22+
"content": "Summarize Cryptocurrency Market Dynamics from last year",
23+
"role": "assistant"
24+
},
25+
{
26+
"content": "What are my health plans?",
27+
"role": "user"
28+
},
29+
{
30+
"content": "Show available health plans",
31+
"role": "assistant"
32+
},
33+
{
34+
"content": "Generate search query for: What is the capital of France?",
35+
"role": "user"
36+
}
37+
],
38+
"props": {
39+
"deployment": "o3-mini",
40+
"model": "o3-mini",
41+
"reasoning_effort": "low",
42+
"token_usage": {
43+
"completion_tokens": 896,
44+
"prompt_tokens": 23,
45+
"reasoning_tokens": 384,
46+
"total_tokens": 919
47+
}
48+
},
49+
"title": "Prompt to generate search query"
50+
},
51+
{
52+
"description": "capital of France",
53+
"props": {
54+
"filter": null,
55+
"top": 3,
56+
"use_query_rewriting": false,
57+
"use_semantic_captions": false,
58+
"use_semantic_ranker": false,
59+
"use_text_search": true,
60+
"use_vector_search": false
61+
},
62+
"title": "Search using generated search query"
63+
},
64+
{
65+
"description": [
66+
{
67+
"captions": [
68+
{
69+
"additional_properties": {},
70+
"highlights": [],
71+
"text": "Caption: A whistleblower policy."
72+
}
73+
],
74+
"category": null,
75+
"content": "There is a whistleblower policy.",
76+
"embedding": null,
77+
"groups": null,
78+
"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2",
79+
"imageEmbedding": null,
80+
"oids": null,
81+
"reranker_score": 3.4577205181121826,
82+
"score": 0.03279569745063782,
83+
"sourcefile": "Benefit_Options.pdf",
84+
"sourcepage": "Benefit_Options-2.pdf"
85+
}
86+
],
87+
"props": null,
88+
"title": "Search results"
89+
},
90+
{
91+
"description": [
92+
{
93+
"content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\nAnswer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\nIf the question is not in English, answer in the language used in the question.\nEach source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].",
94+
"role": "system"
95+
},
96+
{
97+
"content": "What is the capital of France?\n\nSources:\n\nBenefit_Options-2.pdf: There is a whistleblower policy.",
98+
"role": "user"
99+
}
100+
],
101+
"props": {
102+
"deployment": "o3-mini",
103+
"model": "o3-mini",
104+
"reasoning_effort": "low",
105+
"token_usage": {
106+
"completion_tokens": 896,
107+
"prompt_tokens": 23,
108+
"reasoning_tokens": 384,
109+
"total_tokens": 919
110+
}
111+
},
112+
"title": "Prompt to generate answer"
113+
}
114+
]
115+
},
116+
"message": {
117+
"content": "The capital of France is Paris. [Benefit_Options-2.pdf].",
118+
"role": "assistant"
119+
},
120+
"session_state": null
121+
}

0 commit comments

Comments
 (0)