Skip to content

Commit 3d86d24

Browse files
pamelafoxpamelafox
authored
Add search scores, models, and deployments to "Thought process" tab, surface additional properties (#1375)
* update reqs * Add score, model, styling * Update tests * Use a new variable name to avoid type clash * Use correct variable name in thought process
1 parent e191f74 commit 3d86d24

File tree

54 files changed

+961
-328
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+961
-328
lines changed

‎app/backend/approaches/approach.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ class Document:
3030
oids: Optional[List[str]]
3131
groups: Optional[List[str]]
3232
captions: List[QueryCaptionResult]
33+
score: Optional[float] = None
34+
reranker_score: Optional[float] = None
3335

3436
def serialize_for_results(self) -> dict[str, Any]:
3537
return {
@@ -54,6 +56,8 @@ def serialize_for_results(self) -> dict[str, Any]:
5456
if self.captions
5557
else []
5658
),
59+
"score": self.score,
60+
"reranker_score": self.reranker_score,
5761
}
5862

5963
@classmethod
@@ -153,6 +157,8 @@ async def search(
153157
oids=document.get("oids"),
154158
groups=document.get("groups"),
155159
captions=cast(List[QueryCaptionResult], document.get("@search.captions")),
160+
score=document.get("@search.score"),
161+
reranker_score=document.get("@search.reranker_score"),
156162
)
157163
)
158164
return documents

‎app/backend/approaches/chatreadretrieveread.py‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ async def run_until_final_call(
116116
]
117117

118118
# STEP 1: Generate an optimized keyword search query based on the chat history and the last question
119-
messages = self.get_messages_from_history(
119+
query_messages = self.get_messages_from_history(
120120
system_prompt=self.query_prompt_template,
121121
model_id=self.chatgpt_model,
122122
history=history,
@@ -126,7 +126,7 @@ async def run_until_final_call(
126126
)
127127

128128
chat_completion: ChatCompletion = await self.openai_client.chat.completions.create(
129-
messages=messages, # type: ignore
129+
messages=query_messages, # type: ignore
130130
# Azure Open AI takes the deployment name as the model name
131131
model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model,
132132
temperature=0.0, # Minimize creativity for search query generation
@@ -179,16 +179,38 @@ async def run_until_final_call(
179179
"data_points": data_points,
180180
"thoughts": [
181181
ThoughtStep(
182-
"Original user query",
183-
original_user_query,
182+
"Prompt to generate search query",
183+
[str(message) for message in query_messages],
184+
(
185+
{"model": self.chatgpt_model, "deployment": self.chatgpt_deployment}
186+
if self.chatgpt_deployment
187+
else {"model": self.chatgpt_model}
188+
),
184189
),
185190
ThoughtStep(
186-
"Generated search query",
191+
"Search using generated search query",
187192
query_text,
188-
{"use_semantic_captions": use_semantic_captions, "has_vector": has_vector},
193+
{
194+
"use_semantic_captions": use_semantic_captions,
195+
"use_semantic_ranker": use_semantic_ranker,
196+
"top": top,
197+
"filter": filter,
198+
"has_vector": has_vector,
199+
},
200+
),
201+
ThoughtStep(
202+
"Search results",
203+
[result.serialize_for_results() for result in results],
204+
),
205+
ThoughtStep(
206+
"Prompt to generate answer",
207+
[str(message) for message in messages],
208+
(
209+
{"model": self.chatgpt_model, "deployment": self.chatgpt_deployment}
210+
if self.chatgpt_deployment
211+
else {"model": self.chatgpt_model}
212+
),
189213
),
190-
ThoughtStep("Results", [result.serialize_for_results() for result in results]),
191-
ThoughtStep("Prompt", [str(message) for message in messages]),
192214
],
193215
}
194216

‎app/backend/approaches/chatreadretrievereadvision.py‎

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ async def run_until_final_call(
9898
# STEP 1: Generate an optimized keyword search query based on the chat history and the last question
9999
user_query_request = "Generate search query for: " + original_user_query
100100

101-
messages = self.get_messages_from_history(
101+
query_messages = self.get_messages_from_history(
102102
system_prompt=self.query_prompt_template,
103103
model_id=self.gpt4v_model,
104104
history=history,
@@ -109,7 +109,7 @@ async def run_until_final_call(
109109

110110
chat_completion: ChatCompletion = await self.openai_client.chat.completions.create(
111111
model=self.gpt4v_deployment if self.gpt4v_deployment else self.gpt4v_model,
112-
messages=messages,
112+
messages=query_messages,
113113
temperature=0.0, # Minimize creativity for search query generation
114114
max_tokens=100,
115115
n=1,
@@ -178,16 +178,38 @@ async def run_until_final_call(
178178
"data_points": data_points,
179179
"thoughts": [
180180
ThoughtStep(
181-
"Original user query",
182-
original_user_query,
181+
"Prompt to generate search query",
182+
[str(message) for message in query_messages],
183+
(
184+
{"model": self.gpt4v_model, "deployment": self.gpt4v_deployment}
185+
if self.gpt4v_deployment
186+
else {"model": self.gpt4v_model}
187+
),
183188
),
184189
ThoughtStep(
185-
"Generated search query",
190+
"Search using generated search query",
186191
query_text,
187-
{"use_semantic_captions": use_semantic_captions, "vector_fields": vector_fields},
192+
{
193+
"use_semantic_captions": use_semantic_captions,
194+
"use_semantic_ranker": use_semantic_ranker,
195+
"top": top,
196+
"filter": filter,
197+
"vector_fields": vector_fields,
198+
},
199+
),
200+
ThoughtStep(
201+
"Search results",
202+
[result.serialize_for_results() for result in results],
203+
),
204+
ThoughtStep(
205+
"Prompt to generate answer",
206+
[str(message) for message in messages],
207+
(
208+
{"model": self.gpt4v_model, "deployment": self.gpt4v_deployment}
209+
if self.gpt4v_deployment
210+
else {"model": self.gpt4v_model}
211+
),
188212
),
189-
ThoughtStep("Results", [result.serialize_for_results() for result in results]),
190-
ThoughtStep("Prompt", [str(message) for message in messages]),
191213
],
192214
}
193215

‎app/backend/approaches/retrievethenread.py‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -112,12 +112,12 @@ async def run(
112112
message_builder.insert_message("user", user_content)
113113
message_builder.insert_message("assistant", self.answer)
114114
message_builder.insert_message("user", self.question)
115-
115+
updated_messages = message_builder.messages
116116
chat_completion = (
117117
await self.openai_client.chat.completions.create(
118118
# Azure Open AI takes the deployment name as the model name
119119
model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model,
120-
messages=message_builder.messages,
120+
messages=updated_messages,
121121
temperature=overrides.get("temperature", 0.3),
122122
max_tokens=1024,
123123
n=1,
@@ -129,14 +129,29 @@ async def run(
129129
"data_points": data_points,
130130
"thoughts": [
131131
ThoughtStep(
132-
"Search Query",
132+
"Search using user query",
133133
query_text,
134134
{
135135
"use_semantic_captions": use_semantic_captions,
136+
"use_semantic_ranker": use_semantic_ranker,
137+
"top": top,
138+
"filter": filter,
139+
"has_vector": has_vector,
136140
},
137141
),
138-
ThoughtStep("Results", [result.serialize_for_results() for result in results]),
139-
ThoughtStep("Prompt", [str(message) for message in message_builder.messages]),
142+
ThoughtStep(
143+
"Search results",
144+
[result.serialize_for_results() for result in results],
145+
),
146+
ThoughtStep(
147+
"Prompt to generate answer",
148+
[str(message) for message in updated_messages],
149+
(
150+
{"model": self.chatgpt_model, "deployment": self.chatgpt_deployment}
151+
if self.chatgpt_deployment
152+
else {"model": self.chatgpt_model}
153+
),
154+
),
140155
],
141156
}
142157

‎app/backend/approaches/retrievethenreadvision.py‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -132,11 +132,11 @@ async def run(
132132

133133
# Append user message
134134
message_builder.insert_message("user", user_content)
135-
135+
updated_messages = message_builder.messages
136136
chat_completion = (
137137
await self.openai_client.chat.completions.create(
138138
model=self.gpt4v_deployment if self.gpt4v_deployment else self.gpt4v_model,
139-
messages=message_builder.messages,
139+
messages=updated_messages,
140140
temperature=overrides.get("temperature", 0.3),
141141
max_tokens=1024,
142142
n=1,
@@ -152,12 +152,29 @@ async def run(
152152
"data_points": data_points,
153153
"thoughts": [
154154
ThoughtStep(
155-
"Search Query",
155+
"Search using user query",
156156
query_text,
157-
{"use_semantic_captions": use_semantic_captions, "vector_fields": vector_fields},
157+
{
158+
"use_semantic_captions": use_semantic_captions,
159+
"use_semantic_ranker": use_semantic_ranker,
160+
"top": top,
161+
"filter": filter,
162+
"vector_fields": vector_fields,
163+
},
164+
),
165+
ThoughtStep(
166+
"Search results",
167+
[result.serialize_for_results() for result in results],
168+
),
169+
ThoughtStep(
170+
"Prompt to generate answer",
171+
[str(message) for message in updated_messages],
172+
(
173+
{"model": self.gpt4v_model, "deployment": self.gpt4v_deployment}
174+
if self.gpt4v_deployment
175+
else {"model": self.gpt4v_model}
176+
),
158177
),
159-
ThoughtStep("Results", [result.serialize_for_results() for result in results]),
160-
ThoughtStep("Prompt", [str(message) for message in message_builder.messages]),
161178
],
162179
}
163180
chat_completion["choices"][0]["context"] = extra_info

‎app/frontend/src/components/AnalysisPanel/AnalysisPanel.module.css‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@
4040
.tStep {
4141
color: #123bb6;
4242
position: relative;
43-
font-size: 12px;
43+
font-size: 14px;
44+
margin-bottom: 8px;
4445
}
4546

4647
.tCodeBlock {
@@ -53,6 +54,7 @@
5354
font-size: 12px;
5455
padding: 3px 10px;
5556
border-radius: 10px;
57+
margin-bottom: 8px;
5658
}
5759

5860
.citationImg {

‎app/frontend/src/components/AnalysisPanel/AnalysisPanel.tsx‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import { Stack, Pivot, PivotItem } from "@fluentui/react";
2-
import SyntaxHighlighter from "react-syntax-highlighter";
32

43
import styles from "./AnalysisPanel.module.css";
54

‎app/frontend/src/components/AnalysisPanel/ThoughtProcess.tsx‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,22 +16,20 @@ export const ThoughtProcess = ({ thoughts }: Props) => {
1616
return (
1717
<li className={styles.tListItem} key={ind}>
1818
<div className={styles.tStep}>{t.title}</div>
19+
<Stack horizontal tokens={{ childrenGap: 5 }}>
20+
{t.props &&
21+
(Object.keys(t.props) || []).map((k: any) => (
22+
<span className={styles.tProp}>
23+
{k}: {JSON.stringify(t.props?.[k])}
24+
</span>
25+
))}
26+
</Stack>
1927
{Array.isArray(t.description) ? (
2028
<SyntaxHighlighter language="json" wrapLongLines className={styles.tCodeBlock}>
2129
{JSON.stringify(t.description, null, 2)}
2230
</SyntaxHighlighter>
2331
) : (
24-
<>
25-
<div>{t.description}</div>
26-
<Stack horizontal tokens={{ childrenGap: 5 }}>
27-
{t.props &&
28-
(Object.keys(t.props) || []).map((k: any) => (
29-
<span className={styles.tProp}>
30-
{k}: {JSON.stringify(t.props?.[k])}
31-
</span>
32-
))}
33-
</Stack>
34-
</>
32+
<div>{t.description}</div>
3533
)}
3634
</li>
3735
);

‎tests/snapshots/test_app/test_ask_rtr_hybrid/client0/result.json‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,13 @@
1111
{
1212
"description": "What is the capital of France?",
1313
"props": {
14-
"use_semantic_captions": false
14+
"filter": null,
15+
"has_vector": true,
16+
"top": 3,
17+
"use_semantic_captions": false,
18+
"use_semantic_ranker": null
1519
},
16-
"title": "Search Query"
20+
"title": "Search using user query"
1721
},
1822
{
1923
"description": [
@@ -32,12 +36,14 @@
3236
"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2",
3337
"imageEmbedding": null,
3438
"oids": null,
39+
"reranker_score": 3.4577205181121826,
40+
"score": 0.03279569745063782,
3541
"sourcefile": "Benefit_Options.pdf",
3642
"sourcepage": "Benefit_Options-2.pdf"
3743
}
3844
],
3945
"props": null,
40-
"title": "Results"
46+
"title": "Search results"
4147
},
4248
{
4349
"description": [
@@ -46,8 +52,10 @@
4652
"{'role': 'assistant', 'content': 'In-network deductibles are $500 for employee and $1000 for family [info1.txt] and Overlake is in-network for the employee plan [info2.pdf][info4.pdf].'}",
4753
"{'role': 'user', 'content': 'What is the capital of France?\\nSources:\\n Benefit_Options-2.pdf: There is a whistleblower policy.'}"
4854
],
49-
"props": null,
50-
"title": "Prompt"
55+
"props": {
56+
"model": "gpt-35-turbo"
57+
},
58+
"title": "Prompt to generate answer"
5159
}
5260
]
5361
},

‎tests/snapshots/test_app/test_ask_rtr_hybrid/client1/result.json‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,13 @@
1111
{
1212
"description": "What is the capital of France?",
1313
"props": {
14-
"use_semantic_captions": false
14+
"filter": null,
15+
"has_vector": true,
16+
"top": 3,
17+
"use_semantic_captions": false,
18+
"use_semantic_ranker": null
1519
},
16-
"title": "Search Query"
20+
"title": "Search using user query"
1721
},
1822
{
1923
"description": [
@@ -32,12 +36,14 @@
3236
"id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2",
3337
"imageEmbedding": null,
3438
"oids": null,
39+
"reranker_score": 3.4577205181121826,
40+
"score": 0.03279569745063782,
3541
"sourcefile": "Benefit_Options.pdf",
3642
"sourcepage": "Benefit_Options-2.pdf"
3743
}
3844
],
3945
"props": null,
40-
"title": "Results"
46+
"title": "Search results"
4147
},
4248
{
4349
"description": [
@@ -46,8 +52,11 @@
4652
"{'role': 'assistant', 'content': 'In-network deductibles are $500 for employee and $1000 for family [info1.txt] and Overlake is in-network for the employee plan [info2.pdf][info4.pdf].'}",
4753
"{'role': 'user', 'content': 'What is the capital of France?\\nSources:\\n Benefit_Options-2.pdf: There is a whistleblower policy.'}"
4854
],
49-
"props": null,
50-
"title": "Prompt"
55+
"props": {
56+
"deployment": "test-chatgpt",
57+
"model": "gpt-35-turbo"
58+
},
59+
"title": "Prompt to generate answer"
5160
}
5261
]
5362
},

0 commit comments

Comments
 (0)