Skip to content

Commit dd7c1d2

Browse files
pamelafoxpamelafox
authored
Upgrade to latest version of AI Chat Protocol (#1682)
* GPt4 version * Port to chatprotocol v2 * Update approaches and tests * Removing finishReason logprobs index * Get frontend working * Increase test coverage * Update speech code * Fork run and run_stream * Types * Update e2e tests
1 parent c873b49 commit dd7c1d2

File tree

65 files changed

+3011
-3379
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+3011
-3379
lines changed

‎app/backend/app.py‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -215,17 +215,39 @@ async def chat(auth_claims: Dict[str, Any]):
215215

216216
result = await approach.run(
217217
request_json["messages"],
218-
stream=request_json.get("stream", False),
219218
context=context,
220219
session_state=request_json.get("session_state"),
221220
)
222-
if isinstance(result, dict):
223-
return jsonify(result)
221+
return jsonify(result)
222+
except Exception as error:
223+
return error_response(error, "/chat")
224+
225+
226+
@bp.route("/chat/stream", methods=["POST"])
227+
@authenticated
228+
async def chat_stream(auth_claims: Dict[str, Any]):
229+
if not request.is_json:
230+
return jsonify({"error": "request must be json"}), 415
231+
request_json = await request.get_json()
232+
context = request_json.get("context", {})
233+
context["auth_claims"] = auth_claims
234+
try:
235+
use_gpt4v = context.get("overrides", {}).get("use_gpt4v", False)
236+
approach: Approach
237+
if use_gpt4v and CONFIG_CHAT_VISION_APPROACH in current_app.config:
238+
approach = cast(Approach, current_app.config[CONFIG_CHAT_VISION_APPROACH])
224239
else:
225-
response = await make_response(format_as_ndjson(result))
226-
response.timeout = None # type: ignore
227-
response.mimetype = "application/json-lines"
228-
return response
240+
approach = cast(Approach, current_app.config[CONFIG_CHAT_APPROACH])
241+
242+
result = await approach.run_stream(
243+
request_json["messages"],
244+
context=context,
245+
session_state=request_json.get("session_state"),
246+
)
247+
response = await make_response(format_as_ndjson(result))
248+
response.timeout = None # type: ignore
249+
response.mimetype = "application/json-lines"
250+
return response
229251
except Exception as error:
230252
return error_response(error, "/chat")
231253

‎app/backend/approaches/approach.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
List,
1010
Optional,
1111
TypedDict,
12-
Union,
1312
cast,
1413
)
1514
from urllib.parse import urljoin
@@ -257,8 +256,15 @@ async def compute_image_embedding(self, q: str):
257256
async def run(
258257
self,
259258
messages: list[ChatCompletionMessageParam],
260-
stream: bool = False,
261259
session_state: Any = None,
262260
context: dict[str, Any] = {},
263-
) -> Union[dict[str, Any], AsyncGenerator[dict[str, Any], None]]:
261+
) -> dict[str, Any]:
262+
raise NotImplementedError
263+
264+
async def run_stream(
265+
self,
266+
messages: list[ChatCompletionMessageParam],
267+
session_state: Any = None,
268+
context: dict[str, Any] = {},
269+
) -> AsyncGenerator[dict[str, Any], None]:
264270
raise NotImplementedError

‎app/backend/approaches/chatapproach.py‎

Lines changed: 25 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import json
22
import re
33
from abc import ABC, abstractmethod
4-
from typing import Any, AsyncGenerator, Optional, Union
4+
from typing import Any, AsyncGenerator, Optional
55

66
from openai.types.chat import ChatCompletion, ChatCompletionMessageParam
77

@@ -90,12 +90,13 @@ async def run_without_streaming(
9090
)
9191
chat_completion_response: ChatCompletion = await chat_coroutine
9292
chat_resp = chat_completion_response.model_dump() # Convert to dict to make it JSON serializable
93-
chat_resp["choices"][0]["context"] = extra_info
93+
chat_resp = chat_resp["choices"][0]
94+
chat_resp["context"] = extra_info
9495
if overrides.get("suggest_followup_questions"):
95-
content, followup_questions = self.extract_followup_questions(chat_resp["choices"][0]["message"]["content"])
96-
chat_resp["choices"][0]["message"]["content"] = content
97-
chat_resp["choices"][0]["context"]["followup_questions"] = followup_questions
98-
chat_resp["choices"][0]["session_state"] = session_state
96+
content, followup_questions = self.extract_followup_questions(chat_resp["message"]["content"])
97+
chat_resp["message"]["content"] = content
98+
chat_resp["context"]["followup_questions"] = followup_questions
99+
chat_resp["session_state"] = session_state
99100
return chat_resp
100101

101102
async def run_with_streaming(
@@ -108,64 +109,49 @@ async def run_with_streaming(
108109
extra_info, chat_coroutine = await self.run_until_final_call(
109110
messages, overrides, auth_claims, should_stream=True
110111
)
111-
yield {
112-
"choices": [
113-
{
114-
"delta": {"role": "assistant"},
115-
"context": extra_info,
116-
"session_state": session_state,
117-
"finish_reason": None,
118-
"index": 0,
119-
}
120-
],
121-
"object": "chat.completion.chunk",
122-
}
112+
yield {"delta": {"role": "assistant"}, "context": extra_info, "session_state": session_state}
123113

124114
followup_questions_started = False
125115
followup_content = ""
126116
async for event_chunk in await chat_coroutine:
127117
# "2023-07-01-preview" API version has a bug where first response has empty choices
128118
event = event_chunk.model_dump() # Convert pydantic model to dict
129119
if event["choices"]:
120+
completion = {"delta": event["choices"][0]["delta"]}
130121
# if event contains << and not >>, it is start of follow-up question, truncate
131-
content = event["choices"][0]["delta"].get("content")
122+
content = completion["delta"].get("content")
132123
content = content or "" # content may either not exist in delta, or explicitly be None
133124
if overrides.get("suggest_followup_questions") and "<<" in content:
134125
followup_questions_started = True
135126
earlier_content = content[: content.index("<<")]
136127
if earlier_content:
137-
event["choices"][0]["delta"]["content"] = earlier_content
138-
yield event
128+
completion["delta"]["content"] = earlier_content
129+
yield completion
139130
followup_content += content[content.index("<<") :]
140131
elif followup_questions_started:
141132
followup_content += content
142133
else:
143-
yield event
134+
yield completion
144135
if followup_content:
145136
_, followup_questions = self.extract_followup_questions(followup_content)
146-
yield {
147-
"choices": [
148-
{
149-
"delta": {"role": "assistant"},
150-
"context": {"followup_questions": followup_questions},
151-
"finish_reason": None,
152-
"index": 0,
153-
}
154-
],
155-
"object": "chat.completion.chunk",
156-
}
137+
yield {"delta": {"role": "assistant"}, "context": {"followup_questions": followup_questions}}
157138

158139
async def run(
159140
self,
160141
messages: list[ChatCompletionMessageParam],
161-
stream: bool = False,
162142
session_state: Any = None,
163143
context: dict[str, Any] = {},
164-
) -> Union[dict[str, Any], AsyncGenerator[dict[str, Any], None]]:
144+
) -> dict[str, Any]:
165145
overrides = context.get("overrides", {})
166146
auth_claims = context.get("auth_claims", {})
147+
return await self.run_without_streaming(messages, overrides, auth_claims, session_state)
167148

168-
if stream is False:
169-
return await self.run_without_streaming(messages, overrides, auth_claims, session_state)
170-
else:
171-
return self.run_with_streaming(messages, overrides, auth_claims, session_state)
149+
async def run_stream(
150+
self,
151+
messages: list[ChatCompletionMessageParam],
152+
session_state: Any = None,
153+
context: dict[str, Any] = {},
154+
) -> AsyncGenerator[dict[str, Any], None]:
155+
overrides = context.get("overrides", {})
156+
auth_claims = context.get("auth_claims", {})
157+
return self.run_with_streaming(messages, overrides, auth_claims, session_state)

‎app/backend/approaches/retrievethenread.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from typing import Any, AsyncGenerator, Optional, Union
1+
from typing import Any, Optional
22

33
from azure.search.documents.aio import SearchClient
44
from azure.search.documents.models import VectorQuery
@@ -72,10 +72,9 @@ def __init__(
7272
async def run(
7373
self,
7474
messages: list[ChatCompletionMessageParam],
75-
stream: bool = False, # Stream is not used in this approach
7675
session_state: Any = None,
7776
context: dict[str, Any] = {},
78-
) -> Union[dict[str, Any], AsyncGenerator[dict[str, Any], None]]:
77+
) -> dict[str, Any]:
7978
q = messages[-1]["content"]
8079
if not isinstance(q, str):
8180
raise ValueError("The most recent message content must be a string.")
@@ -167,6 +166,8 @@ async def run(
167166
],
168167
}
169168

170-
chat_completion["choices"][0]["context"] = extra_info
171-
chat_completion["choices"][0]["session_state"] = session_state
172-
return chat_completion
169+
completion = {}
170+
completion["message"] = chat_completion["choices"][0]["message"]
171+
completion["context"] = extra_info
172+
completion["session_state"] = session_state
173+
return completion

‎app/backend/approaches/retrievethenreadvision.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from typing import Any, AsyncGenerator, Awaitable, Callable, Optional, Union
1+
from typing import Any, Awaitable, Callable, Optional
22

33
from azure.search.documents.aio import SearchClient
44
from azure.storage.blob.aio import ContainerClient
@@ -72,10 +72,9 @@ def __init__(
7272
async def run(
7373
self,
7474
messages: list[ChatCompletionMessageParam],
75-
stream: bool = False, # Stream is not used in this approach
7675
session_state: Any = None,
7776
context: dict[str, Any] = {},
78-
) -> Union[dict[str, Any], AsyncGenerator[dict[str, Any], None]]:
77+
) -> dict[str, Any]:
7978
q = messages[-1]["content"]
8079
if not isinstance(q, str):
8180
raise ValueError("The most recent message content must be a string.")
@@ -189,6 +188,9 @@ async def run(
189188
),
190189
],
191190
}
192-
chat_completion["choices"][0]["context"] = extra_info
193-
chat_completion["choices"][0]["session_state"] = session_state
194-
return chat_completion
191+
192+
completion = {}
193+
completion["message"] = chat_completion["choices"][0]["message"]
194+
completion["context"] = extra_info
195+
completion["session_state"] = session_state
196+
return completion

‎app/frontend/src/api/api.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,12 @@ export async function askApi(request: ChatAppRequest, idToken: string | undefine
3737
return parsedResponse as ChatAppResponse;
3838
}
3939

40-
export async function chatApi(request: ChatAppRequest, idToken: string | undefined): Promise<Response> {
41-
return await fetch(`${BACKEND_URI}/chat`, {
40+
export async function chatApi(request: ChatAppRequest, shouldStream: boolean, idToken: string | undefined): Promise<Response> {
41+
let url = `${BACKEND_URI}/chat`;
42+
if (shouldStream) {
43+
url += "/stream";
44+
}
45+
return await fetch(url, {
4246
method: "POST",
4347
headers: { ...getHeaders(idToken), "Content-Type": "application/json" },
4448
body: JSON.stringify(request)

‎app/frontend/src/api/models.ts‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -53,20 +53,19 @@ export type ResponseContext = {
5353
thoughts: Thoughts[];
5454
};
5555

56-
export type ResponseChoice = {
57-
index: number;
56+
export type ChatAppResponseOrError = {
5857
message: ResponseMessage;
58+
delta: ResponseMessage;
5959
context: ResponseContext;
6060
session_state: any;
61-
};
62-
63-
export type ChatAppResponseOrError = {
64-
choices?: ResponseChoice[];
6561
error?: string;
6662
};
6763

6864
export type ChatAppResponse = {
69-
choices: ResponseChoice[];
65+
message: ResponseMessage;
66+
delta: ResponseMessage;
67+
context: ResponseContext;
68+
session_state: any;
7069
};
7170

7271
export type ChatAppRequestContext = {
@@ -76,7 +75,6 @@ export type ChatAppRequestContext = {
7675
export type ChatAppRequest = {
7776
messages: ResponseMessage[];
7877
context?: ChatAppRequestContext;
79-
stream?: boolean;
8078
session_state: any;
8179
};
8280

‎app/frontend/src/components/AnalysisPanel/AnalysisPanel.tsx‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ interface Props {
2424
const pivotItemDisabledStyle = { disabled: true, style: { color: "grey" } };
2525

2626
export const AnalysisPanel = ({ answer, activeTab, activeCitation, citationHeight, className, onActiveTabChanged }: Props) => {
27-
const isDisabledThoughtProcessTab: boolean = !answer.choices[0].context.thoughts;
28-
const isDisabledSupportingContentTab: boolean = !answer.choices[0].context.data_points;
27+
const isDisabledThoughtProcessTab: boolean = !answer.context.thoughts;
28+
const isDisabledSupportingContentTab: boolean = !answer.context.data_points;
2929
const isDisabledCitationTab: boolean = !activeCitation;
3030
const [citation, setCitation] = useState("");
3131

@@ -81,14 +81,14 @@ export const AnalysisPanel = ({ answer, activeTab, activeCitation, citationHeigh
8181
headerText="Thought process"
8282
headerButtonProps={isDisabledThoughtProcessTab ? pivotItemDisabledStyle : undefined}
8383
>
84-
<ThoughtProcess thoughts={answer.choices[0].context.thoughts || []} />
84+
<ThoughtProcess thoughts={answer.context.thoughts || []} />
8585
</PivotItem>
8686
<PivotItem
8787
itemKey={AnalysisPanelTabs.SupportingContentTab}
8888
headerText="Supporting content"
8989
headerButtonProps={isDisabledSupportingContentTab ? pivotItemDisabledStyle : undefined}
9090
>
91-
<SupportingContent supportingContent={answer.choices[0].context.data_points} />
91+
<SupportingContent supportingContent={answer.context.data_points} />
9292
</PivotItem>
9393
<PivotItem
9494
itemKey={AnalysisPanelTabs.CitationTab}

‎app/frontend/src/components/Answer/Answer.tsx‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ export const Answer = ({
3636
showSpeechOutputBrowser,
3737
speechUrl
3838
}: Props) => {
39-
const followupQuestions = answer.choices[0].context.followup_questions;
40-
const messageContent = answer.choices[0].message.content;
39+
const followupQuestions = answer.context?.followup_questions;
40+
const messageContent = answer.message.content;
4141
const parsedAnswer = useMemo(() => parseAnswerToHtml(messageContent, isStreaming, onCitationClicked), [answer]);
4242

4343
const sanitizedAnswerHtml = DOMPurify.sanitize(parsedAnswer.answerHtml);
@@ -54,15 +54,15 @@ export const Answer = ({
5454
title="Show thought process"
5555
ariaLabel="Show thought process"
5656
onClick={() => onThoughtProcessClicked()}
57-
disabled={!answer.choices[0].context.thoughts?.length}
57+
disabled={!answer.context.thoughts?.length}
5858
/>
5959
<IconButton
6060
style={{ color: "black" }}
6161
iconProps={{ iconName: "ClipboardList" }}
6262
title="Show supporting content"
6363
ariaLabel="Show supporting content"
6464
onClick={() => onSupportingContentClicked()}
65-
disabled={!answer.choices[0].context.data_points}
65+
disabled={!answer.context.data_points}
6666
/>
6767
{showSpeechOutputAzure && <SpeechOutputAzure url={speechUrl} />}
6868
{showSpeechOutputBrowser && <SpeechOutputBrowser answer={sanitizedAnswerHtml} />}

‎app/frontend/src/pages/ask/Ask.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ export function Component(): JSX.Element {
8181

8282
useEffect(() => {
8383
if (answer && showSpeechOutputAzure) {
84-
getSpeechApi(answer.choices[0].message.content).then(speechUrl => {
84+
getSpeechApi(answer.message.content).then(speechUrl => {
8585
setSpeechUrl(speechUrl);
8686
});
8787
}
@@ -126,7 +126,7 @@ export function Component(): JSX.Element {
126126
}
127127
},
128128
// ChatAppProtocol: Client must pass on any session state received from the server
129-
session_state: answer ? answer.choices[0].session_state : null
129+
session_state: answer ? answer.session_state : null
130130
};
131131
const result = await askApi(request, token);
132132
setAnswer(result);

0 commit comments

Comments
 (0)