On Demand Azure Speech Generation

app/frontend/src/components/Answer/Answer.tsx

@@ -14,6 +14,12 @@ import { SpeechOutputAzure } from "./SpeechOutputAzure";
 
 interface Props {
     answer: ChatAppResponse;
+    index: number;
+    speechUrls: (string | null)[];
+    updateSpeechUrls: (urls: (string | null)[]) => void;
+    audio: HTMLAudioElement;
+    isPlaying: boolean;
+    setIsPlaying: (isPlaying: boolean) => void;
     isSelected?: boolean;
     isStreaming: boolean;
     onCitationClicked: (filePath: string) => void;
@@ -23,11 +29,16 @@ interface Props {
     showFollowupQuestions?: boolean;
     showSpeechOutputBrowser?: boolean;
     showSpeechOutputAzure?: boolean;
-    speechUrl: string | null;
 }
 
 export const Answer = ({
     answer,
+    index,
+    speechUrls,
+    updateSpeechUrls,
+    audio,
+    isPlaying,
+    setIsPlaying,
     isSelected,
     isStreaming,
     onCitationClicked,
@@ -36,13 +47,11 @@ export const Answer = ({
     onFollowupQuestionClicked,
     showFollowupQuestions,
     showSpeechOutputAzure,
-    showSpeechOutputBrowser,
-    speechUrl
+    showSpeechOutputBrowser
 }: Props) => {
     const followupQuestions = answer.context?.followup_questions;
     const messageContent = answer.message.content;
     const parsedAnswer = useMemo(() => parseAnswerToHtml(messageContent, isStreaming, onCitationClicked), [answer]);
-
     const sanitizedAnswerHtml = DOMPurify.sanitize(parsedAnswer.answerHtml);
 
     return (
@@ -67,7 +76,18 @@ export const Answer = ({
                             onClick={() => onSupportingContentClicked()}
                             disabled={!answer.context.data_points}
                         />
-                        {showSpeechOutputAzure && <SpeechOutputAzure url={speechUrl} />}
+                        {showSpeechOutputAzure && (
+                            <SpeechOutputAzure
+                                answer={sanitizedAnswerHtml}
+                                urls={speechUrls}
+                                index={index}
+                                updateSpeechUrls={updateSpeechUrls}
+                                audio={audio}
+                                isPlaying={isPlaying}
+                                setIsPlaying={setIsPlaying}
+                                isStreaming={isStreaming}
+                            />
+                        )}
                         {showSpeechOutputBrowser && <SpeechOutputBrowser answer={sanitizedAnswerHtml} />}
                     </div>
                 </Stack>

app/frontend/src/components/Answer/SpeechOutputAzure.tsx

@@ -1,44 +1,76 @@
 import { useState } from "react";
 
 import { IconButton } from "@fluentui/react";
+import { getSpeechApi } from "../../api";
 
 interface Props {
-    url: string | null;
+    answer: string;
+    urls: (string | null)[];
+    updateSpeechUrls: (urls: (string | null)[]) => void;
+    index: number;
+    audio: HTMLAudioElement;
+    isPlaying: boolean;
+    setIsPlaying: (isPlaying: boolean) => void;
+    isStreaming: boolean;
 }
 
-let audio = new Audio();
+export const SpeechOutputAzure = ({ answer, urls, updateSpeechUrls, index, audio, isPlaying, setIsPlaying, isStreaming }: Props) => {
+    const [isLoading, setIsLoading] = useState(false);
+    const [localPlayingState, setLocalPlayingState] = useState(false);
 
-export const SpeechOutputAzure = ({ url }: Props) => {
-    const [isPlaying, setIsPlaying] = useState(false);
+    const playAudio = async (url: string) => {
+        audio.src = url;
+        await audio
+            .play()
+            .then(() => {
+                audio.onended = () => setIsPlaying(false);
+                setIsPlaying(true);
+                setLocalPlayingState(true);
+            })
+            .catch(() => {
+                alert("Failed to play speech output.");
+                console.error("Failed to play speech output.");
+                setIsPlaying(false);
+                setLocalPlayingState(false);
+            });
+    };
 
-    const startOrStopAudio = async () => {
+    const startOrStopSpeech = async (answer: string) => {
         if (isPlaying) {
             audio.pause();
+            audio.currentTime = 0;
             setIsPlaying(false);
+            setLocalPlayingState(false);
             return;
         }
-
-        if (!url) {
-            console.error("Speech output is not yet available.");
+        if (urls[index]) {
+            playAudio(urls[index]);
             return;
         }
-        audio = new Audio(url);
-        await audio.play();
-        audio.addEventListener("ended", () => {
-            setIsPlaying(false);
+        setIsLoading(true);
+        await getSpeechApi(answer).then(async speechUrl => {
+            if (!speechUrl) {
+                alert("Speech output is not available.");
+                console.error("Speech output is not available.");
+                return;
+            }
+            setIsLoading(false);
+            updateSpeechUrls(urls.map((url, i) => (i === index ? speechUrl : url)));
+            playAudio(speechUrl);
         });
-        setIsPlaying(true);
     };
 
-    const color = isPlaying ? "red" : "black";
-    return (
+    const color = localPlayingState ? "red" : "black";
+    return isLoading ? (
+        <IconButton style={{ color: color }} iconProps={{ iconName: "Sync" }} title="Loading" ariaLabel="Loading answer" disabled={true} />
+    ) : (
         <IconButton
             style={{ color: color }}
             iconProps={{ iconName: "Volume3" }}
             title="Speak answer"
             ariaLabel="Speak answer"
-            onClick={() => startOrStopAudio()}
-            disabled={!url}
+            onClick={() => startOrStopSpeech(answer)}
+            disabled={isStreaming}
         />
     );
 };

app/frontend/src/pages/ask/Ask.tsx

@@ -48,13 +48,15 @@ export function Component(): JSX.Element {
     const [showSpeechInput, setShowSpeechInput] = useState<boolean>(false);
     const [showSpeechOutputBrowser, setShowSpeechOutputBrowser] = useState<boolean>(false);
     const [showSpeechOutputAzure, setShowSpeechOutputAzure] = useState<boolean>(false);
+    const audio = useRef(new Audio()).current;
+    const [isPlaying, setIsPlaying] = useState(false);
 
     const lastQuestionRef = useRef<string>("");
 
     const [isLoading, setIsLoading] = useState<boolean>(false);
     const [error, setError] = useState<unknown>();
     const [answer, setAnswer] = useState<ChatAppResponse>();
-    const [speechUrl, setSpeechUrl] = useState<string | null>(null);
+    const [speechUrl, setSpeechUrl] = useState<(string | null)[]>([]);
 
     const [activeCitation, setActiveCitation] = useState<string>();
     const [activeAnalysisPanelTab, setActiveAnalysisPanelTab] = useState<AnalysisPanelTabs | undefined>(undefined);
@@ -82,14 +84,6 @@ export function Component(): JSX.Element {
         getConfig();
     }, []);
 
-    useEffect(() => {
-        if (answer && showSpeechOutputAzure) {
-            getSpeechApi(answer.message.content).then(speechUrl => {
-                setSpeechUrl(speechUrl);
-            });
-        }
-    }, [answer]);
-
     const makeApiRequest = async (question: string) => {
         lastQuestionRef.current = question;
 
@@ -134,7 +128,7 @@ export function Component(): JSX.Element {
             };
             const result = await askApi(request, token);
             setAnswer(result);
-            setSpeechUrl(null);
+            setSpeechUrl([null]);
         } catch (e) {
             setError(e);
         } finally {
@@ -256,13 +250,18 @@ export function Component(): JSX.Element {
                     <div className={styles.askAnswerContainer}>
                         <Answer
                             answer={answer}
+                            index={0}
+                            speechUrls={speechUrl}
+                            updateSpeechUrls={setSpeechUrl}
+                            audio={audio}
+                            isPlaying={isPlaying}
+                            setIsPlaying={setIsPlaying}
                             isStreaming={false}
                             onCitationClicked={x => onShowCitation(x)}
                             onThoughtProcessClicked={() => onToggleTab(AnalysisPanelTabs.ThoughtProcessTab)}
                             onSupportingContentClicked={() => onToggleTab(AnalysisPanelTabs.SupportingContentTab)}
                             showSpeechOutputAzure={showSpeechOutputAzure}
                             showSpeechOutputBrowser={showSpeechOutputBrowser}
-                            speechUrl={speechUrl}
                         />
                     </div>
                 )}

app/frontend/src/pages/chat/Chat.tsx

@@ -9,7 +9,6 @@ import styles from "./Chat.module.css";
 import {
     chatApi,
     configApi,
-    getSpeechApi,
     RetrievalMode,
     ChatAppResponse,
     ChatAppResponseOrError,
@@ -67,8 +66,8 @@ const Chat = () => {
 
     const [selectedAnswer, setSelectedAnswer] = useState<number>(0);
     const [answers, setAnswers] = useState<[user: string, response: ChatAppResponse][]>([]);
-    const [streamedAnswers, setStreamedAnswers] = useState<[user: string, response: ChatAppResponse][]>([]);
     const [speechUrls, setSpeechUrls] = useState<(string | null)[]>([]);
+    const [streamedAnswers, setStreamedAnswers] = useState<[user: string, response: ChatAppResponse][]>([]);
 
     const [showGPT4VOptions, setShowGPT4VOptions] = useState<boolean>(false);
     const [showSemanticRankerOption, setShowSemanticRankerOption] = useState<boolean>(false);
@@ -77,6 +76,8 @@ const Chat = () => {
     const [showSpeechInput, setShowSpeechInput] = useState<boolean>(false);
     const [showSpeechOutputBrowser, setShowSpeechOutputBrowser] = useState<boolean>(false);
     const [showSpeechOutputAzure, setShowSpeechOutputAzure] = useState<boolean>(false);
+    const audio = useRef(new Audio()).current;
+    const [isPlaying, setIsPlaying] = useState(false);
 
     const getConfig = async () => {
         configApi().then(config => {
@@ -199,6 +200,7 @@ const Chat = () => {
                 }
                 setAnswers([...answers, [question, parsedResponse as ChatAppResponse]]);
             }
+            setSpeechUrls([...speechUrls, null]);
         } catch (e) {
             setError(e);
         } finally {
@@ -212,6 +214,7 @@ const Chat = () => {
         setActiveCitation(undefined);
         setActiveAnalysisPanelTab(undefined);
         setAnswers([]);
+        setSpeechUrls([]);
         setStreamedAnswers([]);
         setIsLoading(false);
         setIsStreaming(false);
@@ -223,19 +226,6 @@ const Chat = () => {
         getConfig();
     }, []);
 
-    useEffect(() => {
-        if (answers && showSpeechOutputAzure) {
-            // For each answer that is missing a speech URL, fetch the speech URL
-            for (let i = 0; i < answers.length; i++) {
-                if (!speechUrls[i]) {
-                    getSpeechApi(answers[i][1].message.content).then(speechUrl => {
-                        setSpeechUrls([...speechUrls.slice(0, i), speechUrl, ...speechUrls.slice(i + 1)]);
-                    });
-                }
-            }
-        }
-    }, [answers]);
-
     const onPromptTemplateChange = (_ev?: React.FormEvent<HTMLInputElement | HTMLTextAreaElement>, newValue?: string) => {
         setPromptTemplate(newValue || "");
     };
@@ -368,6 +358,12 @@ const Chat = () => {
                                                 isStreaming={true}
                                                 key={index}
                                                 answer={streamedAnswer[1]}
+                                                index={index}
+                                                speechUrls={speechUrls}
+                                                updateSpeechUrls={setSpeechUrls}
+                                                audio={audio}
+                                                isPlaying={isPlaying}
+                                                setIsPlaying={setIsPlaying}
                                                 isSelected={false}
                                                 onCitationClicked={c => onShowCitation(c, index)}
                                                 onThoughtProcessClicked={() => onToggleTab(AnalysisPanelTabs.ThoughtProcessTab, index)}
@@ -376,7 +372,6 @@ const Chat = () => {
                                                 showFollowupQuestions={useSuggestFollowupQuestions && answers.length - 1 === index}
                                                 showSpeechOutputAzure={showSpeechOutputAzure}
                                                 showSpeechOutputBrowser={showSpeechOutputBrowser}
-                                                speechUrl={speechUrls[index]}
                                             />
                                         </div>
                                     </div>
@@ -390,6 +385,12 @@ const Chat = () => {
                                                 isStreaming={false}
                                                 key={index}
                                                 answer={answer[1]}
+                                                index={index}
+                                                speechUrls={speechUrls}
+                                                audio={audio}
+                                                updateSpeechUrls={setSpeechUrls}
+                                                isPlaying={isPlaying}
+                                                setIsPlaying={setIsPlaying}
                                                 isSelected={selectedAnswer === index && activeAnalysisPanelTab !== undefined}
                                                 onCitationClicked={c => onShowCitation(c, index)}
                                                 onThoughtProcessClicked={() => onToggleTab(AnalysisPanelTabs.ThoughtProcessTab, index)}
@@ -398,7 +399,6 @@ const Chat = () => {
                                                 showFollowupQuestions={useSuggestFollowupQuestions && answers.length - 1 === index}
                                                 showSpeechOutputAzure={showSpeechOutputAzure}
                                                 showSpeechOutputBrowser={showSpeechOutputBrowser}
-                                                speechUrl={speechUrls[index]}
                                             />
                                         </div>
                                     </div>