feat: add low cost browser text to speech output using WebSpeechAPI (#1671)

* feat: add low cost browser text to speech output using WebSpeechAPI

This PR adds the optional feature to enable the built into browsers text to speech optional feature by setting the  boolean.

* improve and refactor code

* fix typo and remove unnessesary check

* add docs

* fix note

* fix ruff

Author: john0isaac

.azdo/pipelines/azure-dev.yml

@@ -87,6 +87,7 @@ steps:
       AZURE_COMPUTER_VISION_LOCATION: $(AZURE_COMPUTER_VISION_LOCATION)
       AZURE_COMPUTER_VISION_SKU: $(AZURE_COMPUTER_VISION_SKU)
       USE_SPEECH_INPUT_BROWSER: $(USE_SPEECH_INPUT_BROWSER)
+      USE_SPEECH_OUTPUT_BROWSER: $(USE_SPEECH_OUTPUT_BROWSER)
       USE_SPEECH_OUTPUT_AZURE: $(USE_SPEECH_OUTPUT_AZURE)
       AZURE_SPEECH_SERVICE: $(AZURE_SPEECH_SERVICE)
       AZURE_SPEECH_SERVICE_RESOURCE_GROUP: $(AZURE_SPEECH_SERVICE_RESOURCE_GROUP)

.github/workflows/azure-dev.yml

@@ -74,6 +74,7 @@ jobs:
       AZURE_VISION_ENDPOINT: ${{ vars.AZURE_VISION_ENDPOINT }}
       VISION_SECRET_NAME: ${{ vars.VISION_SECRET_NAME }}
       USE_SPEECH_INPUT_BROWSER: ${{ vars.USE_SPEECH_INPUT_BROWSER }}
+      USE_SPEECH_OUTPUT_BROWSER: ${{ vars.USE_SPEECH_OUTPUT_BROWSER }}
       USE_SPEECH_OUTPUT_AZURE: ${{ vars.USE_SPEECH_OUTPUT_AZURE }}
       AZURE_SPEECH_SERVICE: ${{ vars.AZURE_SPEECH_SERVICE }}
       AZURE_SPEECH_SERVICE_RESOURCE_GROUP: ${{ vars.AZURE_SPEECH_RESOURCE_GROUP }}

app/backend/app.py

@@ -63,7 +63,8 @@
     CONFIG_SEARCH_CLIENT,
     CONFIG_SEMANTIC_RANKER_DEPLOYED,
     CONFIG_SPEECH_INPUT_ENABLED,
-    CONFIG_SPEECH_OUTPUT_ENABLED,
+    CONFIG_SPEECH_OUTPUT_AZURE_ENABLED,
+    CONFIG_SPEECH_OUTPUT_BROWSER_ENABLED,
     CONFIG_SPEECH_SERVICE_ID,
     CONFIG_SPEECH_SERVICE_LOCATION,
     CONFIG_SPEECH_SERVICE_TOKEN,
@@ -245,7 +246,8 @@ def config():
             "showVectorOption": current_app.config[CONFIG_VECTOR_SEARCH_ENABLED],
             "showUserUpload": current_app.config[CONFIG_USER_UPLOAD_ENABLED],
             "showSpeechInput": current_app.config[CONFIG_SPEECH_INPUT_ENABLED],
-            "showSpeechOutput": current_app.config[CONFIG_SPEECH_OUTPUT_ENABLED],
+            "showSpeechOutputBrowser": current_app.config[CONFIG_SPEECH_OUTPUT_BROWSER_ENABLED],
+            "showSpeechOutputAzure": current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED],
         }
     )
 
@@ -405,6 +407,7 @@ async def setup_clients():
     USE_GPT4V = os.getenv("USE_GPT4V", "").lower() == "true"
     USE_USER_UPLOAD = os.getenv("USE_USER_UPLOAD", "").lower() == "true"
     USE_SPEECH_INPUT_BROWSER = os.getenv("USE_SPEECH_INPUT_BROWSER", "").lower() == "true"
+    USE_SPEECH_OUTPUT_BROWSER = os.getenv("USE_SPEECH_OUTPUT_BROWSER", "").lower() == "true"
     USE_SPEECH_OUTPUT_AZURE = os.getenv("USE_SPEECH_OUTPUT_AZURE", "").lower() == "true"
 
     # Use the current user identity to authenticate with Azure OpenAI, AI Search and Blob Storage (no secrets needed,
@@ -536,7 +539,8 @@ async def setup_clients():
     current_app.config[CONFIG_VECTOR_SEARCH_ENABLED] = os.getenv("USE_VECTORS", "").lower() != "false"
     current_app.config[CONFIG_USER_UPLOAD_ENABLED] = bool(USE_USER_UPLOAD)
     current_app.config[CONFIG_SPEECH_INPUT_ENABLED] = USE_SPEECH_INPUT_BROWSER
-    current_app.config[CONFIG_SPEECH_OUTPUT_ENABLED] = USE_SPEECH_OUTPUT_AZURE
+    current_app.config[CONFIG_SPEECH_OUTPUT_BROWSER_ENABLED] = USE_SPEECH_OUTPUT_BROWSER
+    current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED] = USE_SPEECH_OUTPUT_AZURE
 
     # Various approaches to integrate GPT and external knowledge, most applications will use a single one of these patterns
     # or some derivative, here we include several for exploration purposes

app/backend/config.py

@@ -15,7 +15,8 @@
 CONFIG_OPENAI_CLIENT = "openai_client"
 CONFIG_INGESTER = "ingester"
 CONFIG_SPEECH_INPUT_ENABLED = "speech_input_enabled"
-CONFIG_SPEECH_OUTPUT_ENABLED = "speech_output_enabled"
+CONFIG_SPEECH_OUTPUT_BROWSER_ENABLED = "speech_output_browser_enabled"
+CONFIG_SPEECH_OUTPUT_AZURE_ENABLED = "speech_output_azure_enabled"
 CONFIG_SPEECH_SERVICE_ID = "speech_service_id"
 CONFIG_SPEECH_SERVICE_LOCATION = "speech_service_location"
 CONFIG_SPEECH_SERVICE_TOKEN = "speech_service_token"

app/frontend/src/api/models.ts

@@ -86,7 +86,8 @@ export type Config = {
     showVectorOption: boolean;
     showUserUpload: boolean;
     showSpeechInput: boolean;
-    showSpeechOutput: boolean;
+    showSpeechOutputBrowser: boolean;
+    showSpeechOutputAzure: boolean;
 };
 
 export type SimpleAPIResponse = {

app/frontend/src/components/Answer/Answer.tsx

@@ -6,7 +6,8 @@ import styles from "./Answer.module.css";
 import { ChatAppResponse, getCitationFilePath } from "../../api";
 import { parseAnswerToHtml } from "./AnswerParser";
 import { AnswerIcon } from "./AnswerIcon";
-import { SpeechOutput } from "./SpeechOutput";
+import { SpeechOutputBrowser } from "./SpeechOutputBrowser";
+import { SpeechOutputAzure } from "./SpeechOutputAzure";
 
 interface Props {
     answer: ChatAppResponse;
@@ -17,7 +18,8 @@ interface Props {
     onSupportingContentClicked: () => void;
     onFollowupQuestionClicked?: (question: string) => void;
     showFollowupQuestions?: boolean;
-    showSpeechOutput?: boolean;
+    showSpeechOutputBrowser?: boolean;
+    showSpeechOutputAzure?: boolean;
     speechUrl: string | null;
 }
 
@@ -30,7 +32,8 @@ export const Answer = ({
     onSupportingContentClicked,
     onFollowupQuestionClicked,
     showFollowupQuestions,
-    showSpeechOutput,
+    showSpeechOutputAzure,
+    showSpeechOutputBrowser,
     speechUrl
 }: Props) => {
     const followupQuestions = answer.choices[0].context.followup_questions;
@@ -61,7 +64,8 @@ export const Answer = ({
                             onClick={() => onSupportingContentClicked()}
                             disabled={!answer.choices[0].context.data_points}
                         />
-                        {showSpeechOutput && <SpeechOutput url={speechUrl} />}
+                        {showSpeechOutputAzure && <SpeechOutputAzure url={speechUrl} />}
+                        {showSpeechOutputBrowser && <SpeechOutputBrowser answer={sanitizedAnswerHtml} />}
                     </div>
                 </Stack>
             </Stack.Item>

app/frontend/src/components/Answer/SpeechOutput.tsx renamed to app/frontend/src/components/Answer/SpeechOutputAzure.tsx

@@ -8,7 +8,7 @@ interface Props {
 
 let audio = new Audio();
 
-export const SpeechOutput = ({ url }: Props) => {
+export const SpeechOutputAzure = ({ url }: Props) => {
     const [isPlaying, setIsPlaying] = useState(false);
 
     const startOrStopAudio = async () => {

app/frontend/src/components/Answer/SpeechOutputBrowser.tsx

@@ -0,0 +1,71 @@
+import { useState } from "react";
+import { IconButton } from "@fluentui/react";
+
+interface Props {
+    answer: string;
+}
+
+const SpeechSynthesis = (window as any).speechSynthesis || (window as any).webkitSpeechSynthesis;
+
+let synth: SpeechSynthesis | null = null;
+
+try {
+    synth = SpeechSynthesis;
+} catch (err) {
+    console.error("SpeechSynthesis is not supported");
+}
+
+const getUtterance = function (text: string) {
+    if (synth) {
+        const utterance = new SpeechSynthesisUtterance(text);
+        utterance.lang = "en-US";
+        utterance.volume = 1;
+        utterance.rate = 1;
+        utterance.pitch = 1;
+        utterance.voice = synth.getVoices().filter((voice: SpeechSynthesisVoice) => voice.lang === "en-US")[0];
+        return utterance;
+    }
+};
+
+export const SpeechOutputBrowser = ({ answer }: Props) => {
+    const [isPlaying, setIsPlaying] = useState<boolean>(false);
+
+    const startOrStopSpeech = (answer: string) => {
+        if (synth != null) {
+            if (isPlaying) {
+                synth.cancel(); // removes all utterances from the utterance queue.
+                setIsPlaying(false);
+                return;
+            }
+            const utterance: SpeechSynthesisUtterance | undefined = getUtterance(answer);
+
+            if (!utterance) {
+                return;
+            }
+
+            synth.speak(utterance);
+
+            utterance.onstart = () => {
+                setIsPlaying(true);
+                return;
+            };
+
+            utterance.onend = () => {
+                setIsPlaying(false);
+                return;
+            };
+        }
+    };
+    const color = isPlaying ? "red" : "black";
+
+    return (
+        <IconButton
+            style={{ color: color }}
+            iconProps={{ iconName: "Volume3" }}
+            title="Speak answer"
+            ariaLabel="Speak answer"
+            onClick={() => startOrStopSpeech(answer)}
+            disabled={!synth}
+        />
+    );
+};

app/frontend/src/components/Answer/index.ts

@@ -1,4 +1,5 @@
 export * from "./Answer";
 export * from "./AnswerLoading";
 export * from "./AnswerError";
-export * from "./SpeechOutput";
+export * from "./SpeechOutputBrowser";
+export * from "./SpeechOutputAzure";

app/frontend/src/pages/ask/Ask.tsx

@@ -41,7 +41,8 @@ export function Component(): JSX.Element {
     const [showVectorOption, setShowVectorOption] = useState<boolean>(false);
     const [showUserUpload, setShowUserUpload] = useState<boolean>(false);
     const [showSpeechInput, setShowSpeechInput] = useState<boolean>(false);
-    const [showSpeechOutput, setShowSpeechOutput] = useState<boolean>(false);
+    const [showSpeechOutputBrowser, setShowSpeechOutputBrowser] = useState<boolean>(false);
+    const [showSpeechOutputAzure, setShowSpeechOutputAzure] = useState<boolean>(false);
 
     const lastQuestionRef = useRef<string>("");
 
@@ -66,7 +67,8 @@ export function Component(): JSX.Element {
             }
             setShowUserUpload(config.showUserUpload);
             setShowSpeechInput(config.showSpeechInput);
-            setShowSpeechOutput(config.showSpeechOutput);
+            setShowSpeechOutputBrowser(config.showSpeechOutputBrowser);
+            setShowSpeechOutputAzure(config.showSpeechOutputAzure);
         });
     };
 
@@ -75,7 +77,7 @@ export function Component(): JSX.Element {
     }, []);
 
     useEffect(() => {
-        if (answer && showSpeechOutput) {
+        if (answer && showSpeechOutputAzure) {
             getSpeechApi(answer.choices[0].message.content).then(speechUrl => {
                 setSpeechUrl(speechUrl);
             });
@@ -239,7 +241,8 @@ export function Component(): JSX.Element {
                             onCitationClicked={x => onShowCitation(x)}
                             onThoughtProcessClicked={() => onToggleTab(AnalysisPanelTabs.ThoughtProcessTab)}
                             onSupportingContentClicked={() => onToggleTab(AnalysisPanelTabs.SupportingContentTab)}
-                            showSpeechOutput={showSpeechOutput}
+                            showSpeechOutputAzure={showSpeechOutputAzure}
+                            showSpeechOutputBrowser={showSpeechOutputBrowser}
                             speechUrl={speechUrl}
                         />
                     </div>