Add multi-language support for speech input on web. (#586)

* Add multi-language support for speech input on web.

* Action PR comments.

Author: superhindupur

Makefile

@@ -20,7 +20,7 @@ include $(AZURE_ENV_FILE)
 help: ## 💬 This help message :)
 	@grep -E '[a-zA-Z_-]+:.*?## .*$$' $(firstword $(MAKEFILE_LIST)) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-23s\033[0m %s\n", $$1, $$2}'
 
-ci: lint unittest functionaltest build-frontend ## 🚀 Continuous Integration (called by Github Actions)
+ci: lint unittest functionaltest build-frontend unittest-frontend ## 🚀 Continuous Integration (called by Github Actions)
 
 lint: ## 🧹 Lint the code
 	@echo -e "\e[34m$@\e[0m" || true
@@ -42,6 +42,10 @@ build-frontend: ## 🏗️ Build the Frontend webapp
 	@echo -e "\e[34m$@\e[0m" || true
 	@cd code/frontend && npm install && npm run build
 
+unittest-frontend: ## 🏗️ Unit test the Frontend webapp
+	@echo -e "\e[34m$@\e[0m" || true
+	@cd code/frontend && npm install && npm run build && npm run test
+
 azd-login: ## 🔑 Login to Azure with azd and a SPN
 	@echo -e "\e[34m$@\e[0m" || true
 	@azd auth login --client-id ${AZURE_CLIENT_ID} --client-secret ${AZURE_CLIENT_SECRET} --tenant-id ${AZURE_TENANT_ID}

code/frontend/.mocharc.json

@@ -0,0 +1,9 @@
+{
+    "loader": "ts-node/esm",
+    "extensions": [
+        "ts"
+    ],
+    "spec": [
+        "./**/test/*.test.ts"
+    ]
+}

code/frontend/package-lock.json

@@ -6,7 +6,8 @@
     "scripts": {
         "dev": "tsc && vite",
         "build": "tsc && vite build",
-        "watch": "tsc && vite build --watch"
+        "watch": "tsc && vite build --watch",
+        "test": "mocha"
     },
     "dependencies": {
         "@babel/traverse": "^7.24.0",
@@ -29,13 +30,18 @@
         "uuid": "^9.0.1"
     },
     "devDependencies": {
+        "@types/chai": "^4.3.14",
         "@types/lodash-es": "^4.17.12",
+        "@types/mocha": "^10.0.6",
         "@types/node": "^20.12.2",
         "@types/react": "^18.2.73",
         "@types/react-dom": "^18.2.23",
         "@types/uuid": "^9.0.8",
         "@vitejs/plugin-react": "^4.2.1",
+        "chai": "^5.1.0",
+        "mocha": "^10.4.0",
         "prettier": "^3.2.5",
+        "ts-node": "^10.9.2",
         "typescript": "^5.4.3",
         "vite": "^5.2.7"
     }

code/frontend/package.json

@@ -6,10 +6,9 @@ import {
   SquareRegular,
 } from "@fluentui/react-icons";
 import {
-  SpeechConfig,
-  AudioConfig,
   SpeechRecognizer,
   ResultReason,
+  AutoDetectSourceLanguageResult,
 } from "microsoft-cognitiveservices-speech-sdk";
 import ReactMarkdown from "react-markdown";
 import remarkGfm from "remark-gfm";
@@ -18,12 +17,12 @@ import { v4 as uuidv4 } from "uuid";
 
 import styles from "./Chat.module.css";
 import Azure from "../../assets/Azure.svg";
+import { multiLingualSpeechRecognizer } from "../../util/SpeechToText";
 
 import {
   ChatMessage,
   ConversationRequest,
-  conversationApi,
-  customConversationApi,
+customConversationApi,
   Citation,
   ToolMessageContent,
   ChatResponse,
@@ -168,12 +167,12 @@ const Chat = () => {
           "Azure Speech subscription key or region is not defined."
         );
       } else {
-        const speechConfig = SpeechConfig.fromSubscription(
-          subscriptionKey,
-          serviceRegion
-        );
-        const audioConfig = AudioConfig.fromDefaultMicrophoneInput();
-        const recognizer = new SpeechRecognizer(speechConfig, audioConfig);
+        const recognizer = multiLingualSpeechRecognizer(subscriptionKey, serviceRegion, [
+          "en-US",
+          "fr-FR",
+          "de-DE",
+          "it-IT"
+        ]);
         recognizerRef.current = recognizer; // Store the recognizer in the ref
 
         recognizerRef.current.recognized = (s, e) => {

code/frontend/src/pages/chat/Chat.tsx

@@ -0,0 +1,21 @@
+import {
+    SpeechConfig,
+    AudioConfig,
+    SpeechRecognizer,
+    AutoDetectSourceLanguageConfig,
+  } from "microsoft-cognitiveservices-speech-sdk";
+
+
+export const multiLingualSpeechRecognizer = (
+    subscriptionKey: string,
+    serviceRegion: string,
+    languages: string[]
+  ) => {
+        const speechConfig = SpeechConfig.fromSubscription(
+          subscriptionKey,
+          serviceRegion
+        );
+        const audioConfig = AudioConfig.fromDefaultMicrophoneInput();
+        const autoDetectSourceLanguageConfig = AutoDetectSourceLanguageConfig.fromLanguages(languages)
+        return SpeechRecognizer.FromConfig(speechConfig, autoDetectSourceLanguageConfig, audioConfig);
+    };

code/frontend/src/util/SpeechToText.tsx

@@ -0,0 +1,16 @@
+import { expect } from "chai";
+import { it } from "mocha";
+import { multiLingualSpeechRecognizer } from "../src/util/SpeechToText.js";
+
+describe("SpeechToText", () => {
+  it("creates a speech recognizer with multiple languages", async () => {
+    const key = "key";
+    const region = "region";
+    const languages = ["en-US", "fr-FR", "de-DE", "it-IT"];
+    const recognizer = multiLingualSpeechRecognizer(key, region, languages);
+
+    expect(recognizer.properties.getProperty("SpeechServiceConnection_Key")).to.equal(key);
+    expect(recognizer.properties.getProperty("SpeechServiceConnection_Region")).to.equal(region);
+    expect(recognizer.properties.getProperty("SpeechServiceConnection_AutoDetectSourceLanguages")).to.equal(languages.join(","));
+  });
+});

code/frontend/test/SpeechToText.test.ts

@@ -31,4 +31,4 @@
       "path": "./tsconfig.node.json"
     }
   ]
-}
+}