Add text-to-speech functionality (#1412)

* Add text-to-speech functionality

* Add speed config option to text-to-speech

* Fix test case for tts speed slider

* Fix test case for tts speed slider (really)

* Disabled error message logging in tts.ts

* ignore markdown and mermaid diagrams in TTS

* add ttsEnabled and ttsSpeed to GlobalStateKey

* fix failing webview test for save button

* Translations

* Fix tests

---------

Co-authored-by: Matt Rubens <[email protected]>

Author: heyseth

package-lock.json

@@ -359,6 +359,7 @@
 		"puppeteer-chromium-resolver": "^23.0.0",
 		"puppeteer-core": "^23.4.0",
 		"reconnecting-eventsource": "^1.6.4",
+		"say": "^0.16.0",
 		"serialize-error": "^11.0.3",
 		"simple-git": "^3.27.0",
 		"sound-play": "^1.1.0",

package.json

@@ -39,6 +39,7 @@ import { BrowserSession } from "../../services/browser/BrowserSession"
 import { discoverChromeInstances } from "../../services/browser/browserDiscovery"
 import { fileExistsAtPath } from "../../utils/fs"
 import { playSound, setSoundEnabled, setSoundVolume } from "../../utils/sound"
+import { playTts, setTtsEnabled, setTtsSpeed } from "../../utils/tts"
 import { singleCompletionHandler } from "../../utils/single-completion-handler"
 import { searchCommits } from "../../utils/git"
 import { getDiffStrategy } from "../diff/DiffStrategy"
@@ -356,6 +357,11 @@ export class ClineProvider extends EventEmitter<ClineProviderEvents> implements
 			setSoundEnabled(soundEnabled ?? false)
 		})
 
+		// Initialize tts enabled state
+		this.getState().then(({ ttsEnabled }) => {
+			setTtsEnabled(ttsEnabled ?? false)
+		})
+
 		webviewView.webview.options = {
 			// Allow scripts in the webview
 			enableScripts: true,
@@ -1233,6 +1239,23 @@ export class ClineProvider extends EventEmitter<ClineProviderEvents> implements
 						setSoundVolume(soundVolume)
 						await this.postStateToWebview()
 						break
+					case "ttsEnabled":
+						const ttsEnabled = message.bool ?? true
+						await this.updateGlobalState("ttsEnabled", ttsEnabled)
+						setTtsEnabled(ttsEnabled) // Add this line to update the tts utility
+						await this.postStateToWebview()
+						break
+					case "ttsSpeed":
+						const ttsSpeed = message.value ?? 1.0
+						await this.updateGlobalState("ttsSpeed", ttsSpeed)
+						setTtsSpeed(ttsSpeed)
+						await this.postStateToWebview()
+						break
+					case "playTts":
+						if (message.text) {
+							playTts(message.text)
+						}
+						break
 					case "diffEnabled":
 						const diffEnabled = message.bool ?? true
 						await this.updateGlobalState("diffEnabled", diffEnabled)
@@ -2333,6 +2356,8 @@ export class ClineProvider extends EventEmitter<ClineProviderEvents> implements
 			alwaysAllowModeSwitch,
 			alwaysAllowSubtasks,
 			soundEnabled,
+			ttsEnabled,
+			ttsSpeed,
 			diffEnabled,
 			enableCheckpoints,
 			checkpointStorage,
@@ -2392,6 +2417,8 @@ export class ClineProvider extends EventEmitter<ClineProviderEvents> implements
 				.filter((item: HistoryItem) => item.ts && item.task)
 				.sort((a: HistoryItem, b: HistoryItem) => b.ts - a.ts),
 			soundEnabled: soundEnabled ?? false,
+			ttsEnabled: ttsEnabled ?? false,
+			ttsSpeed: ttsSpeed ?? 1.0,
 			diffEnabled: diffEnabled ?? true,
 			enableCheckpoints: enableCheckpoints ?? true,
 			checkpointStorage: checkpointStorage ?? "task",
@@ -2551,6 +2578,8 @@ export class ClineProvider extends EventEmitter<ClineProviderEvents> implements
 			taskHistory: stateValues.taskHistory,
 			allowedCommands: stateValues.allowedCommands,
 			soundEnabled: stateValues.soundEnabled ?? false,
+			ttsEnabled: stateValues.ttsEnabled ?? false,
+			ttsSpeed: stateValues.ttsSpeed ?? 1.0,
 			diffEnabled: stateValues.diffEnabled ?? true,
 			enableCheckpoints: stateValues.enableCheckpoints ?? true,
 			checkpointStorage: stateValues.checkpointStorage ?? "task",

src/core/webview/ClineProvider.ts

@@ -7,6 +7,7 @@ import { ClineProvider } from "../ClineProvider"
 import { ExtensionMessage, ExtensionState } from "../../../shared/ExtensionMessage"
 import { GlobalStateKey, SecretKey } from "../../../shared/globalState"
 import { setSoundEnabled } from "../../../utils/sound"
+import { setTtsEnabled } from "../../../utils/tts"
 import { defaultModeSlug } from "../../../shared/modes"
 import { experimentDefault } from "../../../shared/experiments"
 import { Cline } from "../../Cline"
@@ -271,6 +272,11 @@ jest.mock("../../../utils/sound", () => ({
 	setSoundEnabled: jest.fn(),
 }))
 
+// Mock tts utility
+jest.mock("../../../utils/tts", () => ({
+	setTtsEnabled: jest.fn(),
+}))
+
 // Mock ESM modules
 jest.mock("p-wait-for", () => ({
 	__esModule: true,
@@ -506,6 +512,7 @@ describe("ClineProvider", () => {
 			alwaysAllowMcp: false,
 			uriScheme: "vscode",
 			soundEnabled: false,
+			ttsEnabled: false,
 			diffEnabled: false,
 			enableCheckpoints: false,
 			checkpointStorage: "task",
@@ -603,6 +610,7 @@ describe("ClineProvider", () => {
 		expect(state).toHaveProperty("alwaysAllowBrowser")
 		expect(state).toHaveProperty("taskHistory")
 		expect(state).toHaveProperty("soundEnabled")
+		expect(state).toHaveProperty("ttsEnabled")
 		expect(state).toHaveProperty("diffEnabled")
 		expect(state).toHaveProperty("writeDelayMs")
 	})
@@ -666,6 +674,18 @@ describe("ClineProvider", () => {
 		expect(setSoundEnabled).toHaveBeenCalledWith(false)
 		expect(mockContext.globalState.update).toHaveBeenCalledWith("soundEnabled", false)
 		expect(mockPostMessage).toHaveBeenCalled()
+
+		// Simulate setting tts to enabled
+		await messageHandler({ type: "ttsEnabled", bool: true })
+		expect(setTtsEnabled).toHaveBeenCalledWith(true)
+		expect(mockContext.globalState.update).toHaveBeenCalledWith("ttsEnabled", true)
+		expect(mockPostMessage).toHaveBeenCalled()
+
+		// Simulate setting tts to disabled
+		await messageHandler({ type: "ttsEnabled", bool: false })
+		expect(setTtsEnabled).toHaveBeenCalledWith(false)
+		expect(mockContext.globalState.update).toHaveBeenCalledWith("ttsEnabled", false)
+		expect(mockPostMessage).toHaveBeenCalled()
 	})
 
 	test("requestDelaySeconds defaults to 10 seconds", async () => {

src/core/webview/__tests__/ClineProvider.test.ts

@@ -207,6 +207,8 @@ export type GlobalStateKey =
 	| "openRouterUseMiddleOutTransform"
 	| "googleGeminiBaseUrl"
 	| "allowedCommands"
+	| "ttsEnabled"
+	| "ttsSpeed"
 	| "soundEnabled"
 	| "soundVolume"
 	| "diffEnabled"

src/exports/roo-code.d.ts

@@ -124,6 +124,8 @@ export interface ExtensionState {
 	currentTaskItem?: HistoryItem
 	allowedCommands?: string[]
 	soundEnabled?: boolean
+	ttsEnabled?: boolean
+	ttsSpeed?: number
 	soundVolume?: number
 	diffEnabled?: boolean
 	enableCheckpoints: boolean

src/shared/ExtensionMessage.ts

@@ -50,7 +50,10 @@ export interface WebviewMessage {
 		| "alwaysAllowModeSwitch"
 		| "alwaysAllowSubtasks"
 		| "playSound"
+		| "playTts"
 		| "soundEnabled"
+		| "ttsEnabled"
+		| "ttsSpeed"
 		| "soundVolume"
 		| "diffEnabled"
 		| "enableCheckpoints"

src/shared/WebviewMessage.ts

@@ -76,6 +76,8 @@ export const GLOBAL_STATE_KEYS = [
 	"googleGeminiBaseUrl",
 	"allowedCommands",
 	"soundEnabled",
+	"ttsEnabled",
+	"ttsSpeed",
 	"soundVolume",
 	"diffEnabled",
 	"enableCheckpoints",

src/shared/globalState.ts

@@ -0,0 +1,75 @@
+import * as vscode from "vscode"
+
+let isTtsEnabled = false
+let speed = 1.0
+let isSpeaking = false
+const utteranceQueue: string[] = []
+
+/**
+ * Set tts configuration
+ * @param enabled boolean
+ */
+export const setTtsEnabled = (enabled: boolean): void => {
+	isTtsEnabled = enabled
+}
+
+/**
+ * Set tts speed
+ * @param speed number
+ */
+export const setTtsSpeed = (newSpeed: number): void => {
+	speed = newSpeed
+}
+
+/**
+ * Process the next item in the utterance queue
+ */
+const processQueue = async (): Promise<void> => {
+	if (!isTtsEnabled || isSpeaking || utteranceQueue.length === 0) {
+		return
+	}
+
+	try {
+		isSpeaking = true
+		const nextUtterance = utteranceQueue.shift()!
+		const say = require("say")
+
+		// Wrap say.speak in a promise to handle completion
+		await new Promise<void>((resolve, reject) => {
+			say.speak(nextUtterance, null, speed, (err: Error) => {
+				if (err) {
+					reject(err)
+				} else {
+					resolve()
+				}
+			})
+		})
+
+		isSpeaking = false
+		// Process next item in queue if any
+		await processQueue()
+	} catch (error: any) {
+		isSpeaking = false
+		//vscode.window.showErrorMessage(error.message)
+		// Try to continue with next item despite error
+		await processQueue()
+	}
+}
+
+/**
+ * Queue a tts message to be spoken
+ * @param message string
+ * @return void
+ */
+export const playTts = async (message: string): Promise<void> => {
+	if (!isTtsEnabled) {
+		return
+	}
+
+	try {
+		utteranceQueue.push(message)
+		await processQueue()
+	} catch (error: any) {
+		//vscode.window.showErrorMessage(error.message)
+	}
+}