From 22c7fc210b801bf9d52536ccb89b7cbcc138a5ec Mon Sep 17 00:00:00 2001 From: JHarrysP Date: Wed, 6 Aug 2025 00:07:55 +0200 Subject: [PATCH] Adds voice input feature --- frontend/package-lock.json | 97 ++++---- frontend/src/components/InputForm.tsx | 42 +++- frontend/src/components/VoiceInput.tsx | 332 +++++++++++++++++++++++++ 3 files changed, 419 insertions(+), 52 deletions(-) create mode 100644 frontend/src/components/VoiceInput.tsx diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 48599091..3ff6fe12 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -491,9 +491,9 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.20.0", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.20.0.tgz", - "integrity": "sha512-fxlS1kkIjx8+vy2SjuCB94q3htSNrufYTXubwiBFeaQHbH6Ipi43gFJq2zCMt6PHhImH3Xmr0NksKDvchWlpQQ==", + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.0.tgz", + "integrity": "sha512-ENIdc4iLu0d93HeYirvKmrzshzofPw6VkZRKQGe9Nv46ZnWUzcF1xV01dcvEg/1wXUR61OmmlSfyeyO7EvjLxQ==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -506,9 +506,9 @@ } }, "node_modules/@eslint/config-helpers": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.2.1.tgz", - "integrity": "sha512-RI17tsD2frtDu/3dmI7QRrD4bedNKPM08ziRYaC5AhkGrzIAJelm9kJU1TznK+apx6V+cqRz8tfpEeG3oIyjxw==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.3.0.tgz", + "integrity": "sha512-ViuymvFmcJi04qdZeDc2whTHryouGcDlaxPqarTD0ZE10ISpxGUVZGZDx4w01upyIynL3iu6IXH2bS1NhclQMw==", "dev": true, "license": "Apache-2.0", "engines": { @@ -516,9 +516,9 @@ } }, "node_modules/@eslint/core": { - "version": "0.13.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.13.0.tgz", - "integrity": "sha512-yfkgDw1KR66rkT5A8ci4irzDysN7FRpq3ttJolR88OqQikAWqwA8j5VZyas+vjyBNFIJ7MfybJ9plMILI2UrCw==", + "version": "0.15.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.15.1.tgz", + "integrity": "sha512-bkOp+iumZCCbt1K1CmWf0R9pM5yKpDv+ZXtvSyQpudrI9kuFLp+bM2WOPXImuD/ceQuaa8f5pj93Y7zyECIGNA==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -566,13 +566,16 @@ } }, "node_modules/@eslint/js": { - "version": "9.25.1", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.25.1.tgz", - "integrity": "sha512-dEIwmjntEx8u3Uvv+kr3PDeeArL8Hw07H9kyYxCjnM9pBjfEhk6uLXSchxxzgiwtRhhzVzqmUSDFBOi1TuZ7qg==", + "version": "9.32.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.32.0.tgz", + "integrity": "sha512-BBpRFZK3eX6uMLKz8WxFOBIFFcGFJ/g8XuwjTHCqHROSIsopI+ddn/d5Cfh36+7+e5edVS8dbSHnBNhrLEX0zg==", "dev": true, "license": "MIT", "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" } }, "node_modules/@eslint/object-schema": { @@ -586,13 +589,13 @@ } }, "node_modules/@eslint/plugin-kit": { - "version": "0.2.8", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.2.8.tgz", - "integrity": "sha512-ZAoA40rNMPwSm+AeHpCq8STiNAwzWLJuP8Xv4CHIc9wv/PSuExjMrmjfYNj682vW0OOiZ1HKxzvjQr9XZIisQA==", + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.3.4.tgz", + "integrity": "sha512-Ul5l+lHEcw3L5+k8POx6r74mxEYKG5kOb6Xpy2gCRW6zweT6TEhAf8vhxGgjhqrd/VO/Dirhsb+1hNpD1ue9hw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^0.13.0", + "@eslint/core": "^0.15.1", "levn": "^0.4.1" }, "engines": { @@ -2439,9 +2442,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "dev": true, "license": "MIT", "dependencies": { @@ -2526,9 +2529,9 @@ } }, "node_modules/acorn": { - "version": "8.14.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", - "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", "bin": { @@ -2637,9 +2640,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", "dev": true, "license": "MIT", "dependencies": { @@ -2993,20 +2996,20 @@ } }, "node_modules/eslint": { - "version": "9.25.1", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.25.1.tgz", - "integrity": "sha512-E6Mtz9oGQWDCpV12319d59n4tx9zOTXSTmc8BLVxBx+G/0RdM5MvEEJLU9c0+aleoePYYgVTOsRblx433qmhWQ==", + "version": "9.32.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.32.0.tgz", + "integrity": "sha512-LSehfdpgMeWcTZkWZVIJl+tkZ2nuSkyyB9C27MZqFWXuph7DvaowgcTvKqxvpLW1JZIk8PN7hFY3Rj9LQ7m7lg==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.20.0", - "@eslint/config-helpers": "^0.2.1", - "@eslint/core": "^0.13.0", + "@eslint/config-array": "^0.21.0", + "@eslint/config-helpers": "^0.3.0", + "@eslint/core": "^0.15.0", "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.25.1", - "@eslint/plugin-kit": "^0.2.8", + "@eslint/js": "9.32.0", + "@eslint/plugin-kit": "^0.3.4", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", @@ -3017,9 +3020,9 @@ "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.3.0", - "eslint-visitor-keys": "^4.2.0", - "espree": "^10.3.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", "esquery": "^1.5.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", @@ -3077,9 +3080,9 @@ } }, "node_modules/eslint-scope": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.3.0.tgz", - "integrity": "sha512-pUNxi75F8MJ/GdeKtVLSbYg4ZI34J6C0C7sbL4YOp2exGwen7ZsuBqKzUhXd0qMQ362yET3z+uPwKeg/0C2XCQ==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", "dev": true, "license": "BSD-2-Clause", "dependencies": { @@ -3094,9 +3097,9 @@ } }, "node_modules/eslint-visitor-keys": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.0.tgz", - "integrity": "sha512-UyLnSehNt62FFhSwjZlHmeokpRK59rcz29j+F1/aDgbkbRTk7wIc9XzdoasMUbRNKDM0qQt/+BJ4BrpFeABemw==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", "dev": true, "license": "Apache-2.0", "engines": { @@ -3107,15 +3110,15 @@ } }, "node_modules/espree": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.3.0.tgz", - "integrity": "sha512-0QYC8b24HWY8zjRnDTL6RiHfDbAWn63qb4LMj1Z4b076A4une81+z03Kg7l7mn/48PUTqoLptSXez8oknU8Clg==", + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "acorn": "^8.14.0", + "acorn": "^8.15.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.0" + "eslint-visitor-keys": "^4.2.1" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" diff --git a/frontend/src/components/InputForm.tsx b/frontend/src/components/InputForm.tsx index 97aa5c67..c21c6f66 100644 --- a/frontend/src/components/InputForm.tsx +++ b/frontend/src/components/InputForm.tsx @@ -9,6 +9,7 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; +import { VoiceInput } from "@/components/VoiceInput"; // Updated InputFormProps interface InputFormProps { @@ -26,7 +27,7 @@ export const InputForm: React.FC = ({ }) => { const [internalInputValue, setInternalInputValue] = useState(""); const [effort, setEffort] = useState("medium"); - const [model, setModel] = useState("gemini-2.5-flash-preview-04-17"); + const [model, setModel] = useState("gemini-2.5-flash"); const handleInternalSubmit = (e?: React.FormEvent) => { if (e) e.preventDefault(); @@ -36,13 +37,35 @@ export const InputForm: React.FC = ({ }; const handleKeyDown = (e: React.KeyboardEvent) => { - // Submit with Ctrl+Enter (Windows/Linux) or Cmd+Enter (Mac) + // Submit with Enter (not Ctrl+Enter) - this is more intuitive + if (e.key === "Enter" && !e.shiftKey && !e.ctrlKey && !e.metaKey) { + e.preventDefault(); + handleInternalSubmit(); + } + // Allow Shift+Enter for new lines + // Allow Ctrl+Enter and Cmd+Enter as alternative submit methods if (e.key === "Enter" && (e.ctrlKey || e.metaKey)) { e.preventDefault(); handleInternalSubmit(); } }; + const handleVoiceTranscription = (transcription: string, shouldAutoSubmit = false) => { + // Append transcription to existing input value or replace if empty + const newValue = internalInputValue.trim() + ? `${internalInputValue} ${transcription}` + : transcription; + console.log('Setting transcription value:', newValue); // Debug log + setInternalInputValue(newValue); + + // If auto-submit is requested, submit immediately with the new value + if (shouldAutoSubmit && newValue.trim()) { + console.log('Auto-submitting with new value:', newValue); // Debug log + onSubmit(newValue, effort, model); + setInternalInputValue(""); + } + }; + const isSubmitDisabled = !internalInputValue.trim() || isLoading; return ( @@ -64,7 +87,16 @@ export const InputForm: React.FC = ({ md:text-base min-h-[56px] max-h-[200px]`} rows={1} /> -
+
+ {/* Voice Input Button */} + + + {/* Submit/Cancel Button */} {isLoading ? (
@@ -176,4 +208,4 @@ export const InputForm: React.FC = ({
); -}; +}; \ No newline at end of file diff --git a/frontend/src/components/VoiceInput.tsx b/frontend/src/components/VoiceInput.tsx new file mode 100644 index 00000000..30358bea --- /dev/null +++ b/frontend/src/components/VoiceInput.tsx @@ -0,0 +1,332 @@ +import React, { useState, useRef, useEffect, useCallback } from 'react'; +import { Mic, MicOff, Square, AlertCircle } from 'lucide-react'; +import { Button } from '@/components/ui/button'; + +interface VoiceInputProps { + onTranscription: (text: string, shouldAutoSubmit?: boolean) => void; + disabled?: boolean; + language?: string; + continuous?: boolean; + autoSubmit?: boolean; + timeoutDuration?: number; // in milliseconds +} + +// Type declarations for Web Speech API +declare global { + interface Window { + webkitSpeechRecognition: any; + SpeechRecognition: any; + } +} + +interface SpeechRecognitionEvent { + results: SpeechRecognitionResultList; + resultIndex: number; +} + +interface SpeechRecognitionResultList { + length: number; + item(index: number): SpeechRecognitionResult; + [index: number]: SpeechRecognitionResult; +} + +interface SpeechRecognitionResult { + length: number; + item(index: number): SpeechRecognitionAlternative; + [index: number]: SpeechRecognitionAlternative; + isFinal: boolean; +} + +interface SpeechRecognitionAlternative { + transcript: string; + confidence: number; +} + +interface SpeechRecognitionErrorEvent { + error: string; + message: string; +} + +export const VoiceInput: React.FC = ({ + onTranscription, + disabled = false, + language = 'en-US', + continuous = false, + autoSubmit = false, + timeoutDuration = 30000 // 30 seconds default +}) => { + const [isRecording, setIsRecording] = useState(false); + const [isProcessing, setIsProcessing] = useState(false); + const [error, setError] = useState(null); + const [isSupported, setIsSupported] = useState(false); + const [confidence, setConfidence] = useState(0); + + const recognitionRef = useRef(null); + const streamRef = useRef(null); + const timeoutRef = useRef(null); + + // Check browser support on component mount + useEffect(() => { + const checkSupport = () => { + const hasGetUserMedia = !!(navigator.mediaDevices && typeof navigator.mediaDevices.getUserMedia === 'function'); + const hasWebkitSpeechRecognition = 'webkitSpeechRecognition' in window; + const hasSpeechRecognition = 'SpeechRecognition' in window; + + return hasGetUserMedia && (hasWebkitSpeechRecognition || hasSpeechRecognition); + }; + + setIsSupported(checkSupport()); + }, []); + + const cleanupResources = useCallback(() => { + if (recognitionRef.current) { + try { + recognitionRef.current.stop(); + } catch (e) { + // Ignore errors when stopping + } + recognitionRef.current = null; + } + + if (streamRef.current) { + streamRef.current.getTracks().forEach(track => track.stop()); + streamRef.current = null; + } + + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + timeoutRef.current = null; + } + }, []); + + const startRecording = async () => { + try { + setError(null); + setIsProcessing(true); + + // Request microphone access + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + sampleRate: 44100, + } + }); + + streamRef.current = stream; + + // Use Web Speech API for real-time transcription + const SpeechRecognition = window.webkitSpeechRecognition || window.SpeechRecognition; + + if (SpeechRecognition) { + const recognition = new SpeechRecognition(); + recognition.continuous = continuous; + recognition.interimResults = false; + recognition.lang = language; + recognition.maxAlternatives = 1; + + let finalTranscript = ''; + + recognition.onstart = () => { + setIsRecording(true); + setIsProcessing(false); + setError(null); + }; + + recognition.onresult = (event: SpeechRecognitionEvent) => { + let transcript = ''; + let maxConfidence = 0; + + for (let i = event.resultIndex; i < event.results.length; i++) { + const result = event.results[i]; + if (result.isFinal) { + transcript += result[0].transcript; + maxConfidence = Math.max(maxConfidence, result[0].confidence || 0); + } + } + + if (transcript.trim()) { + finalTranscript += transcript; + setConfidence(maxConfidence); + + // For continuous mode, send each final result immediately + if (continuous) { + onTranscription(transcript.trim(), autoSubmit); + } + } + }; + + recognition.onerror = (event: SpeechRecognitionErrorEvent) => { + console.error('Speech recognition error:', event.error); + + let errorMessage = 'Speech recognition error'; + switch (event.error) { + case 'no-speech': + errorMessage = 'No speech detected. Try speaking again.'; + break; + case 'audio-capture': + errorMessage = 'Microphone not available'; + break; + case 'not-allowed': + errorMessage = 'Microphone access denied'; + break; + case 'network': + errorMessage = 'Network error occurred'; + break; + default: + errorMessage = `Recognition error: ${event.error}`; + } + + setError(errorMessage); + setIsRecording(false); + setIsProcessing(false); + cleanupResources(); + }; + + recognition.onend = () => { + console.log('Recognition ended. Final transcript:', finalTranscript); // Debug log + + // Send the final transcript if not in continuous mode + if (!continuous && finalTranscript.trim()) { + console.log('Sending transcription with auto-submit:', autoSubmit); // Debug log + // Pass the auto-submit flag directly to the transcription handler + onTranscription(finalTranscript.trim(), autoSubmit); + } + + setIsRecording(false); + setIsProcessing(false); + cleanupResources(); + }; + + // Auto-stop after specified duration if not continuous + if (!continuous) { + timeoutRef.current = setTimeout(() => { + if (recognitionRef.current) { + recognitionRef.current.stop(); + } + }, timeoutDuration); + } + + recognitionRef.current = recognition; + recognition.start(); + + } else { + throw new Error('Speech recognition not supported in this browser'); + } + + } catch (err) { + console.error('Error starting recording:', err); + let errorMessage = 'Failed to start recording'; + + if (err instanceof Error) { + if (err.name === 'NotAllowedError') { + errorMessage = 'Microphone access denied. Please allow microphone access and try again.'; + } else if (err.name === 'NotFoundError') { + errorMessage = 'No microphone found. Please connect a microphone and try again.'; + } else { + errorMessage = err.message; + } + } + + setError(errorMessage); + setIsRecording(false); + setIsProcessing(false); + cleanupResources(); + } + }; + + const stopRecording = useCallback(() => { + setIsProcessing(true); + + if (recognitionRef.current) { + try { + recognitionRef.current.stop(); + } catch (e) { + // Recognition might already be stopped + setIsRecording(false); + setIsProcessing(false); + cleanupResources(); + } + } else { + setIsRecording(false); + setIsProcessing(false); + cleanupResources(); + } + }, [cleanupResources]); + + // Cleanup on component unmount + useEffect(() => { + return () => { + cleanupResources(); + }; + }, [cleanupResources]); + + if (!isSupported) { + return null; // Don't render if not supported + } + + const getButtonColor = () => { + if (isRecording) return 'text-red-500 hover:text-red-400 hover:bg-red-500/10'; + if (isProcessing) return 'text-yellow-500 hover:text-yellow-400 hover:bg-yellow-500/10'; + if (error) return 'text-red-400 hover:text-red-300 hover:bg-red-500/10'; + return 'text-neutral-400 hover:text-blue-400 hover:bg-blue-500/10'; + }; + + const getButtonTitle = () => { + if (isRecording) return 'Stop recording (click to finish)'; + if (isProcessing) return 'Processing speech...'; + if (error) return 'Voice input error - click to retry'; + return 'Start voice input'; + }; + + return ( +
+ + + {/* Status indicators */} + {isRecording && ( +
+ Recording... +
+ )} + + {isProcessing && !isRecording && ( +
+ Starting... +
+ )} + + {error && !isRecording && !isProcessing && ( +
+ {error.length > 30 ? `${error.substring(0, 30)}...` : error} +
+ )} + + {confidence > 0 && !isRecording && !error && ( +
+ {Math.round(confidence * 100)}% confident +
+ )} +
+ ); +}; \ No newline at end of file