add audio prompt

Author: ci-bot

libs/remix-ui/remix-ai-assistant/src/components/prompt.tsx

@@ -30,6 +30,8 @@ export interface PromptAreaProps {
   handleSetModel: () => void
   handleModelSelection: (modelName: string) => void
   handleGenerateWorkspace: () => void
+  handleRecord: () => void
+  isRecording: boolean
   dispatchActivity: (type: ActivityType, payload?: any) => void
   contextBtnRef: React.RefObject<HTMLButtonElement>
   modelBtnRef: React.RefObject<HTMLButtonElement>
@@ -70,6 +72,8 @@ export const PromptArea: React.FC<PromptAreaProps> = ({
   handleSetModel,
   handleModelSelection,
   handleGenerateWorkspace,
+  handleRecord,
+  isRecording,
   dispatchActivity,
   contextBtnRef,
   modelBtnRef,
@@ -205,6 +209,15 @@ export const PromptArea: React.FC<PromptAreaProps> = ({
                 </button>
               )}
             </div>
+             <button
+              data-id="remix-ai-record-audio"
+              className={`btn btn-text btn-sm small fw-light mt-2 align-self-end border border-text rounded ${isRecording ? 'btn-danger text-white' : 'text-secondary'}`}
+              onClick={handleRecord}
+              title={isRecording ? 'Stop recording' : 'Record audio'}
+            >
+              <i className={`fa ${isRecording ? 'fa-stop' : 'fa-microphone'} me-1`}></i>
+              {isRecording ? 'Stop' : 'Audio Prompt'}
+            </button>
             <button
               data-id="remix-ai-workspace-generate"
               className="btn btn-text btn-sm small fw-light text-secondary mt-2 align-self-end border border-text rounded"

libs/remix-ui/remix-ai-assistant/src/components/remix-ui-remix-ai-assistant.tsx

@@ -12,6 +12,7 @@ import { ActivityType, ChatMessage } from '../lib/types'
 import { groupListType } from '../types/componentTypes'
 import GroupListMenu from './contextOptMenu'
 import { useOnClickOutside } from './onClickOutsideHook'
+import { useAudioTranscription } from '../hooks/useAudioTranscription'
 
 const _paq = (window._paq = window._paq || [])
 
@@ -63,6 +64,58 @@ export const RemixUiRemixAiAssistant = React.forwardRef<
   const textareaRef = useRef<HTMLTextAreaElement>(null)
   const aiChatRef = useRef<HTMLDivElement>(null)
 
+  // Ref to hold the sendPrompt function for audio transcription callback
+  const sendPromptRef = useRef<((prompt: string) => Promise<void>) | null>(null)
+
+  // Audio transcription hook
+  const {
+    isRecording,
+    isTranscribing,
+    error: transcriptionError,
+    toggleRecording
+  } = useAudioTranscription({
+    apiKey: 'fw_3ZZeKZ67JHvZKahmHUvo8XTR',
+    model: 'whisper-v3',
+    onTranscriptionComplete: async (text) => {
+      if (sendPromptRef.current) {
+        await sendPromptRef.current(text)
+        _paq.push(['trackEvent', 'remixAI', 'SpeechToTextPrompt', text])
+      }
+    },
+    onError: (error) => {
+      console.error('Audio transcription error:', error)
+      setMessages(prev => [...prev, {
+        id: crypto.randomUUID(),
+        role: 'assistant',
+        content: `**Audio transcription failed.**\n\nError: ${error.message}`,
+        timestamp: Date.now(),
+        sentiment: 'none'
+      }])
+    }
+  })
+
+  // Show transcribing status
+  useEffect(() => {
+    if (isTranscribing) {
+      setMessages(prev => [...prev, {
+        id: crypto.randomUUID(),
+        role: 'assistant',
+        content: '***Transcribing audio...***',
+        timestamp: Date.now(),
+        sentiment: 'none'
+      }])
+    } else {
+      // Remove transcribing message when done
+      setMessages(prev => {
+        const last = prev[prev.length - 1]
+        if (last?.content === '***Transcribing audio...***') {
+          return prev.slice(0, -1)
+        }
+        return prev
+      })
+    }
+  }, [isTranscribing])
+
   useOnClickOutside([modelBtnRef, contextBtnRef], () => setShowAssistantOptions(false))
   useOnClickOutside([modelBtnRef, contextBtnRef], () => setShowContextOptions(false))
   useOnClickOutside([modelSelectorBtnRef], () => setShowModelOptions(false))
@@ -429,6 +482,12 @@ export const RemixUiRemixAiAssistant = React.forwardRef<
     },
     [isStreaming, props.plugin]
   )
+
+  // Update ref for audio transcription callback
+  useEffect(() => {
+    sendPromptRef.current = sendPrompt
+  }, [sendPrompt])
+
   const handleGenerateWorkspaceWithPrompt = useCallback(async (prompt: string) => {
     dispatchActivity('button', 'generateWorkspace')
     if (prompt && prompt.trim()) {
@@ -647,6 +706,13 @@ export const RemixUiRemixAiAssistant = React.forwardRef<
     )
   }
 
+  const handleRecord = useCallback(async () => {
+    await toggleRecording()
+    if (!isRecording) {
+      _paq.push(['trackEvent', 'remixAI', 'StartAudioRecording'])
+    }
+  }, [toggleRecording, isRecording])
+
   const handleGenerateWorkspace = useCallback(async () => {
     dispatchActivity('button', 'generateWorkspace')
     try {
@@ -802,6 +868,8 @@ export const RemixUiRemixAiAssistant = React.forwardRef<
           handleSetModel={handleSetModel}
           handleModelSelection={handleModelSelection}
           handleGenerateWorkspace={handleGenerateWorkspace}
+          handleRecord={handleRecord}
+          isRecording={isRecording}
           dispatchActivity={dispatchActivity}
           contextBtnRef={contextBtnRef}
           modelBtnRef={modelBtnRef}

libs/remix-ui/remix-ai-assistant/src/hooks/useAudioTranscription.ts

@@ -0,0 +1,105 @@
+/**
+ * React hook for audio recording and transcription
+ */
+
+import { useState, useRef, useCallback } from 'react'
+import { AudioRecorder } from '../utils/audioRecorder'
+import { transcribeAudio, FireworksTranscriptionError } from '../services/fireworksTranscription'
+
+export interface UseAudioTranscriptionOptions {
+  apiKey: string
+  model?: string
+  onTranscriptionComplete?: (text: string) => void
+  onError?: (error: Error) => void
+}
+
+export interface UseAudioTranscriptionResult {
+  isRecording: boolean
+  isTranscribing: boolean
+  error: Error | null
+  startRecording: () => Promise<void>
+  stopRecording: () => void
+  toggleRecording: () => Promise<void>
+}
+
+export function useAudioTranscription(
+  options: UseAudioTranscriptionOptions
+): UseAudioTranscriptionResult {
+  const [isRecording, setIsRecording] = useState(false)
+  const [isTranscribing, setIsTranscribing] = useState(false)
+  const [error, setError] = useState<Error | null>(null)
+  const audioRecorderRef = useRef<AudioRecorder | null>(null)
+
+  const startRecording = useCallback(async () => {
+    try {
+      setError(null)
+
+      // Create new recorder instance
+      if (!audioRecorderRef.current) {
+        audioRecorderRef.current = new AudioRecorder()
+      }
+
+      await audioRecorderRef.current.startRecording({
+        onStop: async (audioBlob) => {
+          setIsRecording(false)
+          setIsTranscribing(true)
+
+          try {
+            // Transcribe the audio
+            const result = await transcribeAudio(audioBlob, {
+              apiKey: options.apiKey,
+              model: options.model
+            })
+
+            setIsTranscribing(false)
+
+            // Call completion callback with transcribed text
+            if (result.text && result.text.trim()) {
+              options.onTranscriptionComplete?.(result.text.trim())
+            }
+          } catch (err) {
+            setIsTranscribing(false)
+            const error = err instanceof Error ? err : new Error('Transcription failed')
+            setError(error)
+            options.onError?.(error)
+          }
+        },
+        onError: (err) => {
+          setIsRecording(false)
+          setError(err)
+          options.onError?.(err)
+        }
+      })
+
+      setIsRecording(true)
+    } catch (err) {
+      const error = err instanceof Error ? err : new Error('Failed to start recording')
+      setError(error)
+      setIsRecording(false)
+      options.onError?.(error)
+    }
+  }, [options])
+
+  const stopRecording = useCallback(() => {
+    if (audioRecorderRef.current) {
+      audioRecorderRef.current.stopRecording()
+    }
+  }, [])
+
+  const toggleRecording = useCallback(async () => {
+    if (isRecording) {
+      stopRecording()
+    } else {
+      await startRecording()
+    }
+  }, [isRecording, startRecording, stopRecording])
+
+  return {
+    isRecording,
+    isTranscribing,
+    error,
+    startRecording,
+    stopRecording,
+    toggleRecording
+  }
+}

libs/remix-ui/remix-ai-assistant/src/services/fireworksTranscription.ts

@@ -0,0 +1,92 @@
+/**
+ * Fireworks AI Transcription Service
+ * Provides speech-to-text transcription using Fireworks API
+ */
+
+export interface TranscriptionOptions {
+  model?: string
+  language?: string
+  apiKey: string
+}
+
+export interface TranscriptionResult {
+  text: string
+  duration?: number
+  language?: string
+}
+
+export class FireworksTranscriptionError extends Error {
+  constructor(
+    message: string,
+    public statusCode?: number,
+    public response?: any
+  ) {
+    super(message)
+    this.name = 'FireworksTranscriptionError'
+  }
+}
+
+/**
+ * Transcribe audio using Fireworks AI Whisper API
+ */
+export async function transcribeAudio(
+  audioBlob: Blob,
+  options: TranscriptionOptions
+): Promise<TranscriptionResult> {
+  const { model = 'whisper-v3', apiKey } = options
+
+  if (!apiKey) {
+    throw new FireworksTranscriptionError('Fireworks API key is required')
+  }
+
+  // Create form data
+  const formData = new FormData()
+  formData.append('file', audioBlob, 'recording.webm')
+  formData.append('model', model)
+
+  try {
+    const response = await fetch('https://api.fireworks.ai/inference/v1/audio/transcriptions', {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${apiKey}`
+      },
+      body: formData
+    })
+
+    if (!response.ok) {
+      const errorText = await response.text().catch(() => response.statusText)
+      throw new FireworksTranscriptionError(
+        `Transcription failed: ${errorText}`,
+        response.status,
+        errorText
+      )
+    }
+
+    const result = await response.json()
+
+    if (!result.text) {
+      throw new FireworksTranscriptionError('No transcription text in response', undefined, result)
+    }
+
+    return {
+      text: result.text,
+      duration: result.duration,
+      language: result.language
+    }
+  } catch (error) {
+    if (error instanceof FireworksTranscriptionError) {
+      throw error
+    }
+
+    // Handle network or other errors
+    const message = error instanceof Error ? error.message : 'Unknown error occurred'
+    throw new FireworksTranscriptionError(`Network error: ${message}`)
+  }
+}
+
+/**
+ * Check if Fireworks API key is configured
+ */
+export function hasApiKey(apiKey: string | null | undefined): apiKey is string {
+  return typeof apiKey === 'string' && apiKey.length > 0
+}