diff --git a/app/frontend/src/App.tsx b/app/frontend/src/App.tsx index ebfde279..56904720 100644 --- a/app/frontend/src/App.tsx +++ b/app/frontend/src/App.tsx @@ -3,24 +3,27 @@ import { Mic, MicOff } from "lucide-react"; import { useTranslation } from "react-i18next"; import { Button } from "@/components/ui/button"; -import { GroundingFiles } from "@/components/ui/grounding-files"; import GroundingFileView from "@/components/ui/grounding-file-view"; import StatusMessage from "@/components/ui/status-message"; +import HistoryPanel from "@/components/ui/history-panel"; import useRealTime from "@/hooks/useRealtime"; import useAudioRecorder from "@/hooks/useAudioRecorder"; import useAudioPlayer from "@/hooks/useAudioPlayer"; -import { GroundingFile, ToolResult } from "./types"; +import { GroundingFile, HistoryItem, ToolResult } from "./types"; import logo from "./assets/logo.svg"; function App() { const [isRecording, setIsRecording] = useState(false); - const [groundingFiles, setGroundingFiles] = useState([]); const [selectedFile, setSelectedFile] = useState(null); + const [groundingFiles, setGroundingFiles] = useState([]); + const [showTranscript, setShowTranscript] = useState(false); + const [history, setHistory] = useState([]); const { startSession, addUserAudio, inputAudioBufferClear } = useRealTime({ + enableInputAudioTranscription: true, // Enable input audio transcription from the user to show in the history onWebSocketOpen: () => console.log("WebSocket connection opened"), onWebSocketClose: () => console.log("WebSocket connection closed"), onWebSocketError: event => console.error("WebSocket error:", event), @@ -33,12 +36,39 @@ function App() { }, onReceivedExtensionMiddleTierToolResponse: message => { const result: ToolResult = JSON.parse(message.tool_result); - const files: GroundingFile[] = result.sources.map(x => { return { id: x.chunk_id, name: x.title, content: x.chunk }; }); - setGroundingFiles(prev => [...prev, ...files]); + setGroundingFiles(files); // Store the grounding files for the assistant + }, + onReceivedInputAudioTranscriptionCompleted: message => { + // Update history with input audio transcription when completed + const newHistoryItem: HistoryItem = { + id: message.event_id, + transcript: message.transcript, + groundingFiles: [], + sender: "user", + timestamp: new Date() // Add timestamp + }; + setHistory(prev => [...prev, newHistoryItem]); + }, + onReceivedResponseDone: message => { + const transcript = message.response.output.map(output => output.content?.map(content => content.transcript).join(" ")).join(" "); + if (!transcript) { + return; + } + + // Update history with response done + const newHistoryItem: HistoryItem = { + id: message.event_id, + transcript: transcript, + groundingFiles: groundingFiles, + sender: "assistant", + timestamp: new Date() // Add timestamp + }; + setHistory(prev => [...prev, newHistoryItem]); + setGroundingFiles([]); // Clear the assistant grounding files after use } }); @@ -91,7 +121,15 @@ function App() { - +
+ +
@@ -99,6 +137,8 @@ function App() {
setSelectedFile(null)} /> + + setShowTranscript(false)} onSelectedGroundingFile={setSelectedFile} /> ); } diff --git a/app/frontend/src/components/ui/history-panel.tsx b/app/frontend/src/components/ui/history-panel.tsx index ccb4ec69..d4c12c54 100644 --- a/app/frontend/src/components/ui/history-panel.tsx +++ b/app/frontend/src/components/ui/history-panel.tsx @@ -1,13 +1,13 @@ +import { useEffect, useRef, useState, memo } from "react"; import { AnimatePresence, motion } from "framer-motion"; import { X } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { Button } from "./button"; import GroundingFile from "./grounding-file"; import { GroundingFile as GroundingFileType, HistoryItem } from "@/types"; -import { useTranslation } from "react-i18next"; - type Properties = { history: HistoryItem[]; show: boolean; @@ -15,8 +15,43 @@ type Properties = { onSelectedGroundingFile: (file: GroundingFileType) => void; }; -export default function HistoryPanel({ show, history, onClosed, onSelectedGroundingFile }: Properties) { +const HistoryPanel = ({ show, history, onClosed, onSelectedGroundingFile }: Properties) => { const { t } = useTranslation(); + const historyEndRef = useRef(null); + const [currentTime, setCurrentTime] = useState(new Date()); + + // Scroll to the bottom whenever the history changes + useEffect(() => { + if (historyEndRef.current) { + historyEndRef.current.scrollIntoView({ behavior: "smooth" }); + } + }, [history]); + + // Update current time every second + useEffect(() => { + const interval = setInterval(() => { + setCurrentTime(new Date()); + }, 1000); + return () => clearInterval(interval); + }, []); + + const formatTimestamp = (timestamp: Date) => { + const options: Intl.DateTimeFormatOptions = { + hour: "numeric", + minute: "numeric", + hour12: true + }; + return new Intl.DateTimeFormat(navigator.language, options).format(timestamp); + }; + + const shouldShowTimestamp = (current: Date, next?: Date) => { + const nextTime = next ? next.getTime() : currentTime.getTime(); + const diff = (nextTime - current.getTime()) / 1000; // Difference in seconds + + return diff > 60; // Show timestamp if more than 60 seconds have passed + }; + + const MemoizedGroundingFile = memo(GroundingFile); return ( @@ -28,27 +63,37 @@ export default function HistoryPanel({ show, history, onClosed, onSelectedGround transition={{ type: "spring", stiffness: 300, damping: 30 }} className="fixed inset-y-0 right-0 z-40 w-full overflow-y-auto bg-white shadow-lg sm:w-96" > +
+

{t("history.transcriptHistory")}

+ +
-
-

{t("history.answerHistory")}

- -
{history.length > 0 ? ( - history.map((item, index) => ( -
-

{item.id}

- 
-                                        {item.transcript}
-
-
- {item.groundingFiles.map((file, index) => ( - onSelectedGroundingFile(file)} /> - ))} -
-
- )) +
+ {history.map((item, index) => { + const nextItem = history[index + 1]; + const showTimestamp = shouldShowTimestamp(item.timestamp, nextItem ? nextItem.timestamp : undefined); + return ( +
+
+

{item.transcript}

+
+ {item.groundingFiles?.map((file, index) => ( + onSelectedGroundingFile(file)} /> + ))} +
+
+ {showTimestamp &&
{formatTimestamp(item.timestamp)}
} +
+ ); + })} +
+
) : (

{t("history.noHistory")}

)} @@ -57,4 +102,6 @@ export default function HistoryPanel({ show, history, onClosed, onSelectedGround )} ); -} +}; + +export default HistoryPanel; diff --git a/app/frontend/src/locales/en/translation.json b/app/frontend/src/locales/en/translation.json index 58524d62..b988029e 100644 --- a/app/frontend/src/locales/en/translation.json +++ b/app/frontend/src/locales/en/translation.json @@ -2,6 +2,7 @@ "app": { "title": "Talk to your data", "footer": "Built with Azure AI Search + Azure OpenAI", + "showTranscript": "Show transcript", "stopRecording": "Stop recording", "startRecording": "Start recording", "stopConversation": "Stop conversation" @@ -11,7 +12,7 @@ "conversationInProgress": "Conversation in progress" }, "history": { - "answerHistory": "Answer history", + "transcriptHistory": "Transcript history", "noHistory": "No history yet." }, "groundingFiles": { diff --git a/app/frontend/src/locales/es/translation.json b/app/frontend/src/locales/es/translation.json index b0a51c11..1955f94d 100644 --- a/app/frontend/src/locales/es/translation.json +++ b/app/frontend/src/locales/es/translation.json @@ -2,6 +2,7 @@ "app": { "title": "Habla con tus datos", "footer": "Creado con Azure AI Search + Azure OpenAI", + "showTranscript": "Mostrar transcripción", "stopRecording": "Detener grabación", "startRecording": "Comenzar grabación", "stopConversation": "Detener conversación" @@ -11,7 +12,7 @@ "conversationInProgress": "Conversación en progreso" }, "history": { - "answerHistory": "Historial de respuestas", + "transcriptHistory": "Historial de transcripciones", "noHistory": "Aún no hay historial." }, "groundingFiles": { diff --git a/app/frontend/src/locales/fr/translation.json b/app/frontend/src/locales/fr/translation.json index 3e381b62..76c47e0f 100644 --- a/app/frontend/src/locales/fr/translation.json +++ b/app/frontend/src/locales/fr/translation.json @@ -2,6 +2,7 @@ "app": { "title": "Parlez à vos données", "footer": "Créée avec Azure AI Search + Azure OpenAI", + "showTranscript": "Afficher la transcription", "stopRecording": "Arrêter l'enregistrement", "startRecording": "Commencer l'enregistrement", "stopConversation": "Arrêter la conversation" @@ -11,7 +12,7 @@ "conversationInProgress": "Conversation en cours" }, "history": { - "answerHistory": "Historique des réponses", + "transcriptHistory": "Historique de la transcription", "noHistory": "Pas encore d'historique." }, "groundingFiles": { diff --git a/app/frontend/src/locales/ja/translation.json b/app/frontend/src/locales/ja/translation.json index 0f332e02..8ba22730 100644 --- a/app/frontend/src/locales/ja/translation.json +++ b/app/frontend/src/locales/ja/translation.json @@ -2,6 +2,7 @@ "app": { "title": "データと話す", "footer": "Azure AI Search + Azure OpenAI で構築", + "showTranscript": "トランスクリプトを表示", "stopRecording": "録音を停止", "startRecording": "録音を開始", "stopConversation": "会話を停止" @@ -11,7 +12,7 @@ "conversationInProgress": "会話が進行中" }, "history": { - "answerHistory": "回答履歴", + "transcriptHistory": "トランスクリプト履歴", "noHistory": "まだ履歴はありません。" }, "groundingFiles": { diff --git a/app/frontend/src/types.ts b/app/frontend/src/types.ts index bef21a02..2c6aaa53 100644 --- a/app/frontend/src/types.ts +++ b/app/frontend/src/types.ts @@ -1,20 +1,28 @@ +// Represents a grounding file export type GroundingFile = { id: string; name: string; content: string; }; +// Represents an item in the history export type HistoryItem = { id: string; transcript: string; - groundingFiles: GroundingFile[]; + groundingFiles?: GroundingFile[]; + sender: "user" | "assistant"; + timestamp: Date; // Add timestamp field }; +// Represents a command to update the session export type SessionUpdateCommand = { type: "session.update"; session: { turn_detection?: { type: "server_vad" | "none"; + threshold?: number; + prefix_padding_ms?: number; + silence_duration_ms?: number; }; input_audio_transcription?: { model: "whisper-1"; @@ -22,29 +30,35 @@ export type SessionUpdateCommand = { }; }; +// Represents a command to append audio to the input buffer export type InputAudioBufferAppendCommand = { type: "input_audio_buffer.append"; - audio: string; + audio: string; // Ensure this is a valid base64-encoded string }; +// Represents a command to clear the input audio buffer export type InputAudioBufferClearCommand = { type: "input_audio_buffer.clear"; }; +// Represents a generic message export type Message = { type: string; }; +// Represents a response containing an audio delta export type ResponseAudioDelta = { type: "response.audio.delta"; delta: string; }; +// Represents a response containing an audio transcript delta export type ResponseAudioTranscriptDelta = { type: "response.audio_transcript.delta"; delta: string; }; +// Represents a response indicating that input audio transcription is completed export type ResponseInputAudioTranscriptionCompleted = { type: "conversation.item.input_audio_transcription.completed"; event_id: string; @@ -53,6 +67,7 @@ export type ResponseInputAudioTranscriptionCompleted = { transcript: string; }; +// Represents a response indicating that the response is done export type ResponseDone = { type: "response.done"; event_id: string; @@ -62,6 +77,7 @@ export type ResponseDone = { }; }; +// Represents a response from an extension middle tier tool export type ExtensionMiddleTierToolResponse = { type: "extension.middle_tier_tool.response"; previous_item_id: string; @@ -69,6 +85,7 @@ export type ExtensionMiddleTierToolResponse = { tool_result: string; // JSON string that needs to be parsed into ToolResult }; +// Represents the result from a tool export type ToolResult = { sources: { chunk_id: string; title: string; chunk: string }[]; };