Add transcription view and move controls to bottom

Author: lukasIO

app/page.tsx

@@ -2,6 +2,8 @@
 
 import { CloseIcon } from "@/components/CloseIcon";
 import { NoAgentNotification } from "@/components/NoAgentNotification";
+import TranscriptionView from "@/components/TranscriptionView";
+import useCombinedTranscriptions from "@/hooks/useCombinedTranscriptions";
 import {
   AgentState,
   BarVisualizer,
@@ -18,8 +20,6 @@ import { useCallback, useEffect, useState } from "react";
 import type { ConnectionDetails } from "./api/connection-details/route";
 
 export default function Page() {
-  const [agentState, setAgentState] = useState<AgentState>("disconnected");
-
   const [room] = useState(new Room());
 
   const onConnectButtonClicked = useCallback(async () => {
@@ -40,6 +40,7 @@ export default function Page() {
     const connectionDetailsData: ConnectionDetails = await response.json();
 
     await room.connect(connectionDetailsData.serverUrl, connectionDetailsData.participantToken);
+    await room.localParticipant.setMicrophoneEnabled(true);
   }, [room]);
 
   useEffect(() => {
@@ -53,36 +54,46 @@ export default function Page() {
   return (
     <main data-lk-theme="default" className="h-full grid content-center bg-[var(--lk-bg)]">
       <RoomContext.Provider value={room}>
-        <div className="lk-room-container grid grid-rows-[2fr_1fr] items-center">
-          <SimpleVoiceAssistant onStateChange={setAgentState} />
-          <ControlBar onConnectButtonClicked={onConnectButtonClicked} agentState={agentState} />
-          <RoomAudioRenderer />
-          <NoAgentNotification state={agentState} />
+        <div className="lk-room-container">
+          <SimpleVoiceAssistant onConnectButtonClicked={onConnectButtonClicked} />
         </div>
       </RoomContext.Provider>
     </main>
   );
 }
 
-function SimpleVoiceAssistant(props: { onStateChange: (state: AgentState) => void }) {
-  const { state, audioTrack } = useVoiceAssistant();
-  useEffect(() => {
-    props.onStateChange(state);
-  }, [props, state]);
+function SimpleVoiceAssistant(props: { onConnectButtonClicked: () => void }) {
+  const { state: agentState } = useVoiceAssistant();
   return (
-    <div className="h-[300px] max-w-[90vw] mx-auto">
-      <BarVisualizer
-        state={state}
-        barCount={5}
-        trackRef={audioTrack}
-        className="agent-visualizer"
-        options={{ minHeight: 24 }}
-      />
-    </div>
+    <>
+      <AnimatePresence>
+        {agentState === "disconnected" && (
+          <motion.button
+            initial={{ opacity: 0, top: 0 }}
+            animate={{ opacity: 1 }}
+            exit={{ opacity: 0, top: "-10px" }}
+            transition={{ duration: 1, ease: [0.09, 1.04, 0.245, 1.055] }}
+            className="uppercase absolute left-1/2 -translate-x-1/2 px-4 py-2 bg-white text-black rounded-md"
+            onClick={() => props.onConnectButtonClicked()}
+          >
+            Start a conversation
+          </motion.button>
+        )}
+        <div className="w-3/4 lg:w-1/2 mx-auto">
+          <TranscriptionView />
+        </div>
+      </AnimatePresence>
+
+      <RoomAudioRenderer />
+      <NoAgentNotification state={agentState} />
+      <div className="fixed bottom-0 w-full">
+        <ControlBar onConnectButtonClicked={props.onConnectButtonClicked} />
+      </div>
+    </>
   );
 }
 
-function ControlBar(props: { onConnectButtonClicked: () => void; agentState: AgentState }) {
+function ControlBar(props: { onConnectButtonClicked: () => void }) {
   /**
    * Use Krisp background noise reduction when available.
    * Note: This is only available on Scale plan, see {@link https://livekit.io/pricing | LiveKit Pricing} for more details.
@@ -92,35 +103,32 @@ function ControlBar(props: { onConnectButtonClicked: () => void; agentState: Age
     krisp.setNoiseFilterEnabled(true);
   }, []);
 
+  const { state: agentState, audioTrack } = useVoiceAssistant();
+
   return (
     <div className="relative h-[100px]">
       <AnimatePresence>
-        {props.agentState === "disconnected" && (
-          <motion.button
-            initial={{ opacity: 0, top: 0 }}
-            animate={{ opacity: 1 }}
-            exit={{ opacity: 0, top: "-10px" }}
-            transition={{ duration: 1, ease: [0.09, 1.04, 0.245, 1.055] }}
-            className="uppercase absolute left-1/2 -translate-x-1/2 px-4 py-2 bg-white text-black rounded-md"
-            onClick={() => props.onConnectButtonClicked()}
-          >
-            Start a conversation
-          </motion.button>
-        )}
-      </AnimatePresence>
-      <AnimatePresence>
-        {props.agentState !== "disconnected" && props.agentState !== "connecting" && (
+        {agentState !== "disconnected" && agentState !== "connecting" && (
           <motion.div
             initial={{ opacity: 0, top: "10px" }}
             animate={{ opacity: 1, top: 0 }}
             exit={{ opacity: 0, top: "-10px" }}
             transition={{ duration: 0.4, ease: [0.09, 1.04, 0.245, 1.055] }}
-            className="flex h-8 absolute left-1/2 -translate-x-1/2  justify-center"
+            className="flex absolute w-full h-full justify-between px-8 sm:px-4"
           >
-            <VoiceAssistantControlBar controls={{ leave: false }} />
-            <DisconnectButton>
-              <CloseIcon />
-            </DisconnectButton>
+            <BarVisualizer
+              state={agentState}
+              barCount={5}
+              trackRef={audioTrack}
+              className="agent-visualizer w-32 gap-2"
+              options={{ minHeight: 12 }}
+            />
+            <div className="flex items-center">
+              <VoiceAssistantControlBar controls={{ leave: false }} />
+              <DisconnectButton>
+                <CloseIcon />
+              </DisconnectButton>
+            </div>
           </motion.div>
         )}
       </AnimatePresence>

components/TranscriptionView.tsx

@@ -0,0 +1,22 @@
+import useCombinedTranscriptions from "@/hooks/useCombinedTranscriptions";
+
+export default function TranscriptionView() {
+  const combinedTranscriptions = useCombinedTranscriptions();
+
+  return (
+    <div className="flex flex-col gap-2 overflow-y-auto">
+      {combinedTranscriptions.map((segment) => (
+        <div
+          key={segment.id}
+          className={
+            segment.role === "assistant"
+              ? "p-2 self-start fit-content"
+              : "bg-gray-800 rounded-md p-2 self-end fit-content"
+          }
+        >
+          {segment.text}
+        </div>
+      ))}
+    </div>
+  );
+}

hooks/useCombinedTranscriptions.ts

@@ -0,0 +1,23 @@
+import { useTrackTranscription, useVoiceAssistant } from "@livekit/components-react";
+import { useMemo } from "react";
+import useLocalMicTrack from "./useLocalMicTrack";
+
+export default function useCombinedTranscriptions() {
+  const { agentTranscriptions } = useVoiceAssistant();
+
+  const micTrackRef = useLocalMicTrack();
+  const { segments: userTranscriptions } = useTrackTranscription(micTrackRef);
+
+  const combinedTranscriptions = useMemo(() => {
+    return [
+      ...agentTranscriptions.map((val) => {
+        return { ...val, role: "assistant" };
+      }),
+      ...userTranscriptions.map((val) => {
+        return { ...val, role: "user" };
+      }),
+    ].sort((a, b) => a.firstReceivedTime - b.firstReceivedTime);
+  }, [agentTranscriptions, userTranscriptions]);
+
+  return combinedTranscriptions;
+}

hooks/useLocalMicTrack.ts

@@ -0,0 +1,17 @@
+import { TrackReferenceOrPlaceholder, useLocalParticipant } from "@livekit/components-react";
+import { Track } from "livekit-client";
+import { useMemo } from "react";
+
+export default function useLocalMicTrack() {
+  const { microphoneTrack, localParticipant } = useLocalParticipant();
+
+  const micTrackRef: TrackReferenceOrPlaceholder = useMemo(() => {
+    return {
+      participant: localParticipant,
+      source: Track.Source.Microphone,
+      publication: microphoneTrack,
+    };
+  }, [localParticipant, microphoneTrack]);
+
+  return micTrackRef;
+}