feat: enable typed chat messages to reach LiveKit agent

- Add data_received event handler to agent worker
- Parse chat messages from data channel (type: 'chat')
- Ingest typed text to Hub for transcript persistence
- Use session.generate_reply() to inject typed text into conversation
- Agent responds to typed messages by speaking aloud
- Improve sendChatMessage() error handling with toast notifications

Author: iamh2o

apps/agent_worker/worker.py

@@ -247,6 +247,58 @@ def on_conversation_item_added(event: ConversationItemAddedEvent) -> None:
 
     session.on("conversation_item_added", on_conversation_item_added)
 
+    # Handle typed chat messages from data channel
+    import json as _json_dc
+
+    @ctx.room.on("data_received")
+    def on_data_received(data: rtc.DataPacket) -> None:
+        """Handle typed chat messages from participants.
+
+        When a user types a message in the chat UI, it's sent via LiveKit's
+        data channel. We parse it and inject it into the conversation as if
+        the user had spoken it.
+        """
+        try:
+            payload = data.data.decode("utf-8")
+            msg = _json_dc.loads(payload)
+
+            # Only handle chat messages
+            if msg.get("type") != "chat":
+                return
+
+            text = msg.get("text", "").strip()
+            if not text:
+                return
+
+            sender = data.participant.identity if data.participant else "user"
+            logger.info(f"Received typed chat from {sender}: {text[:50]}...")
+
+            # Ingest to Hub for persistence
+            hub_ingest_transcript(
+                space_id=space_id,
+                text=text,
+                role="user",
+                participant_identity=sender,
+            )
+
+            # Inject the typed message into the agent's conversation
+            # Use generate_reply with the user's text as instructions
+            asyncio.create_task(_handle_typed_message(text, sender))
+
+        except Exception as e:
+            logger.warning(f"Failed to process data channel message: {e}")
+
+    async def _handle_typed_message(text: str, sender: str) -> None:
+        """Process a typed chat message and generate a response."""
+        try:
+            # Use generate_reply to have the agent respond to the typed text
+            # The agent will speak the response aloud
+            await session.generate_reply(
+                instructions=f"The user '{sender}' typed this message (not spoken): {text}\n\nRespond naturally as if they had said it aloud."
+            )
+        except Exception as e:
+            logger.warning(f"Failed to generate reply for typed message: {e}")
+
     await session.start(
         room=ctx.room,
         agent=ForgeAssistant(),

functions/hub_api/templates/livekit_test.html

@@ -960,12 +960,30 @@ <h4 class="card-title"><i class="fas fa-closed-captioning"></i> Live Transcript<
 
   function sendChatMessage() {
     var text = (chatInputEl.value || '').trim();
-    if (!text || !room) return;
-    var msg = { type: 'chat', text: text, ts: Date.now() };
-    var data = new TextEncoder().encode(JSON.stringify(msg));
-    room.localParticipant.publishData(data, { reliable: true });
-    addChatMessage(myIdentity, text, msg.ts);
-    chatInputEl.value = '';
+    if (!text) {
+      return;
+    }
+    if (!room) {
+      Marvain.showToast('warning', 'Not Connected', 'Please join a room first');
+      log('Cannot send message: not connected to a room');
+      return;
+    }
+    if (room.state !== 'connected') {
+      Marvain.showToast('warning', 'Not Connected', 'Room state: ' + room.state);
+      log('Cannot send message: room state is ' + room.state);
+      return;
+    }
+    try {
+      var msg = { type: 'chat', text: text, ts: Date.now() };
+      var data = new TextEncoder().encode(JSON.stringify(msg));
+      room.localParticipant.publishData(data, { reliable: true });
+      addChatMessage(myIdentity, text, msg.ts);
+      chatInputEl.value = '';
+      log('Sent chat message: ' + text.substring(0, 30) + (text.length > 30 ? '...' : ''));
+    } catch (e) {
+      Marvain.showToast('error', 'Send Failed', e.message);
+      log('Failed to send message: ' + e.message);
+    }
   }
 
   async function join() {