Add preview text and final spoken text by agent to the TTS_EVENT

Author: kPatch

coffee_ws/src/coffee_voice_agent/scripts/agents/coffee_barista_agent.py

@@ -86,8 +86,14 @@ def __init__(self):
     async def tts_node(self, text, model_settings=None):
         """Override TTS node to process delimiter-based responses (emotion:text) with minimal buffering"""
 
+        # Initialize text tracking for TTS events
+        self.state_manager.current_speech_preview = ""
+        self.state_manager.current_speech_full_text = ""
+        preview_set = False
+        
         # Process text stream with minimal buffering for emotion extraction
         async def process_text_stream():
+            nonlocal preview_set
             first_chunk_buffer = ""
             emotion_extracted = False
             emotion_check_limit = 50  # Only check first 50 characters for emotion delimiter
@@ -130,6 +136,11 @@ async def process_text_stream():
                         # Immediately yield the text part (no more buffering)
                         if text_after_delimiter.strip():
                             logger.info(f"💬 TTS streaming text immediately: {text_after_delimiter[:30]}{'...' if len(text_after_delimiter) > 30 else ''}")
+                            # Accumulate text and set preview
+                            self.state_manager.current_speech_full_text += text_after_delimiter
+                            if not preview_set:
+                                self.state_manager.current_speech_preview = text_after_delimiter[:50] + "..." if len(text_after_delimiter) > 50 else text_after_delimiter
+                                preview_set = True
                             yield text_after_delimiter
                         else:
                             logger.warning("🔍 DEBUG: text_after_delimiter is empty or whitespace - nothing to yield!")
@@ -149,6 +160,11 @@ async def process_text_stream():
 
                         # Yield the buffered content immediately
                         logger.info(f"💬 TTS fallback streaming: {first_chunk_buffer[:30]}{'...' if len(first_chunk_buffer) > 30 else ''}")
+                        # Accumulate text and set preview
+                        self.state_manager.current_speech_full_text += first_chunk_buffer
+                        if not preview_set:
+                            self.state_manager.current_speech_preview = first_chunk_buffer[:50] + "..." if len(first_chunk_buffer) > 50 else first_chunk_buffer
+                            preview_set = True
                         yield first_chunk_buffer
 
                     # If we haven't extracted emotion yet and haven't hit limit, continue buffering
@@ -157,6 +173,11 @@ async def process_text_stream():
                 else:
                     # Either emotion already extracted, or we're past the check limit
                     # Stream everything immediately
+                    # Accumulate text and set preview if not set
+                    self.state_manager.current_speech_full_text += text_chunk
+                    if not preview_set:
+                        self.state_manager.current_speech_preview = text_chunk[:50] + "..." if len(text_chunk) > 50 else text_chunk
+                        preview_set = True
                     yield text_chunk
 
         # Process the text stream and pass clean text to default TTS

coffee_ws/src/coffee_voice_agent/scripts/state/state_manager.py

@@ -42,6 +42,10 @@ def __init__(self, agent=None):
         self.current_emotion = "waiting"  # Track current emotional state
         self.emotion_history = []  # Log emotional journey
         self.ending_conversation = False  # Flag to prevent timer conflicts during goodbye
+        
+        # Text tracking for TTS events
+        self.current_speech_preview = ""  # Preview text for "started" events
+        self.current_speech_full_text = ""  # Accumulated full text for "finished" events
         self.virtual_request_queue = []  # Queue for virtual coffee requests
         self.announcing_virtual_request = False  # Flag to prevent conflicts during announcements
         self.recent_greetings = []  # Track recent greetings to avoid repetition
@@ -264,11 +268,15 @@ async def handle_state_change():
                         if event.new_state == "speaking":
                             logger.info("🔍 DEBUG: Agent started speaking - sending TTS started event")
                             current_emotion = self.current_emotion
-                            await self._send_tts_event("started", "Agent Response", current_emotion, "session")
+                            # Use preview text for started event
+                            text_to_send = self.current_speech_preview or "Agent Response"
+                            await self._send_tts_event("started", text_to_send, current_emotion, "session")
                         elif event.old_state == "speaking" and event.new_state != "speaking":
                             logger.info("🔍 DEBUG: Agent stopped speaking - sending TTS finished event")
                             current_emotion = self.current_emotion
-                            await self._send_tts_event("finished", "Agent Response", current_emotion, "session")
+                            # Use full accumulated text for finished event
+                            text_to_send = self.current_speech_full_text or "Agent Response"
+                            await self._send_tts_event("finished", text_to_send, current_emotion, "session")
                     except Exception as e:
                         logger.error(f"Error handling agent state change TTS events: {e}")
 
@@ -502,6 +510,10 @@ async def say_with_emotion(self, text: str, emotion: str = None):
         logger.info(f"🔍 DEBUG: say_with_emotion emotion: {emotion}")
 
         if self.session:
+            # Store text for TTS events
+            self.current_speech_preview = text[:50] + "..." if len(text) > 50 else text
+            self.current_speech_full_text = text
+            
             # Send TTS_STARTED event - COMMENTED OUT to prevent duplicates (using agent_state_changed instead)
             # logger.info("🔍 DEBUG: About to send TTS_STARTED event")
             # await self._send_tts_event("started", text, emotion or self.current_emotion, "manual")