more cleanup for stt

Author: tschellenbach

agents-core/vision_agents/core/agents/agents.py

@@ -177,6 +177,7 @@ async def simple_response(
         """
         Overwrite simple_response if you want to change how the Agent class calls the LLM
         """
+        logger.info("asking LLM to reply to %s", text)
         with self.tracer.start_as_current_span("simple_response") as span:
             response = await self.llm.simple_response(
                 text=text, processors=self.processors, participant=participant
@@ -308,6 +309,7 @@ async def on_realtime_agent_speech_transcription(
 
         @self.events.subscribe
         async def on_stt_transcript_event_create_response(event: STTTranscriptEvent):
+            import pdb; pdb.set_trace()
             if self.realtime_mode or not self.llm:
                 # when running in realtime mode, there is no need to send the response to the LLM
                 return

docs/ai/instructions/ai-stt.md

@@ -8,16 +8,28 @@ class MySTT(stt.STT):
     def __init__(
         self,
         api_key: Optional[str] = None,
-        sample_rate: int = 48000,
-        client: Optional[AsyncDeepgramClient] = None,
+        client: Optional[MyClient] = None,
     ):
-        super().__init__(sample_rate=sample_rate)
+        super().__init__(provider_name="my_stt")
+        # be sure to allow the passing of the client object
+        # if client is not passed, create one
+        # pass the most common settings for the client in the init (like api key)
 
 
-    async def _process_audio_impl(
-        self, pcm_data: PcmData, user_metadata: Optional[Union[Dict[str, Any], Participant]] = None
-    ) -> Optional[List[Tuple[bool, str, Dict[str, Any]]]]:
-        pass
+    async def process_audio(
+        self,
+        pcm_data: PcmData,
+        participant: Optional[Participant] = None,
+    ):
+        parts = self.client.stt(pcm_data, stream=True)
+        full_text = ""
+        for part in parts:
+            # parts that aren't finished
+            self._emit_partial_transcript_event(part, participant, metadata)
+            full_text += part
+            
+        # the full text
+        self._emit_transcript_event(full_text, participant, metadata)
 
 ```
 

examples/01_simple_agent_example/simple_agent_example.py

@@ -33,8 +33,7 @@ async def start_agent() -> None:
     # Create a call
     call = agent.edge.client.video.call("default", str(uuid4()))
 
-    # Open the demo UI
-    await agent.edge.open_demo(call)
+
 
     # Have the agent join the call/room
     with await agent.join(call):
@@ -54,6 +53,8 @@ async def start_agent() -> None:
         # run till the call ends
         # await agent.say("Hello, how are you?")
         # await asyncio.sleep(5)
+        # Open the demo UI
+        await agent.edge.open_demo(call)
 
         await agent.simple_response("tell me something interesting in a short sentence")
         await agent.finish()

plugins/fish/vision_agents/plugins/fish/stt.py

@@ -152,8 +152,7 @@ async def process_audio(
                 },
             )
 
-            # Return as final result (Fish Audio doesn't support streaming/partial results)
-            return [(True, transcript_text, metadata)]
+            self._emit_transcript_event(transcript_text, participant, metadata)
 
         except Exception as e:
             logger.error(