Bugfix gemini (#150)

* fix gemini framerate

* wip

Author: tschellenbach

agents-core/vision_agents/core/utils/audio_forwarder.py

@@ -50,6 +50,7 @@ async def _reader(self):
         """Read audio frames from track and forward to callback."""
         while True:
             try:
+
                 received = await asyncio.wait_for(self.track.recv(), timeout=1.0)
                 frame = cast(av.AudioFrame, received)
 

examples/other_examples/openai_realtime_webrtc/openai_realtime_example.py

@@ -55,8 +55,6 @@ async def join_call(agent: Agent, call_type: str, call_id: str, **kwargs) -> Non
         #TODO: should open demo be done by the CLI instead of the example?
         await agent.edge.open_demo(call)
         logger.info("LLM ready")
-        # await agent.llm.request_session_info()
-        logger.info("Requested session info")
         # Wait for a human to join the call before greeting
         logger.info("Waiting for human to join the call")
         await agent.llm.simple_response(text="Please greet the user.")

plugins/gemini/vision_agents/plugins/gemini/gemini_realtime.py

@@ -147,12 +147,12 @@ async def simple_audio_response(
             return
 
         self._current_participant = participant
-        self.logger.debug(f"Sending audio to gemini: {pcm.duration}")
+
         # Build blob and send directly
         audio_bytes = pcm.resample(
             target_sample_rate=16000, target_channels=1
         ).samples.tobytes()
-        mime = f"audio/pcm;rate={pcm.sample_rate}"
+        mime = f"audio/pcm;rate=16000"
         blob = Blob(data=audio_bytes, mime_type=mime)
 
         await self._require_session().send_realtime_input(audio=blob)

plugins/openai/vision_agents/plugins/openai/openai_realtime.py

@@ -6,7 +6,7 @@
     RealtimeSessionCreateRequestParam,
     ResponseAudioTranscriptDoneEvent,
     InputAudioBufferSpeechStartedEvent,
-    ConversationItemInputAudioTranscriptionCompletedEvent,
+    ConversationItemInputAudioTranscriptionCompletedEvent, SessionUpdatedEvent, ResponseCreatedEvent, ResponseDoneEvent,
 )
 
 from vision_agents.core.llm import realtime
@@ -240,6 +240,20 @@ async def _handle_openai_event(self, event: dict) -> None:
         elif et == "response.tool_call":
             # Handle tool calls from OpenAI realtime
             await self._handle_tool_call_event(event)
+        elif et == "response.created":
+            e = ResponseCreatedEvent(**event)
+            pass
+        elif et == "response.done":
+            logger.info("OpenAI response done %s", event)
+            e = ResponseDoneEvent(**event)
+
+            if e.response.status == "failed":
+                raise Exception("OpenAI realtime failure %s", e.response)
+        elif et == "session.updated":
+            pass
+            #e = SessionUpdatedEvent(**event)
+        else:
+            logger.info(f"Unrecognized OpenAI Realtime event: {et} {event}")
 
     async def _handle_audio_output(self, pcm: PcmData) -> None:
         """Process audio output received from the OpenAI API.

plugins/openai/vision_agents/plugins/openai/rtc_manager.py

@@ -46,6 +46,9 @@ def __init__(self, model: str, voice: str, send_video: bool):
         self.pc = RTCPeerConnection()
         self.data_channel: Optional[RTCDataChannel] = None
 
+        # Set up connection event handlers
+        self._setup_connection_logging()
+
         # on this track we send audio to openAI
         self._audio_to_openai_track: QueuedAudioTrack = QueuedAudioTrack(
             sample_rate=48000
@@ -62,6 +65,51 @@ def __init__(self, model: str, voice: str, send_video: bool):
 
         self.instructions: Optional[str] = None
 
+    def _setup_connection_logging(self) -> None:
+        """Set up event handlers for connection monitoring and error logging."""
+
+        @self.pc.on("connectionstatechange")
+        async def on_connectionstatechange():
+            state = self.pc.connectionState
+            logger.info(f"🔗 RTC connection state changed: {state}")
+            if state == "failed":
+                logger.error("❌ RTC connection failed")
+            elif state == "disconnected":
+                logger.warning("⚠️ RTC connection disconnected")
+            elif state == "connected":
+                logger.info("✅ RTC connection established")
+            elif state == "closed":
+                logger.info("🔒 RTC connection closed")
+
+        @self.pc.on("iceconnectionstatechange")
+        async def on_iceconnectionstatechange():
+            state = self.pc.iceConnectionState
+            logger.info(f"🧊 ICE connection state: {state}")
+            if state == "failed":
+                logger.error("❌ ICE connection failed")
+            elif state == "disconnected":
+                logger.warning("⚠️ ICE connection disconnected")
+            elif state == "connected":
+                logger.info("✅ ICE connection established")
+            elif state == "checking":
+                logger.debug("🔍 ICE checking candidates...")
+
+        @self.pc.on("icegatheringstatechange")
+        async def on_icegatheringstatechange():
+            state = self.pc.iceGatheringState
+            logger.debug(f"🧊 ICE gathering state: {state}")
+            if state == "complete":
+                logger.info("✅ ICE gathering complete")
+
+        @self.pc.on("signalingstatechange")
+        async def on_signalingstatechange():
+            state = self.pc.signalingState
+            logger.debug(f"📡 Signaling state: {state}")
+
+        @self.pc.on("datachannel")
+        async def on_datachannel(channel):
+            logger.info(f"📨 Remote data channel created: {channel.label}")
+
     async def connect(self) -> None:
         """Establish WebRTC connection to OpenAI's Realtime API.
 
@@ -81,7 +129,6 @@ async def on_track(track):
             logger.info("receiving track from openai")
             if track.kind == "audio":
                 track = cast(AudioStreamTrack, track)
-                logger.info("Remote audio track attached; starting audio forwarder")
                 if self._audio_callback:
                     audio_forwarder = AudioForwarder(track, self._audio_callback)
                     await audio_forwarder.start()