diff --git a/src/agents/realtime/__init__.py b/src/agents/realtime/__init__.py index 7675c466f..f4b7621bc 100644 --- a/src/agents/realtime/__init__.py +++ b/src/agents/realtime/__init__.py @@ -59,6 +59,7 @@ RealtimeModelErrorEvent, RealtimeModelEvent, RealtimeModelExceptionEvent, + RealtimeModelInputAudioBufferTimeoutEvent, RealtimeModelInputAudioTranscriptionCompletedEvent, RealtimeModelItemDeletedEvent, RealtimeModelItemUpdatedEvent, @@ -152,6 +153,7 @@ "RealtimeModelErrorEvent", "RealtimeModelEvent", "RealtimeModelExceptionEvent", + "RealtimeModelInputAudioBufferTimeoutEvent", "RealtimeModelInputAudioTranscriptionCompletedEvent", "RealtimeModelItemDeletedEvent", "RealtimeModelItemUpdatedEvent", diff --git a/src/agents/realtime/config.py b/src/agents/realtime/config.py index fdbc19074..41da8d7ae 100644 --- a/src/agents/realtime/config.py +++ b/src/agents/realtime/config.py @@ -97,6 +97,9 @@ class RealtimeSessionModelSettings(TypedDict): speed: NotRequired[float] """The speed of the model's responses.""" + idle_timeout_ms: NotRequired[int] + """The idle timeout before the session is closed, in milliseconds.""" + input_audio_format: NotRequired[RealtimeAudioFormat] """The format for input audio streams.""" diff --git a/src/agents/realtime/model_events.py b/src/agents/realtime/model_events.py index 5aeadc0f9..fbcc1a12e 100644 --- a/src/agents/realtime/model_events.py +++ b/src/agents/realtime/model_events.py @@ -75,6 +75,18 @@ class RealtimeModelAudioDoneEvent: type: Literal["audio_done"] = "audio_done" +@dataclass +class RealtimeModelInputAudioBufferTimeoutEvent: + """Triggered when the input audio buffer times out due to inactivity.""" + + event_id: str + audio_start_ms: int + audio_end_ms: int + item_id: str + + type: Literal["input_audio_buffer.timeout_triggered"] = "input_audio_buffer.timeout_triggered" + + @dataclass class RealtimeModelInputAudioTranscriptionCompletedEvent: """Input audio transcription completed.""" @@ -174,6 +186,7 @@ class RealtimeModelRawServerEvent: RealtimeModelAudioEvent, RealtimeModelAudioInterruptedEvent, RealtimeModelAudioDoneEvent, + RealtimeModelInputAudioBufferTimeoutEvent, RealtimeModelInputAudioTranscriptionCompletedEvent, RealtimeModelTranscriptDeltaEvent, RealtimeModelItemUpdatedEvent, diff --git a/src/agents/realtime/openai_realtime.py b/src/agents/realtime/openai_realtime.py index bbeda20f1..3dfde94da 100644 --- a/src/agents/realtime/openai_realtime.py +++ b/src/agents/realtime/openai_realtime.py @@ -83,6 +83,7 @@ RealtimeModelErrorEvent, RealtimeModelEvent, RealtimeModelExceptionEvent, + RealtimeModelInputAudioBufferTimeoutEvent, RealtimeModelInputAudioTranscriptionCompletedEvent, RealtimeModelItemDeletedEvent, RealtimeModelItemUpdatedEvent, @@ -459,6 +460,16 @@ async def _cancel_response(self) -> None: async def _handle_ws_event(self, event: dict[str, Any]): await self._emit_event(RealtimeModelRawServerEvent(data=event)) + if isinstance(event, dict) and event.get("type") == "input_audio_buffer.timeout_triggered": + await self._emit_event( + RealtimeModelInputAudioBufferTimeoutEvent( + event_id=event["event_id"], + audio_start_ms=event["audio_start_ms"], + audio_end_ms=event["audio_end_ms"], + item_id=event["item_id"], + ) + ) + return try: if "previous_item_id" in event and event["previous_item_id"] is None: event["previous_item_id"] = "" # TODO (rm) remove @@ -580,6 +591,7 @@ def _get_session_config( ), voice=model_settings.get("voice", DEFAULT_MODEL_SETTINGS.get("voice")), speed=model_settings.get("speed", None), + idle_timeout_ms=model_settings.get("idle_timeout_ms", None), modalities=model_settings.get("modalities", DEFAULT_MODEL_SETTINGS.get("modalities")), input_audio_format=model_settings.get( "input_audio_format", diff --git a/src/agents/realtime/session.py b/src/agents/realtime/session.py index 42d61cf2b..7be16a685 100644 --- a/src/agents/realtime/session.py +++ b/src/agents/realtime/session.py @@ -337,6 +337,8 @@ async def on_event(self, event: RealtimeModelEvent) -> None: elif event.type == "exception": # Store the exception to be raised in __aiter__ self._stored_exception = event.exception + elif event.type == "input_audio_buffer.timeout_triggered": + pass elif event.type == "other": pass elif event.type == "raw_server_event": diff --git a/tests/realtime/test_openai_realtime.py b/tests/realtime/test_openai_realtime.py index 4c410bf6e..a4fda4581 100644 --- a/tests/realtime/test_openai_realtime.py +++ b/tests/realtime/test_openai_realtime.py @@ -8,6 +8,7 @@ from agents.realtime.model_events import ( RealtimeModelAudioEvent, RealtimeModelErrorEvent, + RealtimeModelInputAudioBufferTimeoutEvent, RealtimeModelToolCallEvent, ) from agents.realtime.openai_realtime import OpenAIRealtimeWebSocketModel @@ -30,6 +31,15 @@ def mock_websocket(self): return mock_ws +class TestSessionConfig(TestOpenAIRealtimeWebSocketModel): + """Test session configuration helpers.""" + + def test_get_session_config_includes_idle_timeout_ms(self, model): + """Test that _get_session_config includes idle_timeout_ms when provided.""" + session_obj = model._get_session_config({"idle_timeout_ms": 1234}) + assert session_obj.idle_timeout_ms == 1234 + + class TestConnectionLifecycle(TestOpenAIRealtimeWebSocketModel): """Test connection establishment, configuration, and error handling.""" @@ -219,6 +229,32 @@ async def test_handle_unknown_event_type_ignored(self, model): # unknown, it should be ignored pass + @pytest.mark.asyncio + async def test_handle_idle_timeout_event_emits_timeout_event(self, model): + """Test that input audio buffer timeouts are passed through.""" + mock_listener = AsyncMock() + model.add_listener(mock_listener) + + timeout_event = { + "type": "input_audio_buffer.timeout_triggered", + "event_id": "evt_123", + "audio_start_ms": 100, + "audio_end_ms": 200, + "item_id": "item_1", + } + + await model._handle_ws_event(timeout_event) + + assert mock_listener.on_event.call_count == 2 + raw_event = mock_listener.on_event.call_args_list[0][0][0] + assert raw_event.type == "raw_server_event" + timeout = mock_listener.on_event.call_args_list[1][0][0] + assert isinstance(timeout, RealtimeModelInputAudioBufferTimeoutEvent) + assert timeout.event_id == timeout_event["event_id"] + assert timeout.audio_start_ms == timeout_event["audio_start_ms"] + assert timeout.audio_end_ms == timeout_event["audio_end_ms"] + assert timeout.item_id == timeout_event["item_id"] + @pytest.mark.asyncio async def test_handle_audio_delta_event_success(self, model): """Test successful handling of audio delta events.""" diff --git a/tests/realtime/test_session.py b/tests/realtime/test_session.py index cd562c522..c4fcce797 100644 --- a/tests/realtime/test_session.py +++ b/tests/realtime/test_session.py @@ -37,6 +37,7 @@ RealtimeModelAudioInterruptedEvent, RealtimeModelConnectionStatusEvent, RealtimeModelErrorEvent, + RealtimeModelInputAudioBufferTimeoutEvent, RealtimeModelInputAudioTranscriptionCompletedEvent, RealtimeModelItemDeletedEvent, RealtimeModelItemUpdatedEvent, @@ -367,6 +368,25 @@ async def test_ignored_events_only_generate_raw_events(self, mock_model, mock_ag event = await session._event_queue.get() assert isinstance(event, RealtimeRawModelEvent) + @pytest.mark.asyncio + async def test_idle_timeout_event_forwarded(self, mock_model, mock_agent): + """Test that idle timeout events are forwarded as raw model events.""" + session = RealtimeSession(mock_model, mock_agent, None) + + timeout_event = RealtimeModelInputAudioBufferTimeoutEvent( + event_id="evt_1", + audio_start_ms=0, + audio_end_ms=500, + item_id="item_1", + ) + + await session.on_event(timeout_event) + + assert session._event_queue.qsize() == 1 + event = await session._event_queue.get() + assert isinstance(event, RealtimeRawModelEvent) + assert event.data == timeout_event + @pytest.mark.asyncio async def test_function_call_event_triggers_tool_handling(self, mock_model, mock_agent): """Test that function_call events trigger tool call handling""" @@ -1360,6 +1380,27 @@ async def test_session_gets_model_settings_from_agent_during_connection(self): await session.__aexit__(None, None, None) + @pytest.mark.asyncio + async def test_idle_timeout_ms_passed_to_model(self): + """Test that idle_timeout_ms is forwarded to the model connect settings.""" + mock_model = Mock(spec=RealtimeModel) + mock_model.connect = AsyncMock() + mock_model.add_listener = Mock() + + agent = Mock(spec=RealtimeAgent) + agent.get_system_prompt = AsyncMock(return_value="") + agent.get_all_tools = AsyncMock(return_value=[]) + agent.handoffs = [] + + run_config: RealtimeRunConfig = {"model_settings": {"idle_timeout_ms": 1000}} + session = RealtimeSession(mock_model, agent, None, run_config=run_config) + + await session.__aenter__() + connect_config = mock_model.connect.call_args[0][0] + assert connect_config["initial_model_settings"]["idle_timeout_ms"] == 1000 + + await session.__aexit__(None, None, None) + @pytest.mark.asyncio async def test_model_config_overrides_model_settings_not_agent(self): """Test that initial_model_settings from model_config override model settings