fix multimodal agent interrupts itself during function call (#1585)

Author: longcw

.changeset/nervous-birds-relax.md

@@ -0,0 +1,5 @@
+---
+"livekit-plugins-openai": patch
+---
+
+fix multimodal agent interrupts itself when creating function call response

examples/multimodal-agent/openai_agent.py

@@ -81,7 +81,10 @@ async def get_weather(
         model=openai.realtime.RealtimeModel(
             voice="alloy",
             temperature=0.8,
-            instructions="You are a helpful assistant, greet the user and help them with their trip planning",
+            instructions=(
+                "You are a helpful assistant, greet the user and help them with their trip planning. "
+                "When performing function calls, let user know that you are checking the weather."
+            ),
             turn_detection=openai.realtime.ServerVadOptions(
                 threshold=0.6, prefix_padding_ms=200, silence_duration_ms=500
             ),

livekit-agents/livekit/agents/multimodal/multimodal_agent.py

@@ -121,6 +121,11 @@ def _truncate_conversation_item(
         self, item_id: str, content_index: int, audio_end_ms: int
     ) -> None: ...
 
+    @property
+    def playout_complete(self) -> asyncio.Event | None:
+        """Event that is set when the playout is done"""
+        pass
+
 
 @dataclass(frozen=True)
 class AgentTranscriptionOptions:
@@ -435,10 +440,16 @@ async def _main_task(self) -> None:
         )
 
         def _on_playout_started() -> None:
+            if self._session.playout_complete is not None:
+                self._session.playout_complete.clear()
+
             self.emit("agent_started_speaking")
             self._update_state("speaking")
 
         def _on_playout_stopped(interrupted: bool) -> None:
+            if self._session.playout_complete is not None:
+                self._session.playout_complete.set()
+
             self.emit("agent_stopped_speaking")
             self._update_state("listening")
 

livekit-plugins/livekit-plugins-google/livekit/plugins/google/beta/realtime/realtime_api.py

@@ -258,6 +258,8 @@ def __init__(
         self._fnc_ctx = fnc_ctx
         self._fnc_tasks = utils.aio.TaskSet()
         self._is_interrupted = False
+        self._playout_complete = asyncio.Event()
+        self._playout_complete.set()
 
         tools = []
         if self._fnc_ctx is not None:
@@ -317,6 +319,10 @@ async def aclose(self) -> None:
         self._send_ch.close()
         await self._main_atask
 
+    @property
+    def playout_complete(self) -> asyncio.Event | None:
+        return self._playout_complete
+
     @property
     def fnc_ctx(self) -> llm.FunctionContext | None:
         return self._fnc_ctx

livekit-plugins/livekit-plugins-openai/livekit/plugins/openai/realtime/realtime_model.py

@@ -870,6 +870,8 @@ def __init__(
         self._pending_responses: dict[str, RealtimeResponse] = {}
         self._active_response_id: str | None = None
         self._response_create_fut: asyncio.Future[None] | None = None
+        self._playout_complete = asyncio.Event()
+        self._playout_complete.set()
 
         self._session_id = "not-connected"
         self.session_update()  # initial session init
@@ -886,6 +888,10 @@ async def aclose(self) -> None:
         self._send_ch.close()
         await self._main_atask
 
+    @property
+    def playout_complete(self) -> asyncio.Event:
+        return self._playout_complete
+
     @property
     def fnc_ctx(self) -> llm.FunctionContext | None:
         return self._fnc_ctx
@@ -1746,6 +1752,9 @@ async def _run_fnc_task(self, fnc_call_info: llm.FunctionCallInfo, item_id: str)
         called_fnc = fnc_call_info.execute()
         await called_fnc.task
 
+        # wait for the audio to be played before creating the response
+        await self._playout_complete.wait()
+
         tool_call = llm.ChatMessage.create_tool_from_called_function(called_fnc)
         logger.info(
             "creating response for tool call",