Merge branch 'main' into enhancement/add-streaming-inner-events

Author: vrtnis

.vscode/launch.json

@@ -0,0 +1,14 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Python File",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}"
+        }
+    ]
+}

docs/examples.md

@@ -40,3 +40,6 @@ Check out a variety of sample implementations of the SDK in the examples section
 
 - **[voice](https://github.com/openai/openai-agents-python/tree/main/examples/voice):**
   See examples of voice agents, using our TTS and STT models.
+
+- **[realtime](https://github.com/openai/openai-agents-python/tree/main/examples/realtime):**
+  Examples showing how to build realtime experiences using the SDK.

docs/release.md

@@ -19,6 +19,10 @@ We will increment `Z` for non-breaking changes:
 
 ## Breaking change changelog
 
+### 0.2.0
+
+In this version, a few places that used to take `Agent` as an arg, now take `AgentBase` as an arg instead. For example, the `list_tools()` call in MCP servers. This is a purely typing change, you will still receive `Agent` objects. To update, just fix type errors by replacing `Agent` with `AgentBase`.
+
 ### 0.1.0
 
 In this version, [`MCPServer.list_tools()`][agents.mcp.server.MCPServer] has two new params: `run_context` and `agent`. You'll need to add these params to any classes that subclass `MCPServer`.

docs/sessions.md

@@ -101,8 +101,8 @@ result = await Runner.run(
 print(f"Agent: {result.final_output}")
 
 # User wants to correct their question
-user_item = await session.pop_item()  # Remove user's question
 assistant_item = await session.pop_item()  # Remove agent's response
+user_item = await session.pop_item()  # Remove user's question
 
 # Ask a corrected question
 result = await Runner.run(

docs/tools.md

@@ -180,7 +180,7 @@ Sometimes, you don't want to use a Python function as a tool. You can directly c
 -   `name`
 -   `description`
 -   `params_json_schema`, which is the JSON schema for the arguments
--   `on_invoke_tool`, which is an async function that receives the context and the arguments as a JSON string, and must return the tool output as a string.
+-   `on_invoke_tool`, which is an async function that receives a [`ToolContext`][agents.tool_context.ToolContext] and the arguments as a JSON string, and must return the tool output as a string.
 
 ```python
 from typing import Any

examples/agent_patterns/llm_as_a_judge.py

@@ -32,7 +32,7 @@ class EvaluationFeedback:
     instructions=(
         "You evaluate a story outline and decide if it's good enough."
         "If it's not good enough, you provide feedback on what needs to be improved."
-        "Never give it a pass on the first try."
+        "Never give it a pass on the first try. After 5 attempts, you can give it a pass if story outline is good enough - do not go for perfection"
     ),
     output_type=EvaluationFeedback,
 )

examples/mcp/prompt_server/main.py

@@ -17,7 +17,7 @@ async def get_instructions_from_prompt(mcp_server: MCPServer, prompt_name: str,
     try:
         prompt_result = await mcp_server.get_prompt(prompt_name, kwargs)
         content = prompt_result.messages[0].content
-        if hasattr(content, 'text'):
+        if hasattr(content, "text"):
             instructions = content.text
         else:
             instructions = str(content)

examples/realtime/demo.py

@@ -0,0 +1,116 @@
+import asyncio
+import os
+import sys
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from agents.realtime import RealtimeSession
+
+# Add the current directory to path so we can import ui
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from agents import function_tool
+from agents.realtime import RealtimeAgent, RealtimeRunner, RealtimeSessionEvent
+
+if TYPE_CHECKING:
+    from .ui import AppUI
+else:
+    # Try both import styles
+    try:
+        # Try relative import first (when used as a package)
+        from .ui import AppUI
+    except ImportError:
+        # Fall back to direct import (when run as a script)
+        from ui import AppUI
+
+
+@function_tool
+def get_weather(city: str) -> str:
+    """Get the weather in a city."""
+    return f"The weather in {city} is sunny."
+
+
+agent = RealtimeAgent(
+    name="Assistant",
+    instructions="You always greet the user with 'Top of the morning to you'.",
+    tools=[get_weather],
+)
+
+
+def _truncate_str(s: str, max_length: int) -> str:
+    if len(s) > max_length:
+        return s[:max_length] + "..."
+    return s
+
+
+class Example:
+    def __init__(self) -> None:
+        self.ui = AppUI()
+        self.ui.connected = asyncio.Event()
+        self.ui.last_audio_item_id = None
+        # Set the audio callback
+        self.ui.set_audio_callback(self.on_audio_recorded)
+
+        self.session: RealtimeSession | None = None
+
+    async def run(self) -> None:
+        # Start UI in a separate task instead of waiting for it to complete
+        ui_task = asyncio.create_task(self.ui.run_async())
+
+        # Set up session immediately without waiting for UI to finish
+        runner = RealtimeRunner(agent)
+        async with await runner.run() as session:
+            self.session = session
+            self.ui.set_is_connected(True)
+            async for event in session:
+                await self._on_event(event)
+            print("done")
+
+        # Wait for UI task to complete when session ends
+        await ui_task
+
+    async def on_audio_recorded(self, audio_bytes: bytes) -> None:
+        # Send the audio to the session
+        assert self.session is not None
+        await self.session.send_audio(audio_bytes)
+
+    async def _on_event(self, event: RealtimeSessionEvent) -> None:
+        try:
+            if event.type == "agent_start":
+                self.ui.add_transcript(f"Agent started: {event.agent.name}")
+            elif event.type == "agent_end":
+                self.ui.add_transcript(f"Agent ended: {event.agent.name}")
+            elif event.type == "handoff":
+                self.ui.add_transcript(
+                    f"Handoff from {event.from_agent.name} to {event.to_agent.name}"
+                )
+            elif event.type == "tool_start":
+                self.ui.add_transcript(f"Tool started: {event.tool.name}")
+            elif event.type == "tool_end":
+                self.ui.add_transcript(f"Tool ended: {event.tool.name}; output: {event.output}")
+            elif event.type == "audio_end":
+                self.ui.add_transcript("Audio ended")
+            elif event.type == "audio":
+                np_audio = np.frombuffer(event.audio.data, dtype=np.int16)
+                self.ui.play_audio(np_audio)
+            elif event.type == "audio_interrupted":
+                self.ui.add_transcript("Audio interrupted")
+            elif event.type == "error":
+                pass
+            elif event.type == "history_updated":
+                pass
+            elif event.type == "history_added":
+                pass
+            elif event.type == "raw_model_event":
+                if event.data.type != "error" and event.data.type != "exception":
+                    self.ui.log_message(f"Raw model event: {event.data}")
+            else:
+                self.ui.log_message(f"Unknown event type: {event.type}")
+        except Exception as e:
+            self.ui.log_message(f"Error processing event: {_truncate_str(str(e), 50)}")
+
+
+if __name__ == "__main__":
+    example = Example()
+    asyncio.run(example.run())

examples/realtime/no_ui_demo.py

@@ -0,0 +1,178 @@
+import asyncio
+import sys
+
+import numpy as np
+import sounddevice as sd
+
+from agents import function_tool
+from agents.realtime import RealtimeAgent, RealtimeRunner, RealtimeSession, RealtimeSessionEvent
+
+# Audio configuration
+CHUNK_LENGTH_S = 0.05  # 50ms
+SAMPLE_RATE = 24000
+FORMAT = np.int16
+CHANNELS = 1
+
+# Set up logging for OpenAI agents SDK
+# logging.basicConfig(
+#     level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+# )
+# logger.logger.setLevel(logging.ERROR)
+
+
+@function_tool
+def get_weather(city: str) -> str:
+    """Get the weather in a city."""
+    return f"The weather in {city} is sunny."
+
+
+agent = RealtimeAgent(
+    name="Assistant",
+    instructions="You always greet the user with 'Top of the morning to you'.",
+    tools=[get_weather],
+)
+
+
+def _truncate_str(s: str, max_length: int) -> str:
+    if len(s) > max_length:
+        return s[:max_length] + "..."
+    return s
+
+
+class NoUIDemo:
+    def __init__(self) -> None:
+        self.session: RealtimeSession | None = None
+        self.audio_stream: sd.InputStream | None = None
+        self.audio_player: sd.OutputStream | None = None
+        self.recording = False
+
+    async def run(self) -> None:
+        print("Connecting, may take a few seconds...")
+
+        # Initialize audio player
+        self.audio_player = sd.OutputStream(
+            channels=CHANNELS,
+            samplerate=SAMPLE_RATE,
+            dtype=FORMAT,
+        )
+        self.audio_player.start()
+
+        try:
+            runner = RealtimeRunner(agent)
+            async with await runner.run() as session:
+                self.session = session
+                print("Connected. Starting audio recording...")
+
+                # Start audio recording
+                await self.start_audio_recording()
+                print("Audio recording started. You can start speaking - expect lots of logs!")
+
+                # Process session events
+                async for event in session:
+                    await self._on_event(event)
+
+        finally:
+            # Clean up audio player
+            if self.audio_player and self.audio_player.active:
+                self.audio_player.stop()
+            if self.audio_player:
+                self.audio_player.close()
+
+        print("Session ended")
+
+    async def start_audio_recording(self) -> None:
+        """Start recording audio from the microphone."""
+        # Set up audio input stream
+        self.audio_stream = sd.InputStream(
+            channels=CHANNELS,
+            samplerate=SAMPLE_RATE,
+            dtype=FORMAT,
+        )
+
+        self.audio_stream.start()
+        self.recording = True
+
+        # Start audio capture task
+        asyncio.create_task(self.capture_audio())
+
+    async def capture_audio(self) -> None:
+        """Capture audio from the microphone and send to the session."""
+        if not self.audio_stream or not self.session:
+            return
+
+        # Buffer size in samples
+        read_size = int(SAMPLE_RATE * CHUNK_LENGTH_S)
+
+        try:
+            while self.recording:
+                # Check if there's enough data to read
+                if self.audio_stream.read_available < read_size:
+                    await asyncio.sleep(0.01)
+                    continue
+
+                # Read audio data
+                data, _ = self.audio_stream.read(read_size)
+
+                # Convert numpy array to bytes
+                audio_bytes = data.tobytes()
+
+                # Send audio to session
+                await self.session.send_audio(audio_bytes)
+
+                # Yield control back to event loop
+                await asyncio.sleep(0)
+
+        except Exception as e:
+            print(f"Audio capture error: {e}")
+        finally:
+            if self.audio_stream and self.audio_stream.active:
+                self.audio_stream.stop()
+            if self.audio_stream:
+                self.audio_stream.close()
+
+    async def _on_event(self, event: RealtimeSessionEvent) -> None:
+        """Handle session events."""
+        try:
+            if event.type == "agent_start":
+                print(f"Agent started: {event.agent.name}")
+            elif event.type == "agent_end":
+                print(f"Agent ended: {event.agent.name}")
+            elif event.type == "handoff":
+                print(f"Handoff from {event.from_agent.name} to {event.to_agent.name}")
+            elif event.type == "tool_start":
+                print(f"Tool started: {event.tool.name}")
+            elif event.type == "tool_end":
+                print(f"Tool ended: {event.tool.name}; output: {event.output}")
+            elif event.type == "audio_end":
+                print("Audio ended")
+            elif event.type == "audio":
+                # Play audio through speakers
+                np_audio = np.frombuffer(event.audio.data, dtype=np.int16)
+                if self.audio_player:
+                    try:
+                        self.audio_player.write(np_audio)
+                    except Exception as e:
+                        print(f"Audio playback error: {e}")
+            elif event.type == "audio_interrupted":
+                print("Audio interrupted")
+            elif event.type == "error":
+                print(f"Error: {event.error}")
+            elif event.type == "history_updated":
+                pass  # Skip these frequent events
+            elif event.type == "history_added":
+                pass  # Skip these frequent events
+            elif event.type == "raw_model_event":
+                print(f"Raw model event: {_truncate_str(str(event.data), 50)}")
+            else:
+                print(f"Unknown event type: {event.type}")
+        except Exception as e:
+            print(f"Error processing event: {_truncate_str(str(e), 50)}")
+
+
+if __name__ == "__main__":
+    demo = NoUIDemo()
+    try:
+        asyncio.run(demo.run())
+    except KeyboardInterrupt:
+        print("\nExiting...")
+        sys.exit(0)