feat: native openai tools support (#651)

Author: konrad-czarnota-ds

docs/how-to/agents/define_and_use_agents.md

@@ -80,3 +80,35 @@ AgentResult(content='The current temperature in Tokyo is 10°C.', ...)
 For use cases where you want to process partial outputs from the LLM as they arrive (e.g., in chat UIs), the [`Agent`][ragbits.agents.Agent] class supports streaming through the `run_streaming()` method.
 
 This method returns an `AgentResultStreaming` object — an async iterator that yields parts of the LLM response and tool-related events in real time.
+
+## Native OpenAI tools
+Ragbits supports selected native OpenAI tools (web_search_preview, image_generation and code_interpreter). You can use them together with your tools.
+```python
+from ragbits.agents.tools import get_web_search_tool
+
+async def main() -> None:
+    """Run the weather agent with additional tool."""
+    model_name = "gpt-4o-2024-08-06"
+    llm = LiteLLM(model_name=model_name, use_structured_output=True)
+    agent = Agent(llm=llm, prompt=WeatherPrompt, tools=[get_web_search_tool(model_name)], keep_history=True)
+
+    response = await agent.run(WeatherPromptInput(location="Paris"))
+    print(response)
+```
+
+Tool descriptions are available [here](https://platform.openai.com/docs/guides/tools?api-mode=responses). For each of these you can see detailed
+information on the corresponding sub-pages (i.e. [here](https://platform.openai.com/docs/guides/tools-web-search?api-mode=responses#user-location) for web search).
+You can use default parameters or specify your own as a dict. For web search this might look like that:
+```python
+from ragbits.agents.tools import get_web_search_tool
+
+tool_params = {
+        "user_location": {
+            "type": "approximate",
+            "country": "GB",
+            "city": "London",
+            "region": "London",
+        }
+}
+web_search_tool = get_web_search_tool("gpt-4o", tool_params)
+```

examples/README.md

@@ -36,6 +36,7 @@ All necessary details are provided in the comments at the top of each script.
 | [Offline Chat Interface](/examples/chat/offline_chat.py)                                         |            [ragbits-chat](/packages/ragbits-chat)            | Example of how to use the `ChatInterface` to create a simple chat application that works offline.                                                       |
 | [Recontextualize Last Message](/examples/chat/recontextualize_message.py)                        |            [ragbits-chat](/packages/ragbits-chat)            | Example of how to use the `StandaloneMessageCompressor` compressor to recontextualize the last message in a conversation history.                       |
 | [Agents Tool Use](/examples/agents/tool_use.py)                                                  |          [ragbits-agents](/packages/ragbits-agents)          | Example of how to use agent with tools.                                                                                                                 |
+| [Agents OpenAI Native Tool Use](/examples/agents/openai_native_tool_use.py)                      |          [ragbits-agents](/packages/ragbits-agents)          | Example of how to use agent with OpenAI native tools.                                                                                                   |
 | [MCP Local](/examples/agents/mcp/local.py)                                                       |          [ragbits-agents](/packages/ragbits-agents)          | Example of how to use the `Agent` class to connect with a local MCP server.                                                                             |
 | [MCP SSE](/examples/agents/mcp/sse.py)                                                           |          [ragbits-agents](/packages/ragbits-agents)          | Example of how to use the `Agent` class to connect with a remote MCP server via SSE.                                                                    |
 | [MCP Streamable HTTP](/examples/agents/mcp/streamable_http.py)                                   |          [ragbits-agents](/packages/ragbits-agents)          | Example of how to use the `Agent` class to connect with a remote MCP server via HTTP.                                                                   |

examples/agents/openai_native_tool_use.py

@@ -0,0 +1,65 @@
+"""
+Ragbits Agents Example: OpenAI native tool use
+
+This example shows how to use agent with native OpenAI tools.
+We provide a single method as a tool to the agent and expect it to call it when answering query.
+
+To run the script, execute the following command:
+
+    ```bash
+    uv run examples/agents/openai_native_tool_use.py
+    ```
+"""
+
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "ragbits-core",
+#     "ragbits-agents[openai]",
+# ]
+# ///
+import asyncio
+
+from pydantic import BaseModel
+
+from ragbits.agents import Agent
+from ragbits.agents.tools import get_web_search_tool
+from ragbits.core.llms import LiteLLM
+from ragbits.core.prompt import Prompt
+
+
+class SearchPromptInput(BaseModel):
+    """
+    Input format for the Search Prompt.
+    """
+
+    query: str
+
+
+class SearchPrompt(Prompt[SearchPromptInput]):
+    """
+    Prompt that does a web search with a given query.
+    """
+
+    system_prompt = """
+    You are a helpful assistant that responds to user questions.
+    """
+
+    user_prompt = """
+    Search web for {{ query }}.
+    """
+
+
+async def main() -> None:
+    """
+    Run the example.
+    """
+    model_name = "gpt-4o-2024-08-06"
+    llm = LiteLLM(model_name=model_name, use_structured_output=True)
+    agent = Agent(llm=llm, prompt=SearchPrompt, tools=[get_web_search_tool(model_name)])
+    response = await agent.run(SearchPromptInput(query="What date is today?"))
+    print(response)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/agents/tool_use.py

@@ -63,7 +63,7 @@ class WeatherPrompt(Prompt[WeatherPromptInput]):
     """
 
     system_prompt = """
-    You are a helpful assisstant that responds to user questions about weather.
+    You are a helpful assistant that responds to user questions about weather.
     """
 
     user_prompt = """

packages/ragbits-agents/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add native openai tools support (#621)
 - add Context to Agents (#715)
 
 ## 1.1.0 (2025-07-09)

packages/ragbits-agents/pyproject.toml

@@ -42,6 +42,9 @@ a2a = [
 mcp = [
     "mcp>=1.9.4,<2.0.0",
 ]
+openai = [
+    "openai>=1.91.0,<2.0.0",
+]
 
 [project.urls]
 "Homepage" = "https://github.com/deepsense-ai/ragbits"

packages/ragbits-agents/src/ragbits/agents/tools/__init__.py

@@ -0,0 +1,3 @@
+from ragbits.agents.tools.openai import get_code_interpreter_tool, get_image_generation_tool, get_web_search_tool
+
+__all__ = ["get_code_interpreter_tool", "get_image_generation_tool", "get_web_search_tool"]

packages/ragbits-agents/src/ragbits/agents/tools/openai.py

@@ -0,0 +1,141 @@
+import base64
+from collections.abc import Callable
+from typing import cast
+
+from openai import AsyncOpenAI
+from openai.types.responses import Response
+from openai.types.responses.tool_param import CodeInterpreter, ToolParam
+
+
+def get_web_search_tool(model_name: str, additional_params: dict | None = None) -> Callable:
+    """
+    Returns a native OpenAI web search tool as function
+
+    Args:
+        model_name: The name of the model
+        additional_params: The additional tool parameters
+
+    Returns:
+        web search function
+    """
+    params_to_pass = additional_params if additional_params else {}
+    tool_object = OpenAITools(model_name, {"type": "web_search_preview", **params_to_pass})
+    return tool_object.search_web
+
+
+def get_code_interpreter_tool(model_name: str, additional_params: dict | None = None) -> Callable:
+    """
+    Returns a native OpenAI code interpreter tool as function
+
+    Args:
+        model_name: The name of the model
+        additional_params: The additional tool parameters
+
+    Returns:
+        code interpreter function
+    """
+    params_to_pass = additional_params if additional_params else {}
+    tool_object = OpenAITools(model_name, cast(CodeInterpreter, {"type": "code_interpreter", **params_to_pass}))
+    return tool_object.code_interpreter
+
+
+def get_image_generation_tool(model_name: str, additional_params: dict | None = None) -> Callable:
+    """
+    Returns a native OpenAI image generation tool as function
+
+    Args:
+        model_name: The name of the model
+        additional_params: The additional tool parameters
+
+    Returns:
+        image generation function
+    """
+    params_to_pass = additional_params if additional_params else {}
+    tool_object = OpenAITools(model_name, {"type": "image_generation", **params_to_pass})
+    return tool_object.image_generation
+
+
+class OpenAIResponsesLLM:
+    """
+    Class serving as a wrapper for tool calls to responses API of OpenAI
+    """
+
+    def __init__(self, model_name: str, tool_param: ToolParam):
+        self._client = AsyncOpenAI()
+        self._model_name = model_name
+        self._tool_param = tool_param
+
+    async def use_tool(self, query: str) -> Response:
+        """
+        Uses tool based on query and returns output.
+
+        Args:
+            query: query for the tool
+
+        Returns:
+            Output of the tool
+        """
+        return await self._client.responses.create(
+            model=self._model_name,
+            tools=[self._tool_param],
+            tool_choice="required",
+            input=query,
+        )
+
+
+class OpenAITools:
+    """
+    Class wrapping tool calls to responses API of OpenAI
+    """
+
+    AVAILABLE_TOOLS = {"web_search_preview", "code_interpreter", "image_generation"}
+
+    def __init__(self, model_name: str, tool_param: ToolParam):
+        self._responses_llm = OpenAIResponsesLLM(model_name, tool_param)
+
+    async def search_web(self, query: str) -> str:
+        """
+        Searches web for a query
+
+        Args:
+            query: The query to search
+
+        Returns:
+            The web search result
+        """
+        return (await self._responses_llm.use_tool(query)).output_text
+
+    async def code_interpreter(self, query: str) -> str:
+        """
+        Performs actions in code interpreter based on query
+
+        Args:
+            query: The query with instructions
+
+        Returns:
+            Output of the interpreter
+        """
+        return (await self._responses_llm.use_tool(query)).output_text
+
+    async def image_generation(self, query: str, save_path: str = "generated_image.png") -> str:
+        """
+        Generate image based on query.
+
+        Args:
+            query: The query with instructions
+            save_path: The path to save the generated image
+
+        Returns:
+            LLM text output
+        """
+        response = await self._responses_llm.use_tool(query)
+        image_data = next((output.result for output in response.output if output.type == "image_generation_call"), None)
+
+        if image_data:
+            with open(save_path, "wb") as f:
+                f.write(base64.b64decode(image_data))
+            text_prefix = f"Image saved to {save_path}\n"
+        else:
+            text_prefix = "No generated image was returned\n"
+
+        return text_prefix + response.output_text

packages/ragbits-agents/tests/unit/tools/test_openai.py

@@ -0,0 +1,95 @@
+import base64
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from ragbits.agents.tools.openai import OpenAITools
+
+
+@pytest.fixture
+def mock_responses_llm() -> MagicMock:
+    """Fixture for mocking OpenAIResponsesLLM."""
+    mock = MagicMock()
+    mock.use_tool = AsyncMock()
+    return mock
+
+
+@patch("ragbits.agents.tools.openai.AsyncOpenAI")
+async def test_search_web(mock_async_openai: MagicMock, mock_responses_llm: MagicMock) -> None:
+    """Test that search_web calls use_tool and returns output_text."""
+    mock_response = MagicMock()
+    mock_response.output_text = "Test output"
+    mock_responses_llm.use_tool.return_value = mock_response
+    tools = OpenAITools("test_model", tool_param=MagicMock())
+    tools._responses_llm = mock_responses_llm
+
+    result = await tools.search_web("test query")
+
+    mock_responses_llm.use_tool.assert_called_once_with("test query")
+    assert result == "Test output"
+
+
+@patch("ragbits.agents.tools.openai.AsyncOpenAI")
+async def test_code_interpreter(mock_async_openai: MagicMock, mock_responses_llm: MagicMock) -> None:
+    """Test that code_interpreter calls use_tool and returns output_text."""
+    mock_response = MagicMock()
+    mock_response.output_text = "Interpreter output"
+    mock_responses_llm.use_tool.return_value = mock_response
+    tools = OpenAITools("test_model", tool_param=MagicMock())
+    tools._responses_llm = mock_responses_llm
+
+    result = await tools.code_interpreter("run code")
+
+    mock_responses_llm.use_tool.assert_called_once_with("run code")
+    assert result == "Interpreter output"
+
+
+@patch("ragbits.agents.tools.openai.AsyncOpenAI")
+@patch("builtins.open", new_callable=MagicMock)
+async def test_image_generation(
+    mock_open: MagicMock, mock_async_openai: MagicMock, mock_responses_llm: MagicMock
+) -> None:
+    """Test image_generation saves image and returns correct text."""
+    image_data = base64.b64encode(b"test_image_content").decode("utf-8")
+
+    mock_output_item = MagicMock()
+    mock_output_item.type = "image_generation_call"
+    mock_output_item.result = image_data
+
+    mock_response = MagicMock()
+    mock_response.output_text = "Generated image."
+    mock_response.output = [mock_output_item]
+
+    mock_file_handle = MagicMock()
+    mock_open.return_value.__enter__.return_value = mock_file_handle
+
+    mock_responses_llm.use_tool.return_value = mock_response
+    tools = OpenAITools("test_model", tool_param=MagicMock())
+    tools._responses_llm = mock_responses_llm
+
+    result = await tools.image_generation("a cat", save_path="cat.png")
+
+    mock_responses_llm.use_tool.assert_called_once_with("a cat")
+    mock_open.assert_called_once_with("cat.png", "wb")
+    mock_file_handle.write.assert_called_once_with(b"test_image_content")
+    assert result == "Image saved to cat.png\nGenerated image."
+
+
+@patch("ragbits.agents.tools.openai.AsyncOpenAI")
+async def test_image_generation_no_image(mock_async_openai: MagicMock, mock_responses_llm: MagicMock) -> None:
+    """Test image_generation when no image is returned."""
+    mock_output_item = MagicMock()
+    mock_output_item.type = "image_generation_call"
+    mock_output_item.result = None
+
+    mock_response = MagicMock()
+    mock_response.output_text = "No image was generated."
+    mock_response.output = [mock_output_item]
+
+    mock_responses_llm.use_tool.return_value = mock_response
+    tools = OpenAITools("test_model", tool_param=MagicMock())
+    tools._responses_llm = mock_responses_llm
+
+    result = await tools.image_generation("a dog")
+
+    assert result == "No generated image was returned\nNo image was generated."