openai[patch]: support built-in code interpreter and remote MCP tools (#31304)

Author: ccurme

docs/docs/integrations/chat/openai.ipynb

@@ -915,6 +915,175 @@
     "response_2.text()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "34ad0015-688c-4274-be55-93268b44f558",
+   "metadata": {},
+   "source": [
+    "#### Code interpreter\n",
+    "\n",
+    "OpenAI implements a [code interpreter](https://platform.openai.com/docs/guides/tools-code-interpreter) tool to support the sandboxed generation and execution of code.\n",
+    "\n",
+    "Example use:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "34826aae-6d48-4b84-bc00-89594a87d461",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "llm = ChatOpenAI(model=\"o4-mini\", use_responses_api=True)\n",
+    "\n",
+    "llm_with_tools = llm.bind_tools(\n",
+    "    [\n",
+    "        {\n",
+    "            \"type\": \"code_interpreter\",\n",
+    "            # Create a new container\n",
+    "            \"container\": {\"type\": \"auto\"},\n",
+    "        }\n",
+    "    ]\n",
+    ")\n",
+    "response = llm_with_tools.invoke(\n",
+    "    \"Write and run code to answer the question: what is 3^3?\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1b4d92b9-941f-4d54-93a5-b0c73afd66b2",
+   "metadata": {},
+   "source": [
+    "Note that the above command created a new container. We can also specify an existing container ID:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "d8c82895-5011-4062-a1bb-278ec91321e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tool_outputs = response.additional_kwargs[\"tool_outputs\"]\n",
+    "assert len(tool_outputs) == 1\n",
+    "# highlight-next-line\n",
+    "container_id = tool_outputs[0][\"container_id\"]\n",
+    "\n",
+    "llm_with_tools = llm.bind_tools(\n",
+    "    [\n",
+    "        {\n",
+    "            \"type\": \"code_interpreter\",\n",
+    "            # Use an existing container\n",
+    "            # highlight-next-line\n",
+    "            \"container\": container_id,\n",
+    "        }\n",
+    "    ]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8db30501-522c-4915-963d-d60539b5c16e",
+   "metadata": {},
+   "source": [
+    "#### Remote MCP\n",
+    "\n",
+    "OpenAI implements a [remote MCP](https://platform.openai.com/docs/guides/tools-remote-mcp) tool that allows for model-generated calls to MCP servers.\n",
+    "\n",
+    "Example use:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "7044a87b-8b99-49e8-8ca4-e2a8ae49f65a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "llm = ChatOpenAI(model=\"o4-mini\", use_responses_api=True)\n",
+    "\n",
+    "llm_with_tools = llm.bind_tools(\n",
+    "    [\n",
+    "        {\n",
+    "            \"type\": \"mcp\",\n",
+    "            \"server_label\": \"deepwiki\",\n",
+    "            \"server_url\": \"https://mcp.deepwiki.com/mcp\",\n",
+    "            \"require_approval\": \"never\",\n",
+    "        }\n",
+    "    ]\n",
+    ")\n",
+    "response = llm_with_tools.invoke(\n",
+    "    \"What transport protocols does the 2025-03-26 version of the MCP \"\n",
+    "    \"spec (modelcontextprotocol/modelcontextprotocol) support?\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0ed7494e-425d-4bdf-ab83-3164757031dd",
+   "metadata": {},
+   "source": [
+    "<details>\n",
+    "<summary>MCP Approvals</summary>\n",
+    "\n",
+    "OpenAI will at times request approval before sharing data with a remote MCP server.\n",
+    "\n",
+    "In the above command, we instructed the model to never require approval. We can also configure the model to always request approval, or to always request approval for specific tools:\n",
+    "\n",
+    "```python\n",
+    "llm_with_tools = llm.bind_tools(\n",
+    "    [\n",
+    "        {\n",
+    "            \"type\": \"mcp\",\n",
+    "            \"server_label\": \"deepwiki\",\n",
+    "            \"server_url\": \"https://mcp.deepwiki.com/mcp\",\n",
+    "            \"require_approval\": {\n",
+    "                \"always\": {\n",
+    "                    \"tool_names\": [\"read_wiki_structure\"]\n",
+    "                }\n",
+    "            }\n",
+    "        }\n",
+    "    ]\n",
+    ")\n",
+    "response = llm_with_tools.invoke(\n",
+    "    \"What transport protocols does the 2025-03-26 version of the MCP \"\n",
+    "    \"spec (modelcontextprotocol/modelcontextprotocol) support?\"\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "Responses may then include blocks with type `\"mcp_approval_request\"`.\n",
+    "\n",
+    "To submit approvals for an approval request, structure it into a content block in an input message:\n",
+    "\n",
+    "```python\n",
+    "approval_message = {\n",
+    "    \"role\": \"user\",\n",
+    "    \"content\": [\n",
+    "        {\n",
+    "            \"type\": \"mcp_approval_response\",\n",
+    "            \"approve\": True,\n",
+    "            \"approval_request_id\": output[\"id\"],\n",
+    "        }\n",
+    "        for output in response.additional_kwargs[\"tool_outputs\"]\n",
+    "        if output[\"type\"] == \"mcp_approval_request\"\n",
+    "    ]\n",
+    "}\n",
+    "\n",
+    "next_response = llm_with_tools.invoke(\n",
+    "    [approval_message],\n",
+    "    # continue existing thread\n",
+    "    previous_response_id=response.response_metadata[\"id\"]\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "</details>"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "6fda05f0-4b81-4709-9407-f316d760ad50",

libs/core/langchain_core/utils/function_calling.py

@@ -554,9 +554,19 @@ def convert_to_openai_tool(
         Return OpenAI Responses API-style tools unchanged. This includes
         any dict with "type" in "file_search", "function", "computer_use_preview",
         "web_search_preview".
+
+    .. versionchanged:: 0.3.61
+
+        Added support for OpenAI's built-in code interpreter and remote MCP tools.
     """
     if isinstance(tool, dict):
-        if tool.get("type") in ("function", "file_search", "computer_use_preview"):
+        if tool.get("type") in (
+            "function",
+            "file_search",
+            "computer_use_preview",
+            "code_interpreter",
+            "mcp",
+        ):
             return tool
         # As of 03.12.25 can be "web_search_preview" or "web_search_preview_2025_03_11"
         if (tool.get("type") or "").startswith("web_search_preview"):

libs/partners/openai/langchain_openai/chat_models/base.py

@@ -775,16 +775,22 @@ def _stream_responses(
 
         with context_manager as response:
             is_first_chunk = True
+            has_reasoning = False
             for chunk in response:
                 metadata = headers if is_first_chunk else {}
                 if generation_chunk := _convert_responses_chunk_to_generation_chunk(
-                    chunk, schema=original_schema_obj, metadata=metadata
+                    chunk,
+                    schema=original_schema_obj,
+                    metadata=metadata,
+                    has_reasoning=has_reasoning,
                 ):
                     if run_manager:
                         run_manager.on_llm_new_token(
                             generation_chunk.text, chunk=generation_chunk
                         )
                     is_first_chunk = False
+                    if "reasoning" in generation_chunk.message.additional_kwargs:
+                        has_reasoning = True
                     yield generation_chunk
 
     async def _astream_responses(
@@ -811,16 +817,22 @@ async def _astream_responses(
 
         async with context_manager as response:
             is_first_chunk = True
+            has_reasoning = False
             async for chunk in response:
                 metadata = headers if is_first_chunk else {}
                 if generation_chunk := _convert_responses_chunk_to_generation_chunk(
-                    chunk, schema=original_schema_obj, metadata=metadata
+                    chunk,
+                    schema=original_schema_obj,
+                    metadata=metadata,
+                    has_reasoning=has_reasoning,
                 ):
                     if run_manager:
                         await run_manager.on_llm_new_token(
                             generation_chunk.text, chunk=generation_chunk
                         )
                     is_first_chunk = False
+                    if "reasoning" in generation_chunk.message.additional_kwargs:
+                        has_reasoning = True
                     yield generation_chunk
 
     def _should_stream_usage(
@@ -1176,12 +1188,22 @@ def _get_invocation_params(
         self, stop: Optional[list[str]] = None, **kwargs: Any
     ) -> dict[str, Any]:
         """Get the parameters used to invoke the model."""
-        return {
+        params = {
             "model": self.model_name,
             **super()._get_invocation_params(stop=stop),
             **self._default_params,
             **kwargs,
         }
+        # Redact headers from built-in remote MCP tool invocations
+        if (tools := params.get("tools")) and isinstance(tools, list):
+            params["tools"] = [
+                ({**tool, "headers": "**REDACTED**"} if "headers" in tool else tool)
+                if isinstance(tool, dict) and tool.get("type") == "mcp"
+                else tool
+                for tool in tools
+            ]
+
+        return params
 
     def _get_ls_params(
         self, stop: Optional[list[str]] = None, **kwargs: Any
@@ -1456,6 +1478,8 @@ def bind_tools(
                     "file_search",
                     "web_search_preview",
                     "computer_use_preview",
+                    "code_interpreter",
+                    "mcp",
                 ):
                     tool_choice = {"type": tool_choice}
                 # 'any' is not natively supported by OpenAI API.
@@ -3150,12 +3174,22 @@ def _construct_responses_api_input(messages: Sequence[BaseMessage]) -> list:
                     ):
                         function_call["id"] = _id
                     function_calls.append(function_call)
-            # Computer calls
+            # Built-in tool calls
             computer_calls = []
+            code_interpreter_calls = []
+            mcp_calls = []
             tool_outputs = lc_msg.additional_kwargs.get("tool_outputs", [])
             for tool_output in tool_outputs:
                 if tool_output.get("type") == "computer_call":
                     computer_calls.append(tool_output)
+                elif tool_output.get("type") == "code_interpreter_call":
+                    code_interpreter_calls.append(tool_output)
+                elif tool_output.get("type") == "mcp_call":
+                    mcp_calls.append(tool_output)
+                else:
+                    pass
+            input_.extend(code_interpreter_calls)
+            input_.extend(mcp_calls)
             msg["content"] = msg.get("content") or []
             if lc_msg.additional_kwargs.get("refusal"):
                 if isinstance(msg["content"], str):
@@ -3196,6 +3230,7 @@ def _construct_responses_api_input(messages: Sequence[BaseMessage]) -> list:
         elif msg["role"] in ("user", "system", "developer"):
             if isinstance(msg["content"], list):
                 new_blocks = []
+                non_message_item_types = ("mcp_approval_response",)
                 for block in msg["content"]:
                     # chat api: {"type": "text", "text": "..."}
                     # responses api: {"type": "input_text", "text": "..."}
@@ -3216,10 +3251,15 @@ def _construct_responses_api_input(messages: Sequence[BaseMessage]) -> list:
                         new_blocks.append(new_block)
                     elif block["type"] in ("input_text", "input_image", "input_file"):
                         new_blocks.append(block)
+                    elif block["type"] in non_message_item_types:
+                        input_.append(block)
                     else:
                         pass
                 msg["content"] = new_blocks
-            input_.append(msg)
+                if msg["content"]:
+                    input_.append(msg)
+            else:
+                input_.append(msg)
         else:
             input_.append(msg)
 
@@ -3366,7 +3406,10 @@ def _construct_lc_result_from_responses_api(
 
 
 def _convert_responses_chunk_to_generation_chunk(
-    chunk: Any, schema: Optional[type[_BM]] = None, metadata: Optional[dict] = None
+    chunk: Any,
+    schema: Optional[type[_BM]] = None,
+    metadata: Optional[dict] = None,
+    has_reasoning: bool = False,
 ) -> Optional[ChatGenerationChunk]:
     content = []
     tool_call_chunks: list = []
@@ -3429,6 +3472,10 @@ def _convert_responses_chunk_to_generation_chunk(
         "web_search_call",
         "file_search_call",
         "computer_call",
+        "code_interpreter_call",
+        "mcp_call",
+        "mcp_list_tools",
+        "mcp_approval_request",
     ):
         additional_kwargs["tool_outputs"] = [
             chunk.item.model_dump(exclude_none=True, mode="json")
@@ -3444,9 +3491,11 @@ def _convert_responses_chunk_to_generation_chunk(
     elif chunk.type == "response.refusal.done":
         additional_kwargs["refusal"] = chunk.refusal
     elif chunk.type == "response.output_item.added" and chunk.item.type == "reasoning":
-        additional_kwargs["reasoning"] = chunk.item.model_dump(
-            exclude_none=True, mode="json"
-        )
+        if not has_reasoning:
+            # Hack until breaking release: store first reasoning item ID.
+            additional_kwargs["reasoning"] = chunk.item.model_dump(
+                exclude_none=True, mode="json"
+            )
     elif chunk.type == "response.reasoning_summary_part.added":
         additional_kwargs["reasoning"] = {
             # langchain-core uses the `index` key to aggregate text blocks.