Add headers to HttpRequestResponse in OpenAI client. (#1006)

## What changes are proposed in this pull request?
Adding headers to HttpRequestResponse when making http_request

Why?
Basically we need the response headers that is used in the MCP spec:
https://modelcontextprotocol.io/specification/2025-06-18/basic/transports#session-management
Specially Mcp-Session-Id header.


## How is this tested?
```
%pip install git+https://github.com/sunishsheth2009/databricks-sdk-py@sunish-add-headers
import requests
import uuid
from databricks.sdk import WorkspaceClient
from databricks.sdk.service.serving import ExternalFunctionRequestHttpMethod

# Step 1: Initialize session
init_payload = {
    "jsonrpc": "2.0",
    "id": "init-1",
    "method": "initialize",
    "params": {}
}

init_response = WorkspaceClient().serving_endpoints.http_request(
  conn="github-u2m-managed-connection",
  method=ExternalFunctionRequestHttpMethod.POST,
  path="/mcp",
  json=init_payload,
)

session_id = init_response.headers["mcp-session-id"]
if not session_id:
    print("No session ID returned by server.")

print(f"✅ Got MCP Session ID: {session_id}")


Output:
✅ Got MCP Session ID: 53220122-015c-4e2b-9edb-47580cefffd0
```

Author: sunishsheth2009

NEXT_CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Release v0.60.0
 
 ### New Features and Improvements
+* Added headers to HttpRequestResponse in OpenAI client.
 
 ### Bug Fixes
 

databricks/sdk/mixins/open_ai_client.py

@@ -4,6 +4,7 @@
 from requests import Response
 
 from databricks.sdk.service.serving import (ExternalFunctionRequestHttpMethod,
+                                            HttpRequestResponse,
                                             ServingEndpointsAPI)
 
 
@@ -88,15 +89,30 @@ def http_request(
         """
         response = Response()
         response.status_code = 200
-        server_response = super().http_request(
-            connection_name=conn,
-            method=method,
-            path=path,
-            headers=js.dumps(headers) if headers is not None else None,
-            json=js.dumps(json) if json is not None else None,
-            params=js.dumps(params) if params is not None else None,
+
+        # We currently don't call super.http_request because we need to pass in response_headers
+        # This is a temporary fix to get the headers we need for the MCP session id
+        # TODO: Remove this once we have a better way to get back the response headers
+        headers_to_capture = ["mcp-session-id"]
+        res = self._api.do(
+            "POST",
+            "/api/2.0/external-function",
+            body={
+                "connection_name": conn,
+                "method": method.value,
+                "path": path,
+                "headers": js.dumps(headers) if headers is not None else None,
+                "json": js.dumps(json) if json is not None else None,
+                "params": js.dumps(params) if params is not None else None,
+            },
+            headers={"Accept": "text/plain", "Content-Type": "application/json"},
+            raw=True,
+            response_headers=headers_to_capture,
         )
 
+        # Create HttpRequestResponse from the raw response
+        server_response = HttpRequestResponse.from_dict(res)
+
         # Read the content from the HttpRequestResponse object
         if hasattr(server_response, "contents") and hasattr(server_response.contents, "read"):
             raw_content = server_response.contents.read()  # Read the bytes
@@ -109,4 +125,9 @@ def http_request(
         else:
             raise ValueError("Contents must be bytes.")
 
+        # Copy headers from raw response to Response
+        for header_name in headers_to_capture:
+            if header_name in res:
+                response.headers[header_name] = res[header_name]
+
         return response