fix: add stream value in gcp anthropic body (#1370)

**Description**

This PR fixes a bug where streaming chat completion requests from an
OpenAI-compatible client to the GCP Anthropic backend were failing.
Although the API returned a 200 OK status, the response was a
non-streaming JSON object instead of a text/event-stream, causing the
client to receive an empty stream and to fail our integration test
assertions.

Our integration tests output was like 
`>       assert model_output, f"Received empty {model_output=}"
E       AssertionError: Received empty model_output=''
E       assert ''` 

and after adding debugging logs, the response header and the output
message showed that the response in not streamed, even though the url is
set to stream. It seems you can send to the stream URL but if the param
isn't set, that takes precedence.

the header output
{
  "time": "2025-10-14T16:02:11.407-04:00",
  "level": "DEBUG",
  "msg": "response headers processing",
"response_headers": "headers:{key:\":status\" raw_value:\"200\"}
headers:{key:\"content-type\" raw_value:\"application/json\"}
headers:{key:\"server\" raw_value:\"hypercorn-h11\"} ..."
}

This second log shows the entire response body arriving in a single
chunk with end_of_stream:true. This confirmed that instead of streaming
the response piece-by-piece, the server sent the complete final message
all at once.

{
  "time": "2025-10-14T16:02:11.408-04:00",
  "level": "DEBUG",
  "msg": "response body processing",
"request":
"response_body:{body:\"{\\\"id\\\":\\\"msg_vrtx_019F694kiwv6Z5BQApos5MJy\\\",\\\"type\\\":\\\"message\\\",\\\"role\\\":\\\"assistant\\\",
... ,\\\"content\\\":[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"I'm
doing well, thank you for asking! I'm here and ready to help with
whatever you'd like to chat about or work on. How are you doing
today?\\\"}],\\\"stop_reason\\\":\\\"end_turn\\\", ... }\"
end_of_stream:true}"
}

The root cause was the missing "stream": true field in the JSON payload
sent to GCP's :streamRawPredict endpoint. This PR modifies the
OpenAIToGCPAnthropicTranslator to conditionally add this field to the
request body when the original client request is for a stream. For
context, it seems like this is how anthropic's go sdk handles it
https://github.com/anthropics/anthropic-sdk-go/blob/e8befdc7fdceba33c9000b0b50061b8a42cb6c04/message.go#L86
its not a field in the message param object.


The Fix:
The solution implemented in this PR is to ensure the translated request
body always has the field stream: true for streaming requests:

Modify RequestBody: The RequestBody function in openaittogcpanthropic.go
has been updated to check if the incoming openAIReq.Stream is true.

Inject Stream Field: If streaming is requested, we use the sjson library
to inject the "stream": true key-value pair into the final JSON payload
before it is sent to the GCP endpoint.

Add Unit Test: The existing unit test "Streaming Request Path" has been
renamed to "Streaming Request Path and Body" and an assertion has been
added to verify that the stream field is correctly set to true in the
marshaled body, preventing future regressions

---------

Signed-off-by: Alexa Griffith <[email protected]>
Signed-off-by: Sukumar Gaonkar <[email protected]>
Signed-off-by: Dan Sun <[email protected]>

Author: alexagriffith

internal/extproc/translator/openai_gcpanthropic.go

@@ -548,6 +548,10 @@ func (o *openAIToGCPAnthropicTranslatorV1ChatCompletion) RequestBody(_ []byte, o
 	specifier := "rawPredict"
 	if openAIReq.Stream {
 		specifier = "streamRawPredict"
+		body, err = sjson.SetBytes(body, "stream", true)
+		if err != nil {
+			return
+		}
 		o.streamParser = newAnthropicStreamParser(o.requestModel)
 	}
 
@@ -558,7 +562,10 @@ func (o *openAIToGCPAnthropicTranslatorV1ChatCompletion) RequestBody(_ []byte, o
 	if o.apiVersion != "" {
 		anthropicVersion = o.apiVersion
 	}
-	body, _ = sjson.SetBytes(body, anthropicVersionKey, anthropicVersion)
+	body, err = sjson.SetBytes(body, anthropicVersionKey, anthropicVersion)
+	if err != nil {
+		return
+	}
 
 	headerMutation, bodyMutation = buildRequestMutations(pathSuffix, body)
 	return

internal/extproc/translator/openai_gcpanthropic_test.go

@@ -191,15 +191,15 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T
 		require.Equal(t, thirdMsg, gjson.GetBytes(body, "messages.0.content.0.text").String())
 	})
 
-	t.Run("Streaming Request Path", func(t *testing.T) {
+	t.Run("Streaming Request Validation", func(t *testing.T) {
 		streamReq := &openai.ChatCompletionRequest{
 			Model:     claudeTestModel,
 			Messages:  []openai.ChatCompletionMessageParamUnion{},
 			MaxTokens: ptr.To(int64(100)),
 			Stream:    true,
 		}
 		translator := NewChatCompletionOpenAIToGCPAnthropicTranslator("", "")
-		hm, _, err := translator.RequestBody(nil, streamReq, false)
+		hm, bm, err := translator.RequestBody(nil, streamReq, false)
 		require.NoError(t, err)
 		require.NotNil(t, hm)
 
@@ -208,6 +208,9 @@ func TestOpenAIToGCPAnthropicTranslatorV1ChatCompletion_RequestBody(t *testing.T
 		require.Equal(t, ":path", pathHeader[0].Header.Key)
 		expectedPath := fmt.Sprintf("publishers/anthropic/models/%s:streamRawPredict", streamReq.Model)
 		require.Equal(t, expectedPath, string(pathHeader[0].Header.RawValue))
+
+		body := bm.GetBody()
+		require.True(t, gjson.GetBytes(body, "stream").Bool(), `body should contain "stream": true`)
 	})
 
 	t.Run("Test message param", func(t *testing.T) {

tests/extproc/testupstream_test.go

@@ -534,7 +534,7 @@ data: [DONE]
 			method:            http.MethodPost,
 			responseType:      "sse",
 			requestBody:       `{"model":"claude-3-sonnet","max_completion_tokens":1024, "messages":[{"role":"user","content":"Why is the sky blue?"}], "stream": true}`,
-			expRequestBody:    `{"max_tokens":1024,"messages":[{"content":[{"text":"Why is the sky blue?","type":"text"}],"role":"user"}],"anthropic_version":"vertex-2023-10-16"}`,
+			expRequestBody:    `{"max_tokens":1024,"messages":[{"content":[{"text":"Why is the sky blue?","type":"text"}],"role":"user"}],"stream":true,"anthropic_version":"vertex-2023-10-16"}`,
 			expHost:           "gcp-region-aiplatform.googleapis.com",
 			expPath:           "/v1/projects/gcp-project-name/locations/gcp-region/publishers/anthropic/models/claude-3-sonnet:streamRawPredict",
 			expRequestHeaders: map[string]string{"Authorization": "Bearer " + fakeGCPAuthToken},