fix: finish reason should be tool calls when the model responded with a tool call (envoyproxy#1486)

**Description**

Finish reason should be tool calls if the model returns a tool call
response. In vertex api, there is no tool call finish reason, thus need
a work around to make it compatible.

---------

Signed-off-by: yxia216 <[email protected]>
Co-authored-by: Dan Sun <[email protected]>

Author: hustxiayang

internal/extproc/translator/gemini_helper.go

@@ -529,10 +529,12 @@ func geminiCandidatesToOpenAIChoices(candidates []*genai.Candidate, responseMode
 
 		// Create the choice.
 		choice := openai.ChatCompletionResponseChoice{
-			Index:        int64(idx),
-			FinishReason: geminiFinishReasonToOpenAI(candidate.FinishReason),
+			Index: int64(idx),
 		}
 
+		toolCalls := []openai.ChatCompletionMessageToolCallParam{}
+		var err error
+
 		if candidate.Content != nil {
 			message := openai.ChatCompletionResponseChoiceMessage{
 				Role: openai.ChatMessageRoleAssistant,
@@ -542,7 +544,7 @@ func geminiCandidatesToOpenAIChoices(candidates []*genai.Candidate, responseMode
 			message.Content = &content
 
 			// Extract tool calls if any.
-			toolCalls, err := extractToolCallsFromGeminiParts(candidate.Content.Parts)
+			toolCalls, err = extractToolCallsFromGeminiParts(toolCalls, candidate.Content.Parts)
 			if err != nil {
 				return nil, fmt.Errorf("error extracting tool calls: %w", err)
 			}
@@ -569,16 +571,26 @@ func geminiCandidatesToOpenAIChoices(candidates []*genai.Candidate, responseMode
 			choice.Logprobs = geminiLogprobsToOpenAILogprobs(*candidate.LogprobsResult)
 		}
 
+		choice.FinishReason = geminiFinishReasonToOpenAI(candidate.FinishReason, toolCalls)
+
 		choices = append(choices, choice)
 	}
 
 	return choices, nil
 }
 
+// Define a type constraint that includes both stream and non-stream tool call slice types.
+type toolCallSlice interface {
+	[]openai.ChatCompletionMessageToolCallParam | []openai.ChatCompletionChunkChoiceDeltaToolCall
+}
+
 // geminiFinishReasonToOpenAI converts Gemini finish reason to OpenAI finish reason.
-func geminiFinishReasonToOpenAI(reason genai.FinishReason) openai.ChatCompletionChoicesFinishReason {
+func geminiFinishReasonToOpenAI[T toolCallSlice](reason genai.FinishReason, toolCalls T) openai.ChatCompletionChoicesFinishReason {
 	switch reason {
 	case genai.FinishReasonStop:
+		if len(toolCalls) > 0 {
+			return openai.ChatCompletionChoicesFinishReasonToolCalls
+		}
 		return openai.ChatCompletionChoicesFinishReasonStop
 	case genai.FinishReasonMaxTokens:
 		return openai.ChatCompletionChoicesFinishReasonLength
@@ -611,9 +623,7 @@ func extractTextFromGeminiParts(parts []*genai.Part, responseMode geminiResponse
 }
 
 // extractToolCallsFromGeminiParts extracts tool calls from Gemini parts.
-func extractToolCallsFromGeminiParts(parts []*genai.Part) ([]openai.ChatCompletionMessageToolCallParam, error) {
-	var toolCalls []openai.ChatCompletionMessageToolCallParam
-
+func extractToolCallsFromGeminiParts(toolCalls []openai.ChatCompletionMessageToolCallParam, parts []*genai.Part) ([]openai.ChatCompletionMessageToolCallParam, error) {
 	for _, part := range parts {
 		if part == nil || part.FunctionCall == nil {
 			continue
@@ -650,8 +660,7 @@ func extractToolCallsFromGeminiParts(parts []*genai.Part) ([]openai.ChatCompleti
 // extractToolCallsFromGeminiPartsStream extracts tool calls from Gemini parts for streaming responses.
 // Each tool call is assigned an incremental index starting from 0, matching OpenAI's streaming protocol.
 // Returns ChatCompletionChunkChoiceDeltaToolCall types suitable for streaming responses, or nil if no tool calls are found.
-func extractToolCallsFromGeminiPartsStream(parts []*genai.Part) ([]openai.ChatCompletionChunkChoiceDeltaToolCall, error) {
-	var toolCalls []openai.ChatCompletionChunkChoiceDeltaToolCall
+func extractToolCallsFromGeminiPartsStream(toolCalls []openai.ChatCompletionChunkChoiceDeltaToolCall, parts []*genai.Part) ([]openai.ChatCompletionChunkChoiceDeltaToolCall, error) {
 	toolCallIndex := int64(0)
 
 	for _, part := range parts {
@@ -772,10 +781,11 @@ func geminiCandidatesToOpenAIStreamingChoices(candidates []*genai.Candidate, res
 
 		// Create the streaming choice.
 		choice := openai.ChatCompletionResponseChunkChoice{
-			Index:        0,
-			FinishReason: geminiFinishReasonToOpenAI(candidate.FinishReason),
+			Index: 0,
 		}
 
+		toolCalls := []openai.ChatCompletionChunkChoiceDeltaToolCall{}
+		var err error
 		if candidate.Content != nil {
 			delta := &openai.ChatCompletionResponseChunkChoiceDelta{
 				Role: openai.ChatMessageRoleAssistant,
@@ -788,15 +798,15 @@ func geminiCandidatesToOpenAIStreamingChoices(candidates []*genai.Candidate, res
 			}
 
 			// Extract tool calls if any.
-			toolCalls, err := extractToolCallsFromGeminiPartsStream(candidate.Content.Parts)
+			toolCalls, err = extractToolCallsFromGeminiPartsStream(toolCalls, candidate.Content.Parts)
 			if err != nil {
 				return nil, fmt.Errorf("error extracting tool calls: %w", err)
 			}
 			delta.ToolCalls = toolCalls
 
 			choice.Delta = delta
 		}
-
+		choice.FinishReason = geminiFinishReasonToOpenAI(candidate.FinishReason, toolCalls)
 		choices = append(choices, choice)
 	}
 

internal/extproc/translator/gemini_helper_test.go

@@ -1271,6 +1271,7 @@ func TestGeminiLogprobsToOpenAILogprobs(t *testing.T) {
 }
 
 func TestExtractToolCallsFromGeminiParts(t *testing.T) {
+	toolCalls := []openai.ChatCompletionMessageToolCallParam{}
 	tests := []struct {
 		name     string
 		input    []*genai.Part
@@ -1360,7 +1361,7 @@ func TestExtractToolCallsFromGeminiParts(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			calls, err := extractToolCallsFromGeminiParts(tt.input)
+			calls, err := extractToolCallsFromGeminiParts(toolCalls, tt.input)
 
 			if tt.wantErr {
 				require.Error(t, err)
@@ -1381,56 +1382,80 @@ func TestExtractToolCallsFromGeminiParts(t *testing.T) {
 
 func TestGeminiFinishReasonToOpenAI(t *testing.T) {
 	tests := []struct {
-		name     string
-		input    genai.FinishReason
-		expected openai.ChatCompletionChoicesFinishReason
+		name      string
+		input     genai.FinishReason
+		toolCalls []openai.ChatCompletionMessageToolCallParam
+		expected  openai.ChatCompletionChoicesFinishReason
 	}{
 		{
-			name:     "stop reason",
-			input:    genai.FinishReasonStop,
-			expected: openai.ChatCompletionChoicesFinishReasonStop,
+			name:      "stop reason",
+			input:     genai.FinishReasonStop,
+			toolCalls: []openai.ChatCompletionMessageToolCallParam{},
+			expected:  openai.ChatCompletionChoicesFinishReasonStop,
+		},
+		{
+			name:  "tool calls reason",
+			input: genai.FinishReasonStop,
+			toolCalls: []openai.ChatCompletionMessageToolCallParam{
+				{
+					ID: ptr.To("tool_call_1"),
+					Function: openai.ChatCompletionMessageToolCallFunctionParam{
+						Name:      "example_tool",
+						Arguments: "{\"param1\":\"value1\"}",
+					},
+					Type: openai.ChatCompletionMessageToolCallTypeFunction,
+				},
+			},
+			expected: openai.ChatCompletionChoicesFinishReasonToolCalls,
 		},
 		{
-			name:     "max tokens reason",
-			input:    genai.FinishReasonMaxTokens,
-			expected: openai.ChatCompletionChoicesFinishReasonLength,
+			name:      "max tokens reason",
+			input:     genai.FinishReasonMaxTokens,
+			toolCalls: []openai.ChatCompletionMessageToolCallParam{},
+			expected:  openai.ChatCompletionChoicesFinishReasonLength,
 		},
 		{
-			name:     "empty reason for streaming",
-			input:    "",
-			expected: "",
+			name:      "empty reason for streaming",
+			input:     "",
+			toolCalls: []openai.ChatCompletionMessageToolCallParam{},
+			expected:  "",
 		},
 		{
-			name:     "safety reason",
-			input:    genai.FinishReasonSafety,
-			expected: openai.ChatCompletionChoicesFinishReasonContentFilter,
+			name:      "safety reason",
+			input:     genai.FinishReasonSafety,
+			toolCalls: []openai.ChatCompletionMessageToolCallParam{},
+			expected:  openai.ChatCompletionChoicesFinishReasonContentFilter,
 		},
 		{
-			name:     "recitation reason",
-			input:    genai.FinishReasonRecitation,
-			expected: openai.ChatCompletionChoicesFinishReasonContentFilter,
+			name:      "recitation reason",
+			input:     genai.FinishReasonRecitation,
+			toolCalls: []openai.ChatCompletionMessageToolCallParam{},
+			expected:  openai.ChatCompletionChoicesFinishReasonContentFilter,
 		},
 		{
-			name:     "other reason",
-			input:    genai.FinishReasonOther,
-			expected: openai.ChatCompletionChoicesFinishReasonContentFilter,
+			name:      "other reason",
+			input:     genai.FinishReasonOther,
+			toolCalls: []openai.ChatCompletionMessageToolCallParam{},
+			expected:  openai.ChatCompletionChoicesFinishReasonContentFilter,
 		},
 		{
-			name:     "unknown reason",
-			input:    genai.FinishReason("unknown_reason"),
-			expected: openai.ChatCompletionChoicesFinishReasonContentFilter,
+			name:      "unknown reason",
+			input:     genai.FinishReason("unknown_reason"),
+			toolCalls: []openai.ChatCompletionMessageToolCallParam{},
+			expected:  openai.ChatCompletionChoicesFinishReasonContentFilter,
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result := geminiFinishReasonToOpenAI(tt.input)
+			result := geminiFinishReasonToOpenAI(tt.input, tt.toolCalls)
 			require.Equal(t, tt.expected, result)
 		})
 	}
 }
 
 func TestExtractToolCallsFromGeminiPartsStream(t *testing.T) {
+	toolCalls := []openai.ChatCompletionChunkChoiceDeltaToolCall{}
 	tests := []struct {
 		name     string
 		input    []*genai.Part
@@ -1675,7 +1700,7 @@ func TestExtractToolCallsFromGeminiPartsStream(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			calls, err := extractToolCallsFromGeminiPartsStream(tt.input)
+			calls, err := extractToolCallsFromGeminiPartsStream(toolCalls, tt.input)
 
 			if tt.wantErr {
 				require.Error(t, err)
@@ -1696,6 +1721,8 @@ func TestExtractToolCallsFromGeminiPartsStream(t *testing.T) {
 
 // TestExtractToolCallsStreamVsNonStream tests the differences between streaming and non-streaming extraction
 func TestExtractToolCallsStreamVsNonStream(t *testing.T) {
+	toolCalls := []openai.ChatCompletionMessageToolCallParam{}
+	toolCallsStream := []openai.ChatCompletionChunkChoiceDeltaToolCall{}
 	parts := []*genai.Part{
 		{
 			FunctionCall: &genai.FunctionCall{
@@ -1709,11 +1736,11 @@ func TestExtractToolCallsStreamVsNonStream(t *testing.T) {
 	}
 
 	// Get results from both functions
-	streamCalls, err := extractToolCallsFromGeminiPartsStream(parts)
+	streamCalls, err := extractToolCallsFromGeminiPartsStream(toolCallsStream, parts)
 	require.NoError(t, err)
 	require.Len(t, streamCalls, 1)
 
-	nonStreamCalls, err := extractToolCallsFromGeminiParts(parts)
+	nonStreamCalls, err := extractToolCallsFromGeminiParts(toolCalls, parts)
 	require.NoError(t, err)
 	require.Len(t, nonStreamCalls, 1)
 
@@ -1749,6 +1776,7 @@ func TestExtractToolCallsStreamVsNonStream(t *testing.T) {
 
 // TestExtractToolCallsStreamIndexing specifically tests that multiple tool calls get correct indices
 func TestExtractToolCallsStreamIndexing(t *testing.T) {
+	toolCalls := []openai.ChatCompletionChunkChoiceDeltaToolCall{}
 	parts := []*genai.Part{
 		{
 			FunctionCall: &genai.FunctionCall{
@@ -1771,7 +1799,7 @@ func TestExtractToolCallsStreamIndexing(t *testing.T) {
 		},
 	}
 
-	calls, err := extractToolCallsFromGeminiPartsStream(parts)
+	calls, err := extractToolCallsFromGeminiPartsStream(toolCalls, parts)
 	require.NoError(t, err)
 	require.Len(t, calls, 3)
 

tests/extproc/testupstream_test.go

@@ -297,7 +297,7 @@ func TestWithTestUpstream(t *testing.T) {
 			responseStatus:    strconv.Itoa(http.StatusOK),
 			responseBody:      `{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"get_delivery_date","args":{"order_id":"123"}}}]},"finishReason":"STOP","avgLogprobs":0.000001220789272338152}],"usageMetadata":{"promptTokenCount":50,"candidatesTokenCount":11,"totalTokenCount":61,"trafficType":"ON_DEMAND","promptTokensDetails":[{"modality":"TEXT","tokenCount":50}],"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":11}]},"modelVersion":"gemini-2.0-flash-001","createTime":"2025-07-11T22:15:44.956335Z","responseId":"EI5xaK-vOtqJm22IPmuCR14AI"}`,
 			expStatus:         http.StatusOK,
-			expResponseBody:   `{"choices":[{"finish_reason":"stop","index":0,"message":{"role":"assistant","tool_calls":[{"id":"703482f8-2e5b-4dcc-a872-d74bd66c3866","function":{"arguments":"{\"order_id\":\"123\"}","name":"get_delivery_date"},"type":"function"}]}}],"model":"gemini-2.0-flash-001","object":"chat.completion","usage":{"completion_tokens":11,"completion_tokens_details":{},"prompt_tokens":50,"total_tokens":61,"prompt_tokens_details":{}}}`,
+			expResponseBody:   `{"choices":[{"finish_reason":"tool_calls","index":0,"message":{"role":"assistant","tool_calls":[{"id":"703482f8-2e5b-4dcc-a872-d74bd66c3866","function":{"arguments":"{\"order_id\":\"123\"}","name":"get_delivery_date"},"type":"function"}]}}],"model":"gemini-2.0-flash-001","object":"chat.completion","usage":{"completion_tokens":11,"completion_tokens_details":{},"prompt_tokens":50,"total_tokens":61,"prompt_tokens_details":{}}}`,
 		},
 		{
 			name:              "gcp-anthropicai - /v1/chat/completions",