fix: fix tool-calls finish reason conversion in streaming mode for newer gemini models like gemini-3.5-flash (envoyproxy#2399)

**Description**

***Problem***

When the model returns a tool call, OpenAI expects the finish reason to
be `tool_calls` while Gemini returns `STOP`. We convert it via
```
if len(toolCalls) > 0 {
			return openai.ChatCompletionChoicesFinishReasonToolCalls
		}
```
This was correct for streaming mode because the finish reason and the
tool calls are in the same chunk in the previous Gemini models. However,
for newer Gemini models like gemini-3.5-flash, they are split into
**two** chunks. The `functionCall` chunk has
**no** `finishReason`, and a **trailing** chunk carries `finishReason:
STOP` with an empty
text part and **no** `functionCall`:

```jsonc
// raw GCP chunk 1 — has the tool call, no finishReason
{"candidates":[{"content":{"parts":[{"functionCall":{"name":"get_current_weather","args":{"...":"..."}},
                                     "thoughtSignature":"..."}]}}],
 "usageMetadata":{"trafficType":"ON_DEMAND"}}

// raw GCP chunk 2 — terminal STOP, empty text, no functionCall
{"candidates":[{"content":{"parts":[{"text":""}]},"finishReason":"STOP"}],
 "usageMetadata":{"promptTokenCount":107,"candidatesTokenCount":31,"thoughtsTokenCount":148,"...":"..."}}
```


In this case, the conversion would be invalid. Thus, we need to remember
whether the model returned a tool call during the streaming mode.

---------

Signed-off-by: yxia216 <yxia216@bloomberg.net>

Author: hustxiayang

internal/translator/openai_gcpvertexai.go

@@ -82,6 +82,12 @@ type openAIToGCPVertexAITranslatorV1ChatCompletion struct {
 	bufferedBody      []byte // Buffer for incomplete JSON chunks.
 	requestModel      internalapi.RequestModel
 	toolCallIndex     int64
+	// streamedToolCall records whether any tool call has been emitted so far in
+	// the streaming response. Newer Gemini models (e.g. gemini-3.5-flash,
+	// gemini-3.1-flash-lite) stream the terminal STOP on a separate chunk that no
+	// longer carries the functionCall part, so the finish_reason must be derived
+	// from the whole stream, not just the current chunk.
+	streamedToolCall bool
 	// Redaction configuration for debug logging
 	debugLogEnabled bool
 	enableRedaction bool
@@ -444,7 +450,25 @@ func (o *openAIToGCPVertexAITranslatorV1ChatCompletion) geminiCandidatesToOpenAI
 		} else {
 			choice.Delta = &openai.ChatCompletionResponseChunkChoiceDelta{}
 		}
+
+		// Track whether a tool call has been streamed at any point in the response.
+		if len(toolCalls) > 0 {
+			o.streamedToolCall = true
+		}
+
 		choice.FinishReason = geminiFinishReasonToOpenAI(candidate.FinishReason, toolCalls)
+		// Newer Gemini models (e.g. gemini-3.5-flash, gemini-3.1-flash-lite) stream
+		// the terminal STOP on a separate chunk whose parts no longer contain the
+		// functionCall (e.g. an empty text part), so the per-chunk toolCalls slice
+		// is empty and geminiFinishReasonToOpenAI maps it to "stop". If a tool call
+		// was streamed in an earlier chunk, the correct OpenAI finish_reason for the
+		// completion is still "tool_calls". Older Gemini models carried the
+		// functionCall and STOP in the same chunk, so this only affects the newer
+		// split-chunk shape.
+		if choice.FinishReason == openai.ChatCompletionChoicesFinishReasonStop && o.streamedToolCall {
+			choice.FinishReason = openai.ChatCompletionChoicesFinishReasonToolCalls
+		}
+
 		choices = append(choices, choice)
 	}
 

internal/translator/openai_gcpvertexai_test.go

@@ -1944,6 +1944,59 @@ data: {"candidates":[{"content":{"parts":[{"functionCall":{"name":"get_weather",
 	require.Equal(t, uint32(10), outputTokens)
 }
 
+func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingToolCallSplitFinishReason(t *testing.T) {
+	// Newer Gemini models (e.g. gemini-3.5-flash, gemini-3.1-flash-lite) stream
+	// the functionCall and the terminal STOP in separate chunks: the functionCall
+	// chunk carries no finishReason, and a trailing chunk carries finishReason=STOP
+	// with an empty text part (no functionCall). The completion's finish_reason
+	// must still be "tool_calls", not "stop". (Older Gemini models carried both in
+	// a single chunk; that case is covered by
+	// TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingToolCallWithSignature.)
+	translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-3.5-flash").(*openAIToGCPVertexAITranslatorV1ChatCompletion)
+
+	gcpStreamingChunk := `data: {"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"get_weather","args":{"location":"Paris"}},"thoughtSignature":"dG9vbGNhbGxzaWduYXR1cmU="}]}}],"usageMetadata":{"trafficType":"ON_DEMAND"}}
+
+data: {"candidates":[{"content":{"role":"model","parts":[{"text":""}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":15,"candidatesTokenCount":10,"totalTokenCount":25,"thoughtsTokenCount":8}}`
+
+	headerMut, body, tokenUsage, _, err := translator.handleStreamingResponse(
+		bytes.NewReader([]byte(gcpStreamingChunk)),
+		false,
+		nil,
+	)
+
+	require.Nil(t, headerMut)
+	require.NoError(t, err)
+	require.NotNil(t, body)
+
+	chatCompletionChunks := getChatCompletionResponseChunk(body)
+	// We expect 3 chunks: tool call (no finish_reason), terminal STOP, and usage.
+	require.Len(t, chatCompletionChunks, 3)
+
+	// First chunk carries the tool call and no finish_reason yet.
+	firstChunk := chatCompletionChunks[0]
+	require.Len(t, firstChunk.Choices, 1)
+	assert.Equal(t, openai.ChatCompletionChoicesFinishReason(""), firstChunk.Choices[0].FinishReason)
+	require.Len(t, firstChunk.Choices[0].Delta.ToolCalls, 1)
+	assert.Equal(t, "get_weather", firstChunk.Choices[0].Delta.ToolCalls[0].Function.Name)
+	assert.JSONEq(t, `{"location":"Paris"}`, firstChunk.Choices[0].Delta.ToolCalls[0].Function.Arguments)
+
+	// Second chunk carries the terminal STOP but no tool call. The finish_reason
+	// must be rewritten to "tool_calls" because a tool call was already streamed.
+	secondChunk := chatCompletionChunks[1]
+	require.Len(t, secondChunk.Choices, 1)
+	assert.Equal(t, openai.ChatCompletionChoicesFinishReason("tool_calls"), secondChunk.Choices[0].FinishReason)
+	assert.Empty(t, secondChunk.Choices[0].Delta.ToolCalls)
+
+	// Third chunk is usage.
+	thirdChunk := chatCompletionChunks[2]
+	assert.NotNil(t, thirdChunk.Usage)
+
+	// Completion tokens = candidatesTokenCount(10) + thoughtsTokenCount(8).
+	outputTokens, ok := tokenUsage.OutputTokens()
+	require.True(t, ok)
+	require.Equal(t, uint32(18), outputTokens)
+}
+
 func TestOpenAIToGCPVertexAITranslatorV1ChatCompletion_StreamingEndOfStream(t *testing.T) {
 	translator := NewChatCompletionOpenAIToGCPVertexAITranslator("gemini-2.0-flash-001").(*openAIToGCPVertexAITranslatorV1ChatCompletion)