fix: handle primitives error in StreamingToolCallProcessor

Author: NaccOll

src/core/task/__tests__/tool-call-helper.spec.ts

@@ -310,4 +310,64 @@ describe("handleOpenaiToolCallStreaming", () => {
 		expect(xml).toContain("<echo>")
 		expect(xml).toContain("<msg>hi</msg>")
 	})
+
+	it("should delegate to processor.processChunk apply_diff", () => {
+		const processor = new StreamingToolCallProcessor()
+		const chunk = [
+			{
+				index: 0,
+				id: "1",
+				function: {
+					name: "apply_diff",
+					arguments:
+						'{"args":{"file":[{"diff":[{"replace":"catch (Exception e) {if (true) {}throw e;}","search":"catch (Exception e) {throw e;}","start_line":252}],"path":"Test.java"}]}}',
+				},
+			},
+		]
+		const xml = handleOpenaiToolCallStreaming(processor, chunk, "openai").chunkContent
+		expect(xml).toContain("<search>")
+	})
+
+	it("should delegate to processor.processChunk apply_diff2", () => {
+		const processor = new StreamingToolCallProcessor()
+		const chunk1 = [
+			{
+				index: 0,
+				id: "1",
+				function: {
+					name: "apply_diff",
+					arguments:
+						'{"args":{"file":[{"diff":[{"replace":"catch (Exception e) {if (1==1) {}throw e;}","test":tr',
+				},
+			},
+		]
+		const chunk2 = [
+			{
+				index: 0,
+				id: "",
+				function: {
+					name: "",
+					arguments: 'ue,"search":"catch (Exception e) {throw e;}","start_line":25',
+				},
+			},
+		]
+		const chunk3 = [
+			{
+				index: 0,
+				id: "",
+				function: {
+					name: "",
+					arguments: '2}],"path":"Test.java"}]}}',
+				},
+			},
+		]
+		let xml = handleOpenaiToolCallStreaming(processor, chunk1, "openai").chunkContent
+		expect(xml).not.toContain("<search>")
+		expect(xml).not.toContain("true")
+		xml += handleOpenaiToolCallStreaming(processor, chunk2, "openai").chunkContent
+		expect(xml).toContain("<search>")
+		expect(xml).toContain("true")
+		xml += handleOpenaiToolCallStreaming(processor, chunk3, "openai").chunkContent
+		expect(xml).toContain("252")
+	})
 })

src/core/task/tool-call-helper.ts

@@ -12,6 +12,7 @@ import { ToolCallProviderType } from "../../shared/tools"
  * Defines the possible states of the JSON parser.
  */
 enum ParserState {
+	EXPECT_ROOT, // Expecting root object or array
 	EXPECT_VALUE,
 	EXPECT_KEY,
 	EXPECT_COLON,
@@ -41,7 +42,7 @@ class ToolCallProcessingState {
 	cursor = 0
 
 	// The current state of the parser FSM (Finite State Machine).
-	parserState = ParserState.EXPECT_VALUE
+	parserState = ParserState.EXPECT_ROOT
 
 	// Flags for handling string parsing.
 	inString = false
@@ -54,6 +55,8 @@ class ToolCallProcessingState {
 	xmlTagStack: string[] = []
 	// Buffer for the current string literal (key or value) being parsed.
 	currentString = ""
+	// Buffer for accumulating primitive values across chunks
+	primitiveBuffer = ""
 	// Flag to track if we're at the start of an array to prevent duplicate tags.
 	justOpenedArray = false
 }
@@ -315,24 +318,46 @@ export class StreamingToolCallProcessor {
 				continue
 			}
 
-			// Handle primitives (numbers, booleans, null)
+			// Handle primitives - accumulate characters until we hit a delimiter
 			if (state.parserState === ParserState.EXPECT_VALUE) {
-				const primitiveMatch = args.substring(state.cursor).match(/^-?\d+(\.\d+)?|true|false|null/)
-				if (primitiveMatch) {
-					const value = primitiveMatch[0]
-					const tag = state.xmlTagStack.pop()!
-					if (tag) {
-						xml += `${value}${this.onCloseTag(tag, toolName)}`
-					}
-					state.parserState = ParserState.EXPECT_COMMA_OR_CLOSING
-					state.cursor += value.length
+				// Check if this character could be part of a primitive value
+				if (
+					(char >= "0" && char <= "9") ||
+					char === "-" ||
+					char === "." ||
+					(char >= "a" && char <= "z") ||
+					(char >= "A" && char <= "Z")
+				) {
+					// Accumulate the character
+					state.primitiveBuffer += char
+					state.cursor++
 					continue
+				} else if (state.primitiveBuffer.length > 0) {
+					// We've hit a delimiter, check if we have a complete primitive
+					const value = state.primitiveBuffer.trim()
+					if (value === "true" || value === "false" || value === "null" || /^-?\d+(\.\d+)?$/.test(value)) {
+						// We have a valid primitive
+						const tag = state.xmlTagStack.pop()!
+						if (tag) {
+							xml += `${value}${this.onCloseTag(tag, toolName)}`
+						}
+						state.parserState = ParserState.EXPECT_COMMA_OR_CLOSING
+						state.primitiveBuffer = ""
+						// Don't increment cursor - let the delimiter be processed in the switch
+						continue
+					} else {
+						// Invalid primitive, reset buffer and continue
+						state.primitiveBuffer = ""
+					}
 				}
 			}
 
 			switch (char) {
 				case "{":
-					if (state.parserState === ParserState.EXPECT_VALUE) {
+					if (
+						state.parserState === ParserState.EXPECT_VALUE ||
+						state.parserState === ParserState.EXPECT_ROOT
+					) {
 						const parent = state.bracketStack[state.bracketStack.length - 1]
 						if (parent === "[") {
 							// For an object inside an array, we might need to add the repeating tag.
@@ -381,7 +406,10 @@ export class StreamingToolCallProcessor {
 					}
 					break
 				case "[":
-					if (state.parserState === ParserState.EXPECT_VALUE) {
+					if (
+						state.parserState === ParserState.EXPECT_VALUE ||
+						state.parserState === ParserState.EXPECT_ROOT
+					) {
 						state.bracketStack.push("[")
 						state.parserState = ParserState.EXPECT_VALUE // An array contains values
 						state.justOpenedArray = true
@@ -401,11 +429,12 @@ export class StreamingToolCallProcessor {
 					if (state.parserState === ParserState.EXPECT_VALUE) {
 						// We've encountered the start of a string that is a JSON value.
 						state.isStreamingStringValue = true
-					} else {
+						state.inString = true
+					} else if (state.parserState === ParserState.EXPECT_KEY) {
 						// This is the start of a string that is a JSON key.
 						state.isStreamingStringValue = false
+						state.inString = true
 					}
-					state.inString = true
 					break
 				case ":":
 					if (state.parserState === ParserState.EXPECT_COLON) {