[EXPERIMENTAL] Fallback function_calls normalizer

src/core/assistant-message/AssistantMessageParser.ts

@@ -1,6 +1,7 @@
 import { type ToolName, toolNames } from "@roo-code/types"
 import { TextContent, ToolUse, ToolParamName, toolParamNames } from "../../shared/tools"
 import { AssistantMessageContent } from "./parseAssistantMessage"
+import { FunctionCallsStreamingNormalizer } from "./functionCallsNormalizer"
 
 /**
  * Parser for assistant messages. Maintains state between chunks
@@ -17,6 +18,11 @@ export class AssistantMessageParser {
 	private readonly MAX_ACCUMULATOR_SIZE = 1024 * 1024 // 1MB limit
 	private readonly MAX_PARAM_LENGTH = 1024 * 100 // 100KB per parameter limit
 	private accumulator = ""
+	// VSCode-LM function_calls/invoke streaming normalizer
+	private normalizer = new FunctionCallsStreamingNormalizer()
+	// Minimal telemetry flags (readable by caller if needed)
+	public functionCallsNormalized = false
+	public functionCallsToolNamesEncountered = new Set<string>()
 
 	/**
 	 * Initialize a new AssistantMessageParser instance.
@@ -37,6 +43,10 @@ export class AssistantMessageParser {
 		this.currentParamName = undefined
 		this.currentParamValueStartIndex = 0
 		this.accumulator = ""
+		// Reset normalizer and telemetry
+		this.normalizer.reset()
+		this.functionCallsNormalized = false
+		this.functionCallsToolNamesEncountered.clear()
 	}
 
 	/**
@@ -52,14 +62,24 @@ export class AssistantMessageParser {
 	 * @param chunk The new chunk of text to process.
 	 */
 	public processChunk(chunk: string): AssistantMessageContent[] {
-		if (this.accumulator.length + chunk.length > this.MAX_ACCUMULATOR_SIZE) {
+		// Pre-normalize VSCode-LM function_calls/invoke XML to native tool XML
+		const normalizedChunk = this.normalizer.process(chunk)
+		// Collect minimal telemetry
+		if (this.normalizer.normalizedInLastChunk) {
+			this.functionCallsNormalized = true
+		}
+		for (const name of this.normalizer.toolNamesEncountered) {
+			this.functionCallsToolNamesEncountered.add(name)
+		}
+
+		if (this.accumulator.length + normalizedChunk.length > this.MAX_ACCUMULATOR_SIZE) {
 			throw new Error("Assistant message exceeds maximum allowed size")
 		}
 		// Store the current length of the accumulator before adding the new chunk
 		const accumulatorStartLength = this.accumulator.length
 
-		for (let i = 0; i < chunk.length; i++) {
-			const char = chunk[i]
+		for (let i = 0; i < normalizedChunk.length; i++) {
+			const char = normalizedChunk[i]
 			this.accumulator += char
 			const currentPosition = accumulatorStartLength + i
 
@@ -78,10 +98,16 @@ export class AssistantMessageParser {
 					// End of param value.
 					// Do not trim content parameters to preserve newlines, but strip first and last newline only
 					const paramValue = currentParamValue.slice(0, -paramClosingTag.length)
-					this.currentToolUse.params[this.currentParamName] =
-						this.currentParamName === "content"
-							? paramValue.replace(/^\n/, "").replace(/\n$/, "")
-							: paramValue.trim()
+					if (this.currentParamName === "content") {
+						this.currentToolUse.params[this.currentParamName] = paramValue
+							.replace(/^\n/, "")
+							.replace(/\n$/, "")
+					} else if (this.currentParamName === "args") {
+						// Preserve args exactly, including whitespace/newlines
+						this.currentToolUse.params[this.currentParamName] = paramValue
+					} else {
+						this.currentToolUse.params[this.currentParamName] = paramValue.trim()
+					}
 					this.currentParamName = undefined
 					continue
 				} else {

src/core/assistant-message/__tests__/AssistantMessageParser.spec.ts

@@ -392,3 +392,69 @@ describe("AssistantMessageParser (streaming)", () => {
 		})
 	})
 })
+
+// VSCode-LM function_calls normalizer tests (streaming)
+describe("VSCode-LM function_calls normalizer (streaming)", () => {
+	it("should normalize single invoke with args preserved", () => {
+		const parser = new AssistantMessageParser()
+		const argsXml = "<file><path>src/a.ts</path></file>"
+		const message = `<function_calls><invoke name="read_file"><args>${argsXml}</args></invoke></function_calls>`
+		const result = streamChunks(parser, message).filter((block) => !isEmptyTextContent(block))
+		expect(result).toHaveLength(1)
+		const toolUse = result[0] as ToolUse
+		expect(toolUse.type).toBe("tool_use")
+		expect(toolUse.name).toBe("read_file")
+		expect(toolUse.params.args).toBe(argsXml)
+		expect(toolUse.partial).toBe(false)
+	})
+
+	it("should handle multiple invokes with surrounding text", () => {
+		const parser = new AssistantMessageParser()
+		const args1 = "<file><path>file1.ts</path></file>"
+		const args2 = "<file><path>file2.ts</path></file>"
+		const message = `Before <function_calls><invoke name="read_file"><args>${args1}</args></invoke></function_calls> Middle <function_calls><invoke name="read_file"><args>${args2}</args></invoke></function_calls> After`
+		const result = streamChunks(parser, message)
+		expect(result).toHaveLength(5)
+
+		expect(result[0].type).toBe("text")
+		expect((result[0] as TextContent).content).toBe("Before")
+
+		const toolUse1 = result[1] as ToolUse
+		expect(toolUse1.type).toBe("tool_use")
+		expect(toolUse1.name).toBe("read_file")
+		expect(toolUse1.params.args).toBe(args1)
+
+		expect(result[2].type).toBe("text")
+		expect((result[2] as TextContent).content).toBe("Middle")
+
+		const toolUse2 = result[3] as ToolUse
+		expect(toolUse2.type).toBe("tool_use")
+		expect(toolUse2.name).toBe("read_file")
+		expect(toolUse2.params.args).toBe(args2)
+
+		expect(result[4].type).toBe("text")
+		expect((result[4] as TextContent).content).toBe("After")
+	})
+
+	it("should pass through unknown invoke as text and not create tool_use", () => {
+		const parser = new AssistantMessageParser()
+		const message = `<function_calls><invoke name="unknown_tool"><args><x>y</x></args></invoke></function_calls>`
+		const result = streamChunks(parser, message)
+		expect(result).toHaveLength(1)
+		const text = result[0] as TextContent
+		expect(text.type).toBe("text")
+		expect(text.content).toContain('<invoke name="unknown_tool">')
+	})
+
+	it("should preserve multi-file args xml exactly", () => {
+		const parser = new AssistantMessageParser()
+		const argsXml = "<file><path>a.ts</path></file><file><path>b.ts</path></file>"
+		const message = `<function_calls><invoke name="read_file"><args>${argsXml}</args></invoke></function_calls>`
+		const result = streamChunks(parser, message).filter((block) => !isEmptyTextContent(block))
+		expect(result).toHaveLength(1)
+		const toolUse = result[0] as ToolUse
+		expect(toolUse.type).toBe("tool_use")
+		expect(toolUse.name).toBe("read_file")
+		expect(toolUse.params.args).toBe(argsXml)
+	})
+})

src/core/assistant-message/__tests__/parseAssistantMessage.spec.ts

@@ -338,3 +338,78 @@ const isEmptyTextContent = (block: AssistantMessageContent) =>
 		})
 	})
 })
+
+// VSCode-LM function_calls normalizer tests (non-stream)
+;[parseAssistantMessageV1, parseAssistantMessageV2].forEach((parser, index) => {
+	describe(`VSCode-LM function_calls normalizer (non-stream) V${index + 1}`, () => {
+		it("should normalize single invoke with args preserved", () => {
+			const argsXml = "<file><path>src/a.ts</path></file>"
+			const message = `<function_calls><invoke name="read_file"><args>${argsXml}</args></invoke></function_calls>`
+			const result = parser(message).filter((block) => !isEmptyTextContent(block))
+			expect(result).toHaveLength(1)
+			const toolUse = result[0] as ToolUse
+			expect(toolUse.type).toBe("tool_use")
+			expect(toolUse.name).toBe("read_file")
+			expect(toolUse.params.args).toBe(argsXml)
+			expect(toolUse.partial).toBe(false)
+		})
+
+		it("should handle multiple invokes with surrounding text", () => {
+			const args1 = "<file><path>file1.ts</path></file>"
+			const args2 = "<file><path>file2.ts</path></file>"
+			const message = `Before <function_calls><invoke name="read_file"><args>${args1}</args></invoke></function_calls> Middle <function_calls><invoke name="read_file"><args>${args2}</args></invoke></function_calls> After`
+			const result = parser(message)
+
+			expect(result).toHaveLength(5)
+
+			expect(result[0].type).toBe("text")
+			expect((result[0] as TextContent).content).toBe("Before")
+
+			const toolUse1 = result[1] as ToolUse
+			expect(toolUse1.type).toBe("tool_use")
+			expect(toolUse1.name).toBe("read_file")
+			expect(toolUse1.params.args).toBe(args1)
+
+			expect(result[2].type).toBe("text")
+			expect((result[2] as TextContent).content).toBe("Middle")
+
+			const toolUse2 = result[3] as ToolUse
+			expect(toolUse2.type).toBe("tool_use")
+			expect(toolUse2.name).toBe("read_file")
+			expect(toolUse2.params.args).toBe(args2)
+
+			expect(result[4].type).toBe("text")
+			expect((result[4] as TextContent).content).toBe("After")
+		})
+
+		it("should pass through unknown invoke as text and not create tool_use", () => {
+			const message = `<function_calls><invoke name="unknown_tool"><args><x>y</x></args></invoke></function_calls>`
+			const result = parser(message)
+			expect(result).toHaveLength(1)
+			const text = result[0] as TextContent
+			expect(text.type).toBe("text")
+			expect(text.content).toContain('<invoke name="unknown_tool">')
+		})
+
+		it("should preserve multi-file args xml exactly", () => {
+			const argsXml = "<file><path>a.ts</path></file><file><path>b.ts</path></file>"
+			const message = `<function_calls><invoke name="read_file"><args>${argsXml}</args></invoke></function_calls>`
+			const result = parser(message).filter((block) => !isEmptyTextContent(block))
+			expect(result).toHaveLength(1)
+			const toolUse = result[0] as ToolUse
+			expect(toolUse.type).toBe("tool_use")
+			expect(toolUse.name).toBe("read_file")
+			expect(toolUse.params.args).toBe(argsXml)
+		})
+
+		it("should be idempotent for native tool XML (no changes)", () => {
+			const native = "<read_file><path>src/x.ts</path></read_file>"
+			const result = parser(native).filter((b) => !isEmptyTextContent(b))
+			expect(result).toHaveLength(1)
+			const toolUse = result[0] as ToolUse
+			expect(toolUse.name).toBe("read_file")
+			expect(toolUse.params.path).toBe("src/x.ts")
+			expect(toolUse.partial).toBe(false)
+		})
+	})
+})