Merge pull request #3331 from Kilo-Org/mark/stateless-parser

make the autocomplete parser stateless

Author: markijbema

src/services/ghost/GhostProvider.ts

@@ -2,7 +2,7 @@ import crypto from "crypto"
 import * as vscode from "vscode"
 import { t } from "../../i18n"
 import { GhostDocumentStore } from "./GhostDocumentStore"
-import { GhostStreamingParser } from "./classic-auto-complete/GhostStreamingParser"
+import { parseGhostResponse } from "./classic-auto-complete/GhostStreamingParser"
 import { AutoTriggerStrategy } from "./classic-auto-complete/AutoTriggerStrategy"
 import { GhostModel } from "./GhostModel"
 import { GhostSuggestionContext, contextToAutocompleteInput, extractPrefixSuffix } from "./types"
@@ -24,7 +24,6 @@ export class GhostProvider {
 	private static instance: GhostProvider | null = null
 	private documentStore: GhostDocumentStore
 	private model: GhostModel
-	private streamingParser: GhostStreamingParser
 	private autoTriggerStrategy: AutoTriggerStrategy
 	private suggestions: GhostSuggestionsState = new GhostSuggestionsState()
 	private cline: ClineProvider
@@ -58,7 +57,6 @@ export class GhostProvider {
 
 		// Register Internal Components
 		this.documentStore = new GhostDocumentStore()
-		this.streamingParser = new GhostStreamingParser()
 		this.autoTriggerStrategy = new AutoTriggerStrategy()
 		this.providerSettingsManager = new ProviderSettingsManager(context)
 		this.model = new GhostModel()
@@ -313,9 +311,6 @@ export class GhostProvider {
 			await this.load()
 		}
 
-		// Initialize the streaming parser
-		this.streamingParser.initialize(context)
-
 		let hasShownFirstSuggestion = false
 		let cost = 0
 		let inputTokens = 0
@@ -368,7 +363,7 @@ export class GhostProvider {
 			}
 
 			// Finish the streaming parser to apply sanitization if needed
-			const finalParseResult = this.streamingParser.parseResponse(response, prefix, suffix)
+			const finalParseResult = parseGhostResponse(response, prefix, suffix, context.document, context.range)
 
 			if (finalParseResult.suggestions.getFillInAtCursor()) {
 				console.info("Final suggestion:", finalParseResult.suggestions.getFillInAtCursor())

src/services/ghost/__tests__/GhostProvider.spec.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect, beforeEach, vi } from "vitest"
 import { MockWorkspace } from "./MockWorkspace"
 import * as vscode from "vscode"
-import { GhostStreamingParser } from "../classic-auto-complete/GhostStreamingParser"
+import { parseGhostResponse } from "../classic-auto-complete/GhostStreamingParser"
 import { GhostSuggestionContext } from "../types"
 
 vi.mock("vscode", () => ({
@@ -65,11 +65,9 @@ vi.mock("vscode", () => ({
 
 describe("GhostProvider", () => {
 	let mockWorkspace: MockWorkspace
-	let streamingParser: GhostStreamingParser
 
 	beforeEach(() => {
 		vi.clearAllMocks()
-		streamingParser = new GhostStreamingParser()
 		mockWorkspace = new MockWorkspace()
 
 		vi.mocked(vscode.workspace.openTextDocument).mockImplementation(async (uri: any) => {
@@ -107,8 +105,7 @@ describe("GhostProvider", () => {
 			const { context } = await setupTestDocument("empty.js", initialContent)
 
 			// Test empty response
-			streamingParser.initialize(context)
-			const result = streamingParser.parseResponse("", "", "")
+			const result = parseGhostResponse("", "", "", context.document, context.range)
 			expect(result.suggestions.hasSuggestions()).toBe(false)
 		})
 
@@ -118,8 +115,7 @@ describe("GhostProvider", () => {
 
 			// Test invalid XML format
 			const invalidXML = "This is not a valid XML format"
-			streamingParser.initialize(context)
-			const result = streamingParser.parseResponse(invalidXML, "", "")
+			const result = parseGhostResponse(invalidXML, "", "", context.document, context.range)
 			expect(result.suggestions.hasSuggestions()).toBe(false)
 		})
 
@@ -137,8 +133,7 @@ describe("GhostProvider", () => {
 			const xmlResponse = `<change><search><![CDATA[console.log('test');]]></search><replace><![CDATA[// Added comment
 console.log('test');]]></replace></change>`
 
-			streamingParser.initialize(context)
-			const result = streamingParser.parseResponse(xmlResponse, "", "")
+			const result = parseGhostResponse(xmlResponse, "", "", context.document, context.range)
 			// Should work with the XML format
 			expect(result.suggestions.hasSuggestions()).toBe(true)
 		})