Update tree-sitter

Author: cte

pnpm-lock.yaml

@@ -371,11 +371,11 @@
 		"@google/genai": "^1.0.0",
 		"@mistralai/mistralai": "^1.3.6",
 		"@modelcontextprotocol/sdk": "^1.9.0",
+		"@qdrant/js-client-rest": "^1.14.0",
 		"@roo-code/cloud": "workspace:^",
 		"@roo-code/ipc": "workspace:^",
 		"@roo-code/telemetry": "workspace:^",
 		"@roo-code/types": "workspace:^",
-		"@qdrant/js-client-rest": "^1.14.0",
 		"@types/lodash.debounce": "^4.0.9",
 		"@vscode/codicons": "^0.0.36",
 		"async-mutex": "^0.5.0",
@@ -422,11 +422,11 @@
 		"strip-bom": "^5.0.0",
 		"tiktoken": "^1.0.21",
 		"tmp": "^0.2.3",
-		"tree-sitter-wasms": "^0.1.11",
+		"tree-sitter-wasms": "^0.1.12",
 		"turndown": "^7.2.0",
 		"uuid": "^11.1.0",
 		"vscode-material-icons": "^0.1.1",
-		"web-tree-sitter": "^0.22.6",
+		"web-tree-sitter": "^0.25.6",
 		"workerpool": "^9.2.0",
 		"yaml": "^2.8.0",
 		"zod": "^3.25.61"

src/package.json

@@ -1,9 +1,9 @@
 // npx vitest services/code-index/processors/__tests__/parser.spec.ts
 
 import { CodeParser, codeParser } from "../parser"
-import Parser from "web-tree-sitter"
 import { loadRequiredLanguageParsers } from "../../../tree-sitter/languageParser"
 import { readFile } from "fs/promises"
+import { Node } from "web-tree-sitter"
 
 // Override Jest-based fs/promises mock with vitest-compatible version
 vi.mock("fs/promises", () => ({
@@ -202,7 +202,7 @@ describe("CodeParser", () => {
 				startPosition: { row: 10 },
 				endPosition: { row: 12 },
 				type: "function",
-			} as unknown as Parser.SyntaxNode
+			} as unknown as Node
 
 			const result = await parser["_chunkLeafNodeByLines"](mockNode, "test.js", "hash", new Set())
 			expect(result.length).toBeGreaterThan(0)

src/services/code-index/processors/__tests__/parser.spec.ts

@@ -1,7 +1,7 @@
 import { readFile } from "fs/promises"
 import { createHash } from "crypto"
 import * as path from "path"
-import * as treeSitter from "web-tree-sitter"
+import { Node } from "web-tree-sitter"
 import { LanguageParser, loadRequiredLanguageParsers } from "../../tree-sitter/languageParser"
 import { ICodeParser, CodeBlock } from "../interfaces"
 import { scannerExtensions } from "../shared/supported-extensions"
@@ -124,7 +124,8 @@ export class CodeParser implements ICodeParser {
 
 		// We don't need to get the query string from languageQueries since it's already loaded
 		// in the language object
-		const captures = language.query.captures(tree.rootNode)
+		const captures = tree ? language.query.captures(tree.rootNode) : []
+
 		// Check if captures are empty
 		if (captures.length === 0) {
 			if (content.length >= MIN_BLOCK_CHARS) {
@@ -140,7 +141,7 @@ export class CodeParser implements ICodeParser {
 		const results: CodeBlock[] = []
 
 		// Process captures if not empty
-		const queue: treeSitter.SyntaxNode[] = captures.map((capture: any) => capture.node)
+		const queue: Node[] = Array.from(captures).map((capture) => capture.node)
 
 		while (queue.length > 0) {
 			const currentNode = queue.shift()!
@@ -150,9 +151,9 @@ export class CodeParser implements ICodeParser {
 			if (currentNode.text.length >= MIN_BLOCK_CHARS) {
 				// If it also exceeds the maximum character limit, try to break it down
 				if (currentNode.text.length > MAX_BLOCK_CHARS * MAX_CHARS_TOLERANCE_FACTOR) {
-					if (currentNode.children.length > 0) {
+					if (currentNode.children.filter((child) => child !== null).length > 0) {
 						// If it has children, process them instead
-						queue.push(...currentNode.children)
+						queue.push(...currentNode.children.filter((child) => child !== null))
 					} else {
 						// If it's a leaf node, chunk it (passing MIN_BLOCK_CHARS as per Task 1 Step 5)
 						// Note: _chunkLeafNodeByLines logic might need further adjustment later
@@ -168,7 +169,7 @@ export class CodeParser implements ICodeParser {
 					// Node meets min chars and is within max chars, create a block
 					const identifier =
 						currentNode.childForFieldName("name")?.text ||
-						currentNode.children.find((c) => c.type === "identifier")?.text ||
+						currentNode.children.find((c) => c?.type === "identifier")?.text ||
 						null
 					const type = currentNode.type
 					const start_line = currentNode.startPosition.row + 1
@@ -353,7 +354,7 @@ export class CodeParser implements ICodeParser {
 	}
 
 	private _chunkLeafNodeByLines(
-		node: treeSitter.SyntaxNode,
+		node: Node,
 		filePath: string,
 		fileHash: string,
 		seenSegmentHashes: Set<string>,

src/services/code-index/processors/parser.ts

@@ -1,9 +1,9 @@
 import { parseSourceCodeDefinitionsForFile, setMinComponentLines } from ".."
 import * as fs from "fs/promises"
 import * as path from "path"
-import Parser from "web-tree-sitter"
 import tsxQuery from "../queries/tsx"
-// Mock setup
+import { Parser, Language } from "web-tree-sitter"
+
 vi.mock("fs/promises")
 export const mockedFs = vi.mocked(fs)
 
@@ -26,39 +26,28 @@ export const debugLog = (message: string, ...args: any[]) => {
 }
 
 // Store the initialized TreeSitter for reuse
-let initializedTreeSitter: Parser | null = null
+let initializedTreeSitter: { Parser: typeof Parser; Language: typeof Language } | null = null
 
 // Function to initialize tree-sitter
 export async function initializeTreeSitter() {
-	if (initializedTreeSitter) {
-		return initializedTreeSitter
-	}
+	if (!initializedTreeSitter) {
+		// Initialize directly using the default export or the module itself
+		await Parser.init()
 
-	const TreeSitter = await initializeWorkingParser()
+		// Override the Parser.Language.load to use dist directory
+		const originalLoad = Language.load
 
-	initializedTreeSitter = TreeSitter
-	return TreeSitter
-}
+		Language.load = async (wasmPath: string) => {
+			const filename = path.basename(wasmPath)
+			const correctPath = path.join(process.cwd(), "dist", filename)
+			// console.log(`Redirecting WASM load from ${wasmPath} to ${correctPath}`)
+			return originalLoad(correctPath)
+		}
 
-// Function to initialize a working parser with correct WASM path
-// DO NOT CHANGE THIS FUNCTION
-export async function initializeWorkingParser() {
-	const TreeSitter = (await vi.importActual("web-tree-sitter")) as any
-
-	// Initialize directly using the default export or the module itself
-	const ParserConstructor = TreeSitter.default || TreeSitter
-	await ParserConstructor.init()
-
-	// Override the Parser.Language.load to use dist directory
-	const originalLoad = ParserConstructor.Language.load
-	ParserConstructor.Language.load = async (wasmPath: string) => {
-		const filename = path.basename(wasmPath)
-		const correctPath = path.join(process.cwd(), "dist", filename)
-		// console.log(`Redirecting WASM load from ${wasmPath} to ${correctPath}`)
-		return originalLoad(correctPath)
+		initializedTreeSitter = { Parser, Language }
 	}
 
-	return ParserConstructor
+	return initializedTreeSitter
 }
 
 // Test helper for parsing source code definitions
@@ -91,12 +80,12 @@ export async function testParseSourceCodeDefinitions(
 	const mockedLoadRequiredLanguageParsers = loadRequiredLanguageParsers as any
 
 	// Initialize TreeSitter and create a real parser
-	const TreeSitter = await initializeTreeSitter()
-	const parser = new TreeSitter()
+	const { Parser, Language } = await initializeTreeSitter()
+	const parser = new Parser()
 
 	// Load language and configure parser
 	const wasmPath = path.join(process.cwd(), `dist/${wasmFile}`)
-	const lang = await TreeSitter.Language.load(wasmPath)
+	const lang = await Language.load(wasmPath)
 	parser.setLanguage(lang)
 
 	// Create a real query
@@ -122,16 +111,16 @@ export async function testParseSourceCodeDefinitions(
 
 // Helper function to inspect tree structure
 export async function inspectTreeStructure(content: string, language: string = "typescript"): Promise<string> {
-	const TreeSitter = await initializeTreeSitter()
-	const parser = new TreeSitter()
+	const { Parser, Language } = await initializeTreeSitter()
+	const parser = new Parser()
 	const wasmPath = path.join(process.cwd(), `dist/tree-sitter-${language}.wasm`)
-	const lang = await TreeSitter.Language.load(wasmPath)
+	const lang = await Language.load(wasmPath)
 	parser.setLanguage(lang)
 
 	// Parse the content
 	const tree = parser.parse(content)
 
 	// Print the tree structure
-	debugLog(`TREE STRUCTURE (${language}):\n${tree.rootNode.toString()}`)
-	return tree.rootNode.toString()
+	debugLog(`TREE STRUCTURE (${language}):\n${tree?.rootNode.toString()}`)
+	return tree?.rootNode.toString() || ""
 }

src/services/tree-sitter/__tests__/helpers.ts

@@ -1,8 +1,11 @@
+// npx vitest services/tree-sitter/__tests__/inspectSwift.spec.ts
+
 import { inspectTreeStructure, testParseSourceCodeDefinitions, debugLog } from "./helpers"
 import { swiftQuery } from "../queries"
 import sampleSwiftContent from "./fixtures/sample-swift"
 
-describe("inspectSwift", () => {
+// This is insanely slow for some reason.
+describe.skip("inspectSwift", () => {
 	const testOptions = {
 		language: "swift",
 		wasmFile: "tree-sitter-swift.wasm",