Fix memory leak in DirectoryScanner

- Remove codeBlocks accumulation that was causing memory exhaustion
- Fix batch processing bugs where file info was added multiple times per file
- Move totalBlockCount increment outside block loop to fix counting bug
- Return empty codeBlocks array since it's not used by main orchestrator logic
- Update tests to expect empty codeBlocks array

This fixes the extension running out of memory during indexing of large codebases.
The memory usage should drop from ~500MB-1GB to ~10-50MB for large projects.

Author: daniel-lxs

src/services/code-index/processors/__tests__/scanner.spec.ts

@@ -168,7 +168,16 @@ describe("DirectoryScanner", () => {
 			expect(mockCodeParser.parseFile).not.toHaveBeenCalled()
 		})
 
-		it("should parse changed files and return code blocks", async () => {
+		it("should parse changed files and return empty codeBlocks array", async () => {
+			// Create scanner without embedder to test the non-embedding path
+			const scannerNoEmbeddings = new DirectoryScanner(
+				null as any, // No embedder
+				null as any, // No vector store
+				mockCodeParser,
+				mockCacheManager,
+				mockIgnoreInstance,
+			)
+
 			const { listFiles } = await import("../../../glob/list-files")
 			vi.mocked(listFiles).mockResolvedValue([["test/file1.js"], false])
 			const mockBlocks: any[] = [
@@ -185,8 +194,8 @@ describe("DirectoryScanner", () => {
 			]
 			;(mockCodeParser.parseFile as any).mockResolvedValue(mockBlocks)
 
-			const result = await scanner.scanDirectory("/test")
-			expect(result.codeBlocks).toEqual(mockBlocks)
+			const result = await scannerNoEmbeddings.scanDirectory("/test")
+			expect(result.codeBlocks).toEqual([]) // Now returns empty array for memory optimization
 			expect(result.stats.processed).toBe(1)
 		})
 
@@ -252,6 +261,15 @@ describe("DirectoryScanner", () => {
 		})
 
 		it("should process markdown files alongside code files", async () => {
+			// Create scanner without embedder to test the non-embedding path
+			const scannerNoEmbeddings = new DirectoryScanner(
+				null as any, // No embedder
+				null as any, // No vector store
+				mockCodeParser,
+				mockCacheManager,
+				mockIgnoreInstance,
+			)
+
 			const { listFiles } = await import("../../../glob/list-files")
 			vi.mocked(listFiles).mockResolvedValue([["test/README.md", "test/app.js", "docs/guide.markdown"], false])
 
@@ -306,23 +324,16 @@ describe("DirectoryScanner", () => {
 				return []
 			})
 
-			const result = await scanner.scanDirectory("/test")
+			const result = await scannerNoEmbeddings.scanDirectory("/test")
 
 			// Verify all files were processed
 			expect(mockCodeParser.parseFile).toHaveBeenCalledTimes(3)
 			expect(mockCodeParser.parseFile).toHaveBeenCalledWith("test/README.md", expect.any(Object))
 			expect(mockCodeParser.parseFile).toHaveBeenCalledWith("test/app.js", expect.any(Object))
 			expect(mockCodeParser.parseFile).toHaveBeenCalledWith("docs/guide.markdown", expect.any(Object))
 
-			// Verify code blocks include both markdown and code content
-			expect(result.codeBlocks).toHaveLength(3)
-			expect(result.codeBlocks).toEqual(
-				expect.arrayContaining([
-					expect.objectContaining({ type: "markdown_header_h1" }),
-					expect.objectContaining({ type: "function" }),
-					expect.objectContaining({ type: "markdown_header_h2" }),
-				]),
-			)
+			// Verify codeBlocks is empty (memory optimization) but processing still works
+			expect(result.codeBlocks).toEqual([])
 
 			expect(result.stats.processed).toBe(3)
 		})

src/services/code-index/processors/scanner.ts

@@ -85,7 +85,6 @@ export class DirectoryScanner implements IDirectoryScanner {
 
 		// Initialize tracking variables
 		const processedFiles = new Set<string>()
-		const codeBlocks: CodeBlock[] = []
 		let processedCount = 0
 		let skippedCount = 0
 
@@ -135,7 +134,6 @@ export class DirectoryScanner implements IDirectoryScanner {
 					const blocks = await this.codeParser.parseFile(filePath, { content, fileHash: currentFileHash })
 					const fileBlockCount = blocks.length
 					onFileParsed?.(fileBlockCount)
-					codeBlocks.push(...blocks)
 					processedCount++
 
 					// Process embeddings if configured
@@ -146,20 +144,11 @@ export class DirectoryScanner implements IDirectoryScanner {
 							const trimmedContent = block.content.trim()
 							if (trimmedContent) {
 								const release = await mutex.acquire()
-								totalBlockCount += fileBlockCount
 								try {
 									currentBatchBlocks.push(block)
 									currentBatchTexts.push(trimmedContent)
 									addedBlocksFromFile = true
 
-									if (addedBlocksFromFile) {
-										currentBatchFileInfos.push({
-											filePath,
-											fileHash: currentFileHash,
-											isNew: !this.cacheManager.getHash(filePath),
-										})
-									}
-
 									// Check if batch threshold is met
 									if (currentBatchBlocks.length >= BATCH_SEGMENT_THRESHOLD) {
 										// Copy current batch data and clear accumulators
@@ -188,6 +177,16 @@ export class DirectoryScanner implements IDirectoryScanner {
 								}
 							}
 						}
+
+						// Add file info once per file (outside the block loop)
+						if (addedBlocksFromFile) {
+							totalBlockCount += fileBlockCount
+							currentBatchFileInfos.push({
+								filePath,
+								fileHash: currentFileHash,
+								isNew: !this.cacheManager.getHash(filePath),
+							})
+						}
 					} else {
 						// Only update hash if not being processed in a batch
 						await this.cacheManager.updateHash(filePath, currentFileHash)
@@ -280,7 +279,7 @@ export class DirectoryScanner implements IDirectoryScanner {
 		}
 
 		return {
-			codeBlocks,
+			codeBlocks: [], // Return empty array to prevent memory accumulation
 			stats: {
 				processed: processedCount,
 				skipped: skippedCount,