feat: implement unified .gitignore/.rooignore handling for code indexing

- Enhanced RooIgnoreController to support .gitignore fallback when .rooignore is missing/empty
- Added priority system: .rooignore → .gitignore → default patterns
- Removed dual filtering that caused conflicts between ignore systems
- Updated DirectoryScanner and FileWatcher to use unified ignore controller
- Fixed service factory to remove redundant ignore instance parameter
- Updated tests to reflect new constructor signatures

Fixes #5655

Author: roomote

src/core/ignore/RooIgnoreController.ts

@@ -16,85 +16,161 @@ export class RooIgnoreController {
 	private ignoreInstance: Ignore
 	private disposables: vscode.Disposable[] = []
 	rooIgnoreContent: string | undefined
+	private gitIgnoreContent: string | undefined
+	private currentIgnoreSource: "rooignore" | "gitignore" | "default" | "none" = "none"
 
 	constructor(cwd: string) {
 		this.cwd = cwd
 		this.ignoreInstance = ignore()
 		this.rooIgnoreContent = undefined
-		// Set up file watcher for .rooignore
-		this.setupFileWatcher()
+		// Set up file watchers for both .rooignore and .gitignore
+		this.setupFileWatchers()
 	}
 
 	/**
 	 * Initialize the controller by loading custom patterns
 	 * Must be called after construction and before using the controller
 	 */
 	async initialize(): Promise<void> {
-		await this.loadRooIgnore()
+		await this.loadIgnorePatterns()
 	}
 
 	/**
-	 * Set up the file watcher for .rooignore changes
+	 * Set up file watchers for both .rooignore and .gitignore changes
 	 */
-	private setupFileWatcher(): void {
+	private setupFileWatchers(): void {
+		// Watch .rooignore
 		const rooignorePattern = new vscode.RelativePattern(this.cwd, ".rooignore")
-		const fileWatcher = vscode.workspace.createFileSystemWatcher(rooignorePattern)
+		const rooignoreWatcher = vscode.workspace.createFileSystemWatcher(rooignorePattern)
 
-		// Watch for changes and updates
+		// Watch .gitignore
+		const gitignorePattern = new vscode.RelativePattern(this.cwd, ".gitignore")
+		const gitignoreWatcher = vscode.workspace.createFileSystemWatcher(gitignorePattern)
+
+		// Watch for changes and updates to both files
 		this.disposables.push(
-			fileWatcher.onDidChange(() => {
-				this.loadRooIgnore()
+			rooignoreWatcher.onDidChange(() => {
+				this.loadIgnorePatterns()
+			}),
+			rooignoreWatcher.onDidCreate(() => {
+				this.loadIgnorePatterns()
+			}),
+			rooignoreWatcher.onDidDelete(() => {
+				this.loadIgnorePatterns()
 			}),
-			fileWatcher.onDidCreate(() => {
-				this.loadRooIgnore()
+			gitignoreWatcher.onDidChange(() => {
+				this.loadIgnorePatterns()
 			}),
-			fileWatcher.onDidDelete(() => {
-				this.loadRooIgnore()
+			gitignoreWatcher.onDidCreate(() => {
+				this.loadIgnorePatterns()
+			}),
+			gitignoreWatcher.onDidDelete(() => {
+				this.loadIgnorePatterns()
 			}),
 		)
 
-		// Add fileWatcher itself to disposables
-		this.disposables.push(fileWatcher)
+		// Add fileWatchers themselves to disposables
+		this.disposables.push(rooignoreWatcher, gitignoreWatcher)
 	}
 
 	/**
-	 * Load custom patterns from .rooignore if it exists
+	 * Load ignore patterns with .gitignore fallback support
 	 */
-	private async loadRooIgnore(): Promise<void> {
+	private async loadIgnorePatterns(): Promise<void> {
 		try {
 			// Reset ignore instance to prevent duplicate patterns
 			this.ignoreInstance = ignore()
-			const ignorePath = path.join(this.cwd, ".rooignore")
-			if (await fileExistsAtPath(ignorePath)) {
-				const content = await fs.readFile(ignorePath, "utf8")
-				this.rooIgnoreContent = content
-				this.ignoreInstance.add(content)
-				this.ignoreInstance.add(".rooignore")
+
+			// Try to load .rooignore first
+			const rooIgnorePath = path.join(this.cwd, ".rooignore")
+			const rooIgnoreExists = await fileExistsAtPath(rooIgnorePath)
+
+			if (rooIgnoreExists) {
+				const content = await fs.readFile(rooIgnorePath, "utf8")
+				const trimmedContent = content.trim()
+
+				if (trimmedContent) {
+					// .rooignore exists and has content - use it exclusively
+					this.rooIgnoreContent = content
+					this.gitIgnoreContent = undefined
+					this.currentIgnoreSource = "rooignore"
+					this.ignoreInstance.add(content)
+					this.ignoreInstance.add(".rooignore")
+					return
+				} else {
+					// .rooignore exists but is empty - fall back to .gitignore
+					this.rooIgnoreContent = content
+				}
 			} else {
 				this.rooIgnoreContent = undefined
 			}
+
+			// Try to load .gitignore as fallback
+			const gitIgnorePath = path.join(this.cwd, ".gitignore")
+			const gitIgnoreExists = await fileExistsAtPath(gitIgnorePath)
+
+			if (gitIgnoreExists) {
+				const content = await fs.readFile(gitIgnorePath, "utf8")
+				this.gitIgnoreContent = content
+				this.currentIgnoreSource = "gitignore"
+				this.ignoreInstance.add(content)
+				this.ignoreInstance.add(".gitignore")
+			} else {
+				// Neither file exists - use default patterns for common directories
+				this.gitIgnoreContent = undefined
+				this.currentIgnoreSource = "default"
+				this.addDefaultIgnorePatterns()
+			}
 		} catch (error) {
-			// Should never happen: reading file failed even though it exists
-			console.error("Unexpected error loading .rooignore:", error)
+			console.error("Unexpected error loading ignore patterns:", error)
+			// Fallback to default patterns on error
+			this.currentIgnoreSource = "default"
+			this.addDefaultIgnorePatterns()
 		}
 	}
 
+	/**
+	 * Add default ignore patterns for common directories that should typically be excluded
+	 */
+	private addDefaultIgnorePatterns(): void {
+		const defaultPatterns = [
+			"node_modules/",
+			"vendor/",
+			".git/",
+			".svn/",
+			".hg/",
+			"dist/",
+			"build/",
+			"out/",
+			"target/",
+			"*.log",
+			".DS_Store",
+			"Thumbs.db",
+		]
+
+		this.ignoreInstance.add(defaultPatterns)
+	}
+
+	/**
+	 * Load custom patterns from .rooignore if it exists
+	 * @deprecated Use loadIgnorePatterns() instead
+	 */
+	private async loadRooIgnore(): Promise<void> {
+		await this.loadIgnorePatterns()
+	}
+
 	/**
 	 * Check if a file should be accessible to the LLM
 	 * @param filePath - Path to check (relative to cwd)
 	 * @returns true if file is accessible, false if ignored
 	 */
 	validateAccess(filePath: string): boolean {
-		// Always allow access if .rooignore does not exist
-		if (!this.rooIgnoreContent) {
-			return true
-		}
 		try {
 			// Normalize path to be relative to cwd and use forward slashes
 			const absolutePath = path.resolve(this.cwd, filePath)
 			const relativePath = path.relative(this.cwd, absolutePath).toPosix()
 
-			// Ignore expects paths to be path.relative()'d
+			// Use the unified ignore instance which now handles .rooignore, .gitignore, or default patterns
 			return !this.ignoreInstance.ignores(relativePath)
 		} catch (error) {
 			// console.error(`Error validating access for ${filePath}:`, error)
@@ -109,10 +185,7 @@ export class RooIgnoreController {
 	 * @returns path of file that is being accessed if it is being accessed, undefined if command is allowed
 	 */
 	validateCommand(command: string): string | undefined {
-		// Always allow if no .rooignore exists
-		if (!this.rooIgnoreContent) {
-			return undefined
-		}
+		// Use unified ignore patterns (rooignore, gitignore, or defaults)
 
 		// Split command into parts and get the base command
 		const parts = command.trim().split(/\s+/)
@@ -188,14 +261,30 @@ export class RooIgnoreController {
 	}
 
 	/**
-	 * Get formatted instructions about the .rooignore file for the LLM
-	 * @returns Formatted instructions or undefined if .rooignore doesn't exist
+	 * Get the current ignore source being used
+	 * @returns The source of ignore patterns currently in use
+	 */
+	getIgnoreSource(): "rooignore" | "gitignore" | "default" | "none" {
+		return this.currentIgnoreSource
+	}
+
+	/**
+	 * Get formatted instructions about the ignore patterns for the LLM
+	 * @returns Formatted instructions or undefined if no patterns are active
 	 */
 	getInstructions(): string | undefined {
-		if (!this.rooIgnoreContent) {
-			return undefined
-		}
+		switch (this.currentIgnoreSource) {
+			case "rooignore":
+				return `# .rooignore\n\n(The following is provided by a root-level .rooignore file where the user has specified files and directories that should not be accessed. When using list_files, you'll notice a ${LOCK_TEXT_SYMBOL} next to files that are blocked. Attempting to access the file's contents e.g. through read_file will result in an error.)\n\n${this.rooIgnoreContent}\n.rooignore`
+
+			case "gitignore":
+				return `# .gitignore (fallback)\n\n(The following is provided by a root-level .gitignore file being used as fallback since no .rooignore file was found or it was empty. When using list_files, you'll notice a ${LOCK_TEXT_SYMBOL} next to files that are blocked. Attempting to access the file's contents e.g. through read_file will result in an error.)\n\n${this.gitIgnoreContent}\n.gitignore`
 
-		return `# .rooignore\n\n(The following is provided by a root-level .rooignore file where the user has specified files and directories that should not be accessed. When using list_files, you'll notice a ${LOCK_TEXT_SYMBOL} next to files that are blocked. Attempting to access the file's contents e.g. through read_file will result in an error.)\n\n${this.rooIgnoreContent}\n.rooignore`
+			case "default":
+				return `# Default ignore patterns\n\n(The following default ignore patterns are being used since neither .rooignore nor .gitignore files were found. These patterns exclude common directories that are typically not needed for code analysis. When using list_files, you'll notice a ${LOCK_TEXT_SYMBOL} next to files that are blocked.)\n\nnode_modules/\nvendor/\n.git/\n.svn/\n.hg/\ndist/\nbuild/\nout/\ntarget/\n*.log\n.DS_Store\nThumbs.db`
+
+			default:
+				return undefined
+		}
 	}
 }

src/services/code-index/manager.ts

@@ -236,34 +236,17 @@ export class CodeIndexManager {
 			this._cacheManager!,
 		)
 
-		const ignoreInstance = ignore()
 		const workspacePath = getWorkspacePath()
 
 		if (!workspacePath) {
 			this._stateManager.setSystemState("Standby", "")
 			return
 		}
 
-		const ignorePath = path.join(workspacePath, ".gitignore")
-		try {
-			const content = await fs.readFile(ignorePath, "utf8")
-			ignoreInstance.add(content)
-			ignoreInstance.add(".gitignore")
-		} catch (error) {
-			// Should never happen: reading file failed even though it exists
-			console.error("Unexpected error loading .gitignore:", error)
-			TelemetryService.instance.captureEvent(TelemetryEventName.CODE_INDEX_ERROR, {
-				error: error instanceof Error ? error.message : String(error),
-				stack: error instanceof Error ? error.stack : undefined,
-				location: "_recreateServices",
-			})
-		}
-
-		// (Re)Create shared service instances
+		// (Re)Create shared service instances - ignore patterns are now handled by RooIgnoreController
 		const { embedder, vectorStore, scanner, fileWatcher } = this._serviceFactory.createServices(
 			this.context,
 			this._cacheManager!,
-			ignoreInstance,
 		)
 
 		// Validate embedder configuration before proceeding

src/services/code-index/processors/__tests__/scanner.spec.ts

@@ -104,13 +104,7 @@ describe("DirectoryScanner", () => {
 			ignores: vi.fn().mockReturnValue(false),
 		}
 
-		scanner = new DirectoryScanner(
-			mockEmbedder,
-			mockVectorStore,
-			mockCodeParser,
-			mockCacheManager,
-			mockIgnoreInstance,
-		)
+		scanner = new DirectoryScanner(mockEmbedder, mockVectorStore, mockCodeParser, mockCacheManager)
 
 		// Mock default implementations - create proper Stats object
 		mockStats = {

src/services/code-index/processors/file-watcher.ts

@@ -31,7 +31,6 @@ import { sanitizeErrorMessage } from "../shared/validation-helpers"
  * Implementation of the file watcher interface
  */
 export class FileWatcher implements IFileWatcher {
-	private ignoreInstance?: Ignore
 	private fileWatcher?: vscode.FileSystemWatcher
 	private ignoreController: RooIgnoreController
 	private accumulatedEvents: Map<string, { uri: vscode.Uri; type: "create" | "change" | "delete" }> = new Map()
@@ -76,19 +75,18 @@ export class FileWatcher implements IFileWatcher {
 		private readonly cacheManager: CacheManager,
 		private embedder?: IEmbedder,
 		private vectorStore?: IVectorStore,
-		ignoreInstance?: Ignore,
 		ignoreController?: RooIgnoreController,
 	) {
 		this.ignoreController = ignoreController || new RooIgnoreController(workspacePath)
-		if (ignoreInstance) {
-			this.ignoreInstance = ignoreInstance
-		}
 	}
 
 	/**
 	 * Initializes the file watcher
 	 */
 	async initialize(): Promise<void> {
+		// Initialize the ignore controller
+		await this.ignoreController.initialize()
+
 		// Create file watcher
 		const filePattern = new vscode.RelativePattern(
 			this.workspacePath,
@@ -495,12 +493,8 @@ export class FileWatcher implements IFileWatcher {
 				}
 			}
 
-			// Check if file should be ignored
-			const relativeFilePath = generateRelativeFilePath(filePath, this.workspacePath)
-			if (
-				!this.ignoreController.validateAccess(filePath) ||
-				(this.ignoreInstance && this.ignoreInstance.ignores(relativeFilePath))
-			) {
+			// Check if file should be ignored using unified ignore controller
+			if (!this.ignoreController.validateAccess(filePath)) {
 				return {
 					path: filePath,
 					status: "skipped" as const,

src/services/code-index/processors/scanner.ts

@@ -35,7 +35,6 @@ export class DirectoryScanner implements IDirectoryScanner {
 		private readonly qdrantClient: IVectorStore,
 		private readonly codeParser: ICodeParser,
 		private readonly cacheManager: CacheManager,
-		private readonly ignoreInstance: Ignore,
 	) {}
 
 	/**
@@ -70,17 +69,16 @@ export class DirectoryScanner implements IDirectoryScanner {
 		// Filter paths using .rooignore
 		const allowedPaths = ignoreController.filterPaths(filePaths)
 
-		// Filter by supported extensions, ignore patterns, and excluded directories
+		// Filter by supported extensions and excluded directories
 		const supportedPaths = allowedPaths.filter((filePath) => {
 			const ext = path.extname(filePath).toLowerCase()
-			const relativeFilePath = generateRelativeFilePath(filePath, scanWorkspace)
 
 			// Check if file is in an ignored directory using the shared helper
 			if (isPathInIgnoredDirectory(filePath)) {
 				return false
 			}
 
-			return scannerExtensions.includes(ext) && !this.ignoreInstance.ignores(relativeFilePath)
+			return scannerExtensions.includes(ext)
 		})
 
 		// Initialize tracking variables