fix: Improve @ file search compatibility with large or complext projects

- Increase file listing limit to handle larger projects (5000 -> 500000)
- Implement caching for file lists to prevent performance degradation
- Respect VSCode search configuration (`search.useIgnoreFiles`, `search.useGlobalIgnoreFiles`, `search.useParentIgnoreFiles`) when listing files

fixes #5721

Author: Naituw

src/core/task/__tests__/Task.spec.ts

@@ -116,6 +116,7 @@ vi.mock("vscode", () => {
 				stat: vi.fn().mockResolvedValue({ type: 1 }), // FileType.File = 1
 			},
 			onDidSaveTextDocument: vi.fn(() => mockDisposable),
+			onDidChangeConfiguration: vi.fn(() => mockDisposable),
 			getConfiguration: vi.fn(() => ({ get: (key: string, defaultValue: any) => defaultValue })),
 		},
 		env: {

src/core/webview/webviewMessageHandler.ts

@@ -1400,10 +1400,16 @@ export const webviewMessageHandler = async (
 				break
 			}
 			try {
-				// Call file search service with query from message
+				// Get cached file list from WorkspaceTracker
+				const workspaceFiles = provider.workspaceTracker
+					? await provider.workspaceTracker.getRipgrepFileList()
+					: []
+
+				// Call file search service with query and cached files from message
 				const results = await searchWorkspaceFiles(
 					message.query || "",
 					workspacePath,
+					workspaceFiles,
 					20, // Use default limit, as filtering is now done in the backend
 				)
 

src/integrations/workspace/WorkspaceTracker.ts

@@ -4,6 +4,8 @@ import * as path from "path"
 import { listFiles } from "../../services/glob/list-files"
 import { ClineProvider } from "../../core/webview/ClineProvider"
 import { toRelativePath, getWorkspacePath } from "../../utils/path"
+import { executeRipgrepForFiles, FileResult } from "../../services/search/file-search"
+import { isPathInIgnoredDirectory } from "../../services/glob/ignore-utils"
 
 const MAX_INITIAL_FILES = 1_000
 
@@ -16,14 +18,105 @@ class WorkspaceTracker {
 	private prevWorkSpacePath: string | undefined
 	private resetTimer: NodeJS.Timeout | null = null
 
+	// Ripgrep cache related properties
+	private ripgrepFileCache: FileResult[] | null = null
+	private ripgrepCacheWorkspacePath: string | undefined = undefined
+	private ripgrepOperationPromise: Promise<FileResult[]> | null = null
+
 	get cwd() {
 		return getWorkspacePath()
 	}
+
 	constructor(provider: ClineProvider) {
 		this.providerRef = new WeakRef(provider)
 		this.registerListeners()
 	}
 
+	/**
+	 * Get ripgrep extra options based on VSCode search configuration
+	 */
+	private getRipgrepExtraOptions(): string[] {
+		const config = vscode.workspace.getConfiguration("search")
+		const extraOptions: string[] = []
+
+		const useIgnoreFiles = config.get<boolean>("useIgnoreFiles", true)
+
+		if (!useIgnoreFiles) {
+			extraOptions.push("--no-ignore")
+		} else {
+			const useGlobalIgnoreFiles = config.get<boolean>("useGlobalIgnoreFiles", true)
+			const useParentIgnoreFiles = config.get<boolean>("useParentIgnoreFiles", true)
+
+			if (!useGlobalIgnoreFiles) {
+				extraOptions.push("--no-ignore-global")
+			}
+
+			if (!useParentIgnoreFiles) {
+				extraOptions.push("--no-ignore-parent")
+			}
+		}
+
+		return extraOptions
+	}
+
+	/**
+	 * Get comprehensive file list using ripgrep with caching
+	 * This provides a more complete file list than the limited filePaths set
+	 */
+	async getRipgrepFileList(): Promise<FileResult[]> {
+		const currentWorkspacePath = this.cwd
+
+		if (!currentWorkspacePath) {
+			return []
+		}
+
+		// Return cached results if available and workspace hasn't changed
+		if (this.ripgrepFileCache && this.ripgrepCacheWorkspacePath === currentWorkspacePath) {
+			return this.ripgrepFileCache
+		}
+
+		// If there's an ongoing operation, wait for it
+		if (this.ripgrepOperationPromise) {
+			try {
+				return await this.ripgrepOperationPromise
+			} catch (error) {
+				// If the ongoing operation failed, we'll start a new one below
+				this.ripgrepOperationPromise = null
+			}
+		}
+
+		try {
+			// Start new operation and store the promise
+			this.ripgrepOperationPromise = executeRipgrepForFiles(
+				currentWorkspacePath,
+				500000,
+				this.getRipgrepExtraOptions(),
+			)
+			const fileResults = await this.ripgrepOperationPromise
+
+			// Cache the results and clear the operation promise
+			this.ripgrepFileCache = fileResults
+			this.ripgrepCacheWorkspacePath = currentWorkspacePath
+			this.ripgrepOperationPromise = null
+
+			return fileResults
+		} catch (error) {
+			console.error("Error getting ripgrep file list:", error)
+			this.ripgrepOperationPromise = null
+			return []
+		}
+	}
+
+	/**
+	 * Clear the ripgrep file cache
+	 * Called when workspace changes or files are modified
+	 */
+	private clearRipgrepCache() {
+		this.ripgrepFileCache = null
+		this.ripgrepCacheWorkspacePath = undefined
+		this.ripgrepOperationPromise = null
+	}
+
 	async initializeFilePaths() {
 		// should not auto get filepaths for desktop since it would immediately show permission popup before cline ever creates a file
 		if (!this.cwd) {
@@ -36,13 +129,19 @@ class WorkspaceTracker {
 		}
 		files.slice(0, MAX_INITIAL_FILES).forEach((file) => this.filePaths.add(this.normalizeFilePath(file)))
 		this.workspaceDidUpdate()
+
+		// preheat filelist
+		this.getRipgrepFileList()
 	}
 
 	private registerListeners() {
 		const watcher = vscode.workspace.createFileSystemWatcher("**")
 		this.prevWorkSpacePath = this.cwd
 		this.disposables.push(
 			watcher.onDidCreate(async (uri) => {
+				if (!isPathInIgnoredDirectory(uri.fsPath)) {
+					this.clearRipgrepCache()
+				}
 				await this.addFilePath(uri.fsPath)
 				this.workspaceDidUpdate()
 			}),
@@ -51,6 +150,9 @@ class WorkspaceTracker {
 		// Renaming files triggers a delete and create event
 		this.disposables.push(
 			watcher.onDidDelete(async (uri) => {
+				if (!isPathInIgnoredDirectory(uri.fsPath)) {
+					this.clearRipgrepCache()
+				}
 				if (await this.removeFilePath(uri.fsPath)) {
 					this.workspaceDidUpdate()
 				}
@@ -71,6 +173,20 @@ class WorkspaceTracker {
 				}
 			}),
 		)
+
+		// Listen for VSCode configuration changes
+		this.disposables.push(
+			vscode.workspace.onDidChangeConfiguration((event: vscode.ConfigurationChangeEvent) => {
+				// Clear cache when search-related configuration changes
+				if (
+					event.affectsConfiguration("search.useIgnoreFiles") ||
+					event.affectsConfiguration("search.useGlobalIgnoreFiles") ||
+					event.affectsConfiguration("search.useParentIgnoreFiles")
+				) {
+					this.clearRipgrepCache()
+				}
+			}),
+		)
 	}
 
 	private getOpenedTabsInfo() {
@@ -97,6 +213,8 @@ class WorkspaceTracker {
 		}
 		this.resetTimer = setTimeout(async () => {
 			if (this.prevWorkSpacePath !== this.cwd) {
+				// Clear cache when workspace changes
+				this.clearRipgrepCache()
 				await this.providerRef.deref()?.postMessageToWebview({
 					type: "workspaceUpdated",
 					filePaths: [],