File search service

Author: Roo Code

package-lock.json

@@ -276,7 +276,7 @@
 		"package": "npm run build:webview && npm run check-types && npm run lint && node esbuild.js --production",
 		"pretest": "npm run compile && npm run compile:integration",
 		"dev": "cd webview-ui && npm run dev",
-		"test": "jest && npm run test:webview",
+		"test": "jest && vitest && npm run test:webview",
 		"test:webview": "cd webview-ui && npm run test",
 		"test:integration": "npm run build && npm run compile:integration && npx dotenvx run -f .env.integration -- vscode-test",
 		"prepare": "husky",
@@ -314,7 +314,8 @@
 		"diff-match-patch": "^1.0.5",
 		"fast-deep-equal": "^3.1.3",
 		"fastest-levenshtein": "^1.0.16",
-		"globby": "^14.0.2",
+		"fuzzysort": "^3.1.0",
+		"globby": "^14.1.0",
 		"isbinaryfile": "^5.0.2",
 		"mammoth": "^1.8.0",
 		"monaco-vscode-textmate-theme-converter": "^0.1.7",
@@ -351,17 +352,18 @@
 		"@vscode/test-cli": "^0.0.9",
 		"@vscode/test-electron": "^2.4.0",
 		"esbuild": "^0.24.0",
-		"mkdirp": "^3.0.1",
-		"rimraf": "^6.0.1",
 		"eslint": "^8.57.0",
 		"husky": "^9.1.7",
 		"jest": "^29.7.0",
 		"jest-simple-dot-reporter": "^1.0.5",
 		"lint-staged": "^15.2.11",
+		"mkdirp": "^3.0.1",
 		"npm-run-all": "^4.1.5",
 		"prettier": "^3.4.2",
+		"rimraf": "^6.0.1",
 		"ts-jest": "^29.2.5",
-		"typescript": "^5.4.5"
+		"typescript": "^5.4.5",
+		"vitest": "^3.0.5"
 	},
 	"lint-staged": {
 		"*.{js,jsx,ts,tsx,json,css,md}": [

package.json

@@ -0,0 +1,51 @@
+import { globbyStream, Options } from "globby"
+import fuzzysort from "fuzzysort"
+
+import { IGNORE } from "../glob/ignore"
+
+export class FileSearchService {
+	private files: string[] = []
+
+	constructor(private readonly dirPath: string) {}
+
+	async indexFiles(globbyOptions: Options = {}) {
+		const options: Options = {
+			cwd: this.dirPath,
+			// Do not ignore hidden files/directories.
+			dot: true,
+			absolute: true,
+			// Append a / on any directories matched (/ is used on windows as well,
+			// so dont use path.sep).
+			markDirectories: true,
+			// Globby ignores any files that are in .gitignore.
+			gitignore: true,
+			// Just in case there is no gitignore, we ignore sensible defaults.
+			ignore: IGNORE,
+			// List directories on their own too.
+			onlyFiles: false,
+			...globbyOptions,
+		}
+
+		const stream = globbyStream("**", options)
+
+		for await (const file of stream) {
+			this.files.push(file.toString())
+		}
+	}
+
+	public addFiles(files: string[]) {
+		this.files.push(...files)
+	}
+
+	public removeFiles(files: string[]) {
+		this.files = this.files.filter((file) => !files.includes(file))
+	}
+
+	public get count() {
+		return this.files.length
+	}
+
+	public search(query: string, { limit = 25, threshold = 0.5 }: { limit?: number; threshold?: number } = {}) {
+		return fuzzysort.go(query, this.files, { limit, threshold })
+	}
+}

src/services/file-search/FileSearchService.ts

@@ -0,0 +1,30 @@
+// npx vitest src/services/file-search/__tests__/FileSearchService.spec.ts
+
+import path from "path"
+import { describe, expect, test, beforeAll } from "vitest"
+
+import { FileSearchService } from "../FileSearchService"
+
+describe("listFiles", () => {
+	const service = new FileSearchService(path.join(process.cwd(), "node_modules"))
+
+	beforeAll(async () => {
+		await service.indexFiles({ gitignore: false, ignore: undefined })
+	})
+
+	test("it indexes files", async () => {
+		expect(service.count).toBeGreaterThan(25_000)
+	})
+
+	test("it searches for files", async () => {
+		const results = await service.search("zod")
+		expect(results.length).toBe(25)
+		expect(results.every((result) => result.target.includes("zod"))).toBe(true)
+		expect(results.every((result) => result.score > 0.5)).toBe(true)
+
+		const results2 = await service.search("zod/index.d.ts")
+		expect(results2.length).toBe(2)
+		expect(results2.some((result) => result.target.includes("zod/index.d.ts"))).toBe(true)
+		expect(results2.some((result) => result.target.includes("zod/lib/index.d.ts"))).toBe(true)
+	})
+})

src/services/file-search/__tests__/FileSearchService.spec.ts

@@ -0,0 +1,14 @@
+// npx vitest src/services/glob/__tests__/list-files.spec.ts
+
+import path from "path"
+import { describe, expect, test } from "vitest"
+
+import { listFiles } from "../list-files"
+
+describe("listFiles", () => {
+	test("it lists files in a directory", async () => {
+		const dirPath = path.join(process.cwd(), "node_modules")
+		const [files, _] = await listFiles(dirPath, true, 100_000, { gitignore: false, ignore: undefined })
+		expect(files.length).toBeGreaterThan(25_000)
+	})
+})

src/services/glob/__tests__/list-files.spec.ts

@@ -0,0 +1,21 @@
+export const IGNORE = [
+	"node_modules",
+	"__pycache__",
+	"env",
+	"venv",
+	"target/dependency",
+	"build/dependencies",
+	"dist",
+	"out",
+	"bundle",
+	"vendor",
+	"tmp",
+	"temp",
+	"deps",
+	"pkg",
+	"Pods",
+	// '!**/.*' excludes hidden directories, while '!**/.*/**' excludes only
+	// their contents. This way we are at least aware of the existence of
+	// hidden directories.
+	".*",
+].map((dir) => `**/${dir}/**`)

src/services/glob/ignore.ts

@@ -2,8 +2,14 @@ import { globby, Options } from "globby"
 import os from "os"
 import * as path from "path"
 import { arePathsEqual } from "../../utils/path"
+import { IGNORE as dirsToIgnore } from "./ignore"
 
-export async function listFiles(dirPath: string, recursive: boolean, limit: number): Promise<[string[], boolean]> {
+export async function listFiles(
+	dirPath: string,
+	recursive: boolean,
+	limit: number,
+	globbyOptions: Options = {},
+): Promise<[string[], boolean]> {
 	const absolutePath = path.resolve(dirPath)
 	// Do not allow listing files in root or home directory, which cline tends to want to do when the user's prompt is vague.
 	const root = process.platform === "win32" ? path.parse(absolutePath).root : "/"
@@ -17,25 +23,6 @@ export async function listFiles(dirPath: string, recursive: boolean, limit: numb
 		return [[homeDir], false]
 	}
 
-	const dirsToIgnore = [
-		"node_modules",
-		"__pycache__",
-		"env",
-		"venv",
-		"target/dependency",
-		"build/dependencies",
-		"dist",
-		"out",
-		"bundle",
-		"vendor",
-		"tmp",
-		"temp",
-		"deps",
-		"pkg",
-		"Pods",
-		".*", // '!**/.*' excludes hidden directories, while '!**/.*/**' excludes only their contents. This way we are at least aware of the existence of hidden directories.
-	].map((dir) => `**/${dir}/**`)
-
 	const options = {
 		cwd: dirPath,
 		dot: true, // do not ignore hidden files/directories
@@ -44,6 +31,7 @@ export async function listFiles(dirPath: string, recursive: boolean, limit: numb
 		gitignore: recursive, // globby ignores any files that are gitignored
 		ignore: recursive ? dirsToIgnore : undefined, // just in case there is no gitignore, we ignore sensible defaults
 		onlyFiles: false, // true by default, false means it will list directories on their own too
+		...globbyOptions,
 	}
 	// * globs all files in one dir, ** globs files in nested directories
 	const files = recursive ? await globbyLevelByLevel(limit, options) : (await globby("*", options)).slice(0, limit)

src/services/glob/list-files.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from "vitest/config"
+
+export default defineConfig({
+	test: {
+		include: ["src/**/*.spec.ts"],
+		watch: false,
+	},
+})