From 4c0add7bf4bc93af0295d4175f58a19225e38328 Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Fri, 4 Apr 2025 14:55:33 +0300 Subject: [PATCH] feat: Add a way to limit the number of results from search_files tool Currently search_files will produce up to 300 results (an arbitrary number). In cortain cases this is too large number because only first few occurrences are wanted. Unfortunately if too generic search pattern is selected, lots of results will be returned to the LLM and this will reduce remaining context significantly and require more time to process than needed. To address this, an explicit optional max_results parameter has been added to the search_files tool. Its explanation is simple and it does not increase the Roo Code prompt size too much. If the max_results parameter is not supplied, code works exactly as before. --- src/core/Cline.ts | 8 ++ src/core/assistant-message/index.ts | 3 +- .../__snapshots__/system.test.ts.snap | 75 +++++++++++++++---- src/core/prompts/responses.ts | 3 + src/core/prompts/tools/search-files.ts | 5 +- src/core/tools/searchFilesTool.ts | 9 +++ src/services/ripgrep/index.ts | 23 +++--- src/shared/ExtensionMessage.ts | 1 + 8 files changed, 101 insertions(+), 26 deletions(-) diff --git a/src/core/Cline.ts b/src/core/Cline.ts index c7e9fe66a55..47208958e25 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -582,6 +582,14 @@ export class Cline extends EventEmitter { return formatResponse.toolError(formatResponse.missingToolParameterError(paramName)) } + async sayAndCreateInvalidNumberParamError(toolName: ToolUseName, paramName: string) { + await this.say( + "error", + `Roo tried to use ${toolName} with invalid number value for parameter '${paramName}'. Retrying...`, + ) + return formatResponse.toolError(formatResponse.invalidNumberParameterError(paramName)) + } + // Task lifecycle private async startTask(task?: string, images?: string[]): Promise { diff --git a/src/core/assistant-message/index.ts b/src/core/assistant-message/index.ts index 59f9a578b4b..bd2bd20ec32 100644 --- a/src/core/assistant-message/index.ts +++ b/src/core/assistant-message/index.ts @@ -38,6 +38,7 @@ export const toolParamNames = [ "line_count", "regex", "file_pattern", + "max_results", "recursive", "action", "url", @@ -100,7 +101,7 @@ export interface InsertCodeBlockToolUse extends ToolUse { export interface SearchFilesToolUse extends ToolUse { name: "search_files" - params: Partial, "path" | "regex" | "file_pattern">> + params: Partial, "path" | "regex" | "file_pattern" | "max_results">> } export interface ListFilesToolUse extends ToolUse { diff --git a/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap b/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap index e9538bc308b..abf442cabfc 100644 --- a/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap +++ b/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap @@ -90,18 +90,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -487,18 +490,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -973,18 +979,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -1423,18 +1432,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -1820,18 +1832,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -2217,18 +2232,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -2614,18 +2632,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -3060,18 +3081,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -3525,18 +3549,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -3971,18 +3998,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -4464,18 +4494,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -4903,18 +4936,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -5462,18 +5498,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -5935,18 +5974,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files @@ -6306,18 +6348,21 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory /test/path). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ## list_files diff --git a/src/core/prompts/responses.ts b/src/core/prompts/responses.ts index 3a1eb92a2ee..a0786020a7a 100644 --- a/src/core/prompts/responses.ts +++ b/src/core/prompts/responses.ts @@ -35,6 +35,9 @@ Otherwise, if you have not completed the task and do not need additional informa missingToolParameterError: (paramName: string) => `Missing value for required parameter '${paramName}'. Please retry with complete response.\n\n${toolUseInstructionsReminder}`, + invalidNumberParameterError: (paramName: string) => + `Incorrect number value for parameter '${paramName}'. Please retry and use a number.`, + invalidMcpToolArgumentError: (serverName: string, toolName: string) => `Invalid JSON argument used with ${serverName} for ${toolName}. Please retry with a properly formatted JSON argument.`, diff --git a/src/core/prompts/tools/search-files.ts b/src/core/prompts/tools/search-files.ts index 8353cc43c8d..c5131b58a6b 100644 --- a/src/core/prompts/tools/search-files.ts +++ b/src/core/prompts/tools/search-files.ts @@ -7,17 +7,20 @@ Parameters: - path: (required) The path of the directory to search in (relative to the current working directory ${args.cwd}). This directory will be recursively searched. - regex: (required) The regular expression pattern to search for. Uses Rust regex syntax. - file_pattern: (optional) Glob pattern to filter files (e.g., '*.ts' for TypeScript files). If not provided, it will search all files (*). +- max_results: (optional) The maximum number of results to return. If not provided, a reasonable default is used. Usage: Directory path here Your regex pattern here file pattern here (optional) +Maximum number of results to return (optional) -Example: Requesting to search for all .ts files in the current directory +Example: Requesting to search for all .ts files in the current directory and limit the result count to 100. . .* *.ts +100 ` } diff --git a/src/core/tools/searchFilesTool.ts b/src/core/tools/searchFilesTool.ts index e3659da9a11..aec8c5cae0a 100644 --- a/src/core/tools/searchFilesTool.ts +++ b/src/core/tools/searchFilesTool.ts @@ -17,11 +17,13 @@ export async function searchFilesTool( const relDirPath: string | undefined = block.params.path const regex: string | undefined = block.params.regex const filePattern: string | undefined = block.params.file_pattern + const maxResults: string | undefined = block.params.max_results const sharedMessageProps: ClineSayTool = { tool: "searchFiles", path: getReadablePath(cline.cwd, removeClosingTag("path", relDirPath)), regex: removeClosingTag("regex", regex), filePattern: removeClosingTag("file_pattern", filePattern), + maxResults: removeClosingTag("max_results", maxResults), } try { if (block.partial) { @@ -42,6 +44,12 @@ export async function searchFilesTool( pushToolResult(await cline.sayAndCreateMissingParamError("search_files", "regex")) return } + const maxResultsNumber = parseInt(maxResults ?? "") + if (maxResults !== undefined && isNaN(maxResultsNumber)) { + cline.consecutiveMistakeCount++ + pushToolResult(await cline.sayAndCreateInvalidNumberParamError("search_files", "max_results")) + return + } cline.consecutiveMistakeCount = 0 const absolutePath = path.resolve(cline.cwd, relDirPath) const results = await regexSearchFiles( @@ -49,6 +57,7 @@ export async function searchFilesTool( absolutePath, regex, filePattern, + maxResultsNumber, cline.rooIgnoreController, ) const completeMessage = JSON.stringify({ diff --git a/src/services/ripgrep/index.ts b/src/services/ripgrep/index.ts index 89e1da62f80..86a4579f892 100644 --- a/src/services/ripgrep/index.ts +++ b/src/services/ripgrep/index.ts @@ -66,7 +66,7 @@ interface SearchLineResult { column?: number } // Constants -const MAX_RESULTS = 300 +const DEFAULT_MAX_RESULTS = 300 const MAX_LINE_LENGTH = 500 /** @@ -95,7 +95,7 @@ export async function getBinPath(vscodeAppRoot: string): Promise { +async function execRipgrep(bin: string, args: string[], maxResults: number): Promise { return new Promise((resolve, reject) => { const rgProcess = childProcess.spawn(bin, args) // cross-platform alternative to head, which is ripgrep author's recommendation for limiting output. @@ -106,7 +106,7 @@ async function execRipgrep(bin: string, args: string[]): Promise { let output = "" let lineCount = 0 - const maxLines = MAX_RESULTS * 5 // limiting ripgrep output with max lines since there's no other way to limit results. it's okay that we're outputting as json, since we're parsing it line by line and ignore anything that's not part of a match. This assumes each result is at most 5 lines. + const maxLines = maxResults * 5 // limiting ripgrep output with max lines since there's no other way to limit results. it's okay that we're outputting as json, since we're parsing it line by line and ignore anything that's not part of a match. This assumes each result is at most 5 lines. rl.on("line", (line) => { if (lineCount < maxLines) { @@ -140,6 +140,7 @@ export async function regexSearchFiles( directoryPath: string, regex: string, filePattern?: string, + maxResults?: number, rooIgnoreController?: RooIgnoreController, ): Promise { const vscodeAppRoot = vscode.env.appRoot @@ -151,9 +152,13 @@ export async function regexSearchFiles( const args = ["--json", "-e", regex, "--glob", filePattern || "*", "--context", "1", directoryPath] + if (maxResults === undefined) { + maxResults = DEFAULT_MAX_RESULTS + } + let output: string try { - output = await execRipgrep(rgPath, args) + output = await execRipgrep(rgPath, args, maxResults) } catch (error) { console.error("Error executing ripgrep:", error) return "No results found" @@ -217,22 +222,22 @@ export async function regexSearchFiles( ? results.filter((result) => rooIgnoreController.validateAccess(result.file)) : results - return formatResults(filteredResults, cwd) + return formatResults(filteredResults, cwd, maxResults) } -function formatResults(fileResults: SearchFileResult[], cwd: string): string { +function formatResults(fileResults: SearchFileResult[], cwd: string, maxResults: number): string { const groupedResults: { [key: string]: SearchResult[] } = {} let totalResults = fileResults.reduce((sum, file) => sum + file.searchResults.length, 0) let output = "" - if (totalResults >= MAX_RESULTS) { - output += `Showing first ${MAX_RESULTS} of ${MAX_RESULTS}+ results. Use a more specific search if necessary.\n\n` + if (totalResults >= maxResults) { + output += `Showing first ${maxResults} of ${maxResults}+ results. Use a more specific search if necessary.\n\n` } else { output += `Found ${totalResults === 1 ? "1 result" : `${totalResults.toLocaleString()} results`}.\n\n` } // Group results by file name - fileResults.slice(0, MAX_RESULTS).forEach((file) => { + fileResults.slice(0, maxResults).forEach((file) => { const relativeFilePath = path.relative(cwd, file.file) if (!groupedResults[relativeFilePath]) { groupedResults[relativeFilePath] = [] diff --git a/src/shared/ExtensionMessage.ts b/src/shared/ExtensionMessage.ts index 095279ffded..ebb0785ca46 100644 --- a/src/shared/ExtensionMessage.ts +++ b/src/shared/ExtensionMessage.ts @@ -226,6 +226,7 @@ export interface ClineSayTool { content?: string regex?: string filePattern?: string + maxResults?: string mode?: string reason?: string isOutsideWorkspace?: boolean