Merge pull request #6208 from continuedev/dallin/limit-glob-output

Limit glob tool output

Author: Patrick-Erichsen

core/index.d.ts

@@ -783,7 +783,7 @@ export interface IDE {
 
   getSearchResults(query: string, maxResults?: number): Promise<string>;
 
-  getFileResults(pattern: string): Promise<string[]>;
+  getFileResults(pattern: string, maxResults?: number): Promise<string[]>;
 
   subprocess(command: string, cwd?: string): Promise<[string, string]>;
 

core/protocol/ide.ts

@@ -30,7 +30,7 @@ export type ToIdeFromWebviewOrCoreProtocol = {
   openUrl: [string, void];
   runCommand: [{ command: string; options?: TerminalOptions }, void];
   getSearchResults: [{ query: string; maxResults?: number }, string];
-  getFileResults: [{ pattern: string }, string[]];
+  getFileResults: [{ pattern: string; maxResults?: number }, string[]];
   subprocess: [{ command: string; cwd?: string }, [string, string]];
   saveFile: [{ filepath: string }, void];
   fileExists: [{ filepath: string }, boolean];

core/protocol/messenger/reverseMessageIde.ts

@@ -163,7 +163,7 @@ export class ReverseMessageIde {
     });
 
     this.on("getFileResults", (data) => {
-      return this.ide.getFileResults(data.pattern);
+      return this.ide.getFileResults(data.pattern, data.maxResults);
     });
 
     this.on("getProblems", (data) => {

core/tools/definitions/globSearch.ts

@@ -12,7 +12,8 @@ export const globSearchTool: Tool = {
   group: BUILT_IN_GROUP_NAME,
   function: {
     name: BuiltInToolNames.FileGlobSearch,
-    description: "Search for files in the project",
+    description:
+      "Search for files in the project. Output may be truncated; use targeted patterns",
     parameters: {
       type: "object",
       required: ["pattern"],

core/tools/implementations/globSearch.ts

@@ -1,12 +1,28 @@
 import { ToolImpl } from ".";
+import { ContextItem } from "../..";
 
+const MAX_AGENT_GLOB_RESULTS = 100;
 export const fileGlobSearchImpl: ToolImpl = async (args, extras) => {
-  const results = await extras.ide.getFileResults(args.pattern);
-  return [
+  const results = await extras.ide.getFileResults(
+    args.pattern,
+    MAX_AGENT_GLOB_RESULTS,
+  );
+  const contextItems: ContextItem[] = [
     {
       name: "File results",
       description: "Results from file glob search",
       content: results.join("\n"),
     },
   ];
+
+  // In case of truncation, add a warning
+  if (results.length === MAX_AGENT_GLOB_RESULTS) {
+    contextItems.push({
+      name: "Truncation warning",
+      description: "Inform the model that results were truncated",
+      content: `Warning: the results above were truncated to the first ${MAX_AGENT_GLOB_RESULTS} files. If the results are not satisfactory, refine your search pattern`,
+    });
+  }
+
+  return contextItems;
 };

core/util/filesystem.ts

@@ -238,7 +238,10 @@ class FileSystemIde implements IDE {
     return "";
   }
 
-  async getFileResults(pattern: string): Promise<string[]> {
+  async getFileResults(
+    pattern: string,
+    maxResults?: number,
+  ): Promise<string[]> {
     return [];
   }
 

docs/docs/agent/how-it-works.md

@@ -31,7 +31,8 @@ Continue includes several built-in tools which provide the model access to IDE f
 - **Read file** (`builtin_read_file`): read the contents of a file within the project
 - **Read currently open file** (`builtin_read_currently_open_file`): read the contents of the currently open file
 - **Create new file** (`builtin_create_new_file`): Create a new file within the project, with path and contents specified by the model
-- **Grep search** (`builtin_grep_search`): perform a `ripgrep` search within the project. Results are limited to 100 results and 5000 characters (model is warned if this is exceeded)
+- **Exact search** (`builtin_exact_search`): perform a `ripgrep` search within the project
+- **Glob search** (`builtin_file_glob_search`): perform a glob search on files in the project. Currently limits to 100 output and warns the Agent if limit reached.
 - **Run terminal command** (`builtin_run_terminal_command`): run a terminal command from the workspace root
 - **Search web** (`builtin_search_web`): Perform a web search to get top results
 - **View diff** (`builtin_view_diff`): View the current working git diff

extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IdeProtocolClient.kt

@@ -365,7 +365,7 @@ class IdeProtocolClient(
                             dataElement.toString(),
                             GetFileResultsParams::class.java
                         )
-                        val results = ide.getFileResults(params.pattern)
+                        val results = ide.getFileResults(params.pattern, params.maxResults)
                         respond(results)
                     }
 

extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/continue/IntelliJIde.kt

@@ -315,20 +315,26 @@ class IntelliJIDE(
         return getOpenFiles()
     }
 
-    override suspend fun getFileResults(pattern: String): List<String> {
+    override suspend fun getFileResults(pattern: String, maxResults: Int?): List<String> {
         val ideInfo = this.getIdeInfo()
         if (ideInfo.remoteName == "local") {
             try {
-                val command = GeneralCommandLine(
+                var commandArgs = mutableListOf<String>(
                     ripgrep,
                     "--files",
                     "--iglob",
                     pattern,
                     "--ignore-file",
                     ".continueignore",
                     "--ignore-file",
-                    ".gitignore",
+                    ".gitignore"
                 )
+                if (maxResults != null) {
+                    commandArgs.add("--max-count")
+                    commandArgs.add(maxResults.toString())
+                }
+
+                val command = GeneralCommandLine(commandArgs)
 
                 command.setWorkDirectory(project.basePath)
                 val results = ExecUtil.execAndGetOutput(command).stdout

extensions/intellij/src/main/kotlin/com/github/continuedev/continueintellijextension/protocol/ide.kt

@@ -23,7 +23,7 @@ typealias getTagsParams = String
 
 data class GetSearchResultsParams(val query: String, val maxResults: Int?)
 
-data class GetFileResultsParams(val pattern: String)
+data class GetFileResultsParams(val pattern: String, val maxResults: Int?)
 
 data class SaveFileParams(val filepath: String)