Merge pull request aws#7042 from jguoamz/listDirFix

fix(chat): Ignore directories and files that are not relevant to the code analysis in listDirectory

Author: mr-lee

packages/core/src/codewhispererChat/constants.ts

@@ -55,3 +55,14 @@ export const defaultContextLengths: ContextLengths = {
 }
 
 export const defaultStreamingResponseTimeoutInMs = 180_000
+
+export const ignoredDirectoriesAndFiles = [
+    // Dependency directories
+    'node_modules',
+    // Build outputs
+    'dist',
+    'build',
+    'out',
+    // OS specific files
+    '.DS_Store',
+]

packages/core/src/codewhispererChat/tools/tool_index.json

@@ -83,7 +83,7 @@
     },
     "listDirectory": {
         "name": "listDirectory",
-        "description": "List the contents of a directory and its subdirectories.\n * Use this tool for discovery, before using more targeted tools like fsRead.\n *Useful to try to understand the file structure before diving deeper into specific files.\n *Can be used to explore the codebase.\n *Results clearly distinguish between files, directories or symlinks with [FILE], [DIR] and [LINK] prefixes.",
+        "description": "List the contents of a directory and its subdirectories, it will filter out build outputs such as `build/`, `out/` and `dist` and dependency directory such as `node_modules/`.\n * Use this tool for discovery, before using more targeted tools like fsRead.\n *Useful to try to understand the file structure before diving deeper into specific files.\n *Can be used to explore the codebase.\n *Results clearly distinguish between files, directories or symlinks with [F], [D] and [L] prefixes.",
         "inputSchema": {
             "type": "object",
             "properties": {

packages/core/src/shared/utilities/workspaceUtils.ts

@@ -20,6 +20,7 @@ import fs from '../fs/fs'
 import { ChildProcess } from './processUtils'
 import { isWin } from '../vscode/env'
 import { maxRepoSizeBytes } from '../../amazonqFeatureDev/constants'
+import { ignoredDirectoriesAndFiles } from '../../codewhispererChat/constants'
 
 type GitIgnoreRelativeAcceptor = {
     folderPath: string
@@ -673,14 +674,14 @@ export async function findStringInDirectory(searchStr: string, dirPath: string)
 }
 
 /**
- * Returns a prefix for a directory ('[DIR]'), symlink ('[LINK]'), or file ('[FILE]').
+ * Returns a prefix for a directory ('[D]'), symlink ('[L]'), or file ('[F]').
  */
 export function formatListing(name: string, fileType: vscode.FileType, fullPath: string): string {
-    let typeChar = '[FILE]'
+    let typeChar = '[F]'
     if (fileType === vscode.FileType.Directory) {
-        typeChar = '[DIR]'
+        typeChar = '[D]'
     } else if (fileType === vscode.FileType.SymbolicLink) {
-        typeChar = '[LINK]'
+        typeChar = '[L]'
     }
     return `${typeChar} ${fullPath}`
 }
@@ -689,6 +690,7 @@ export function formatListing(name: string, fileType: vscode.FileType, fullPath:
  * Recursively lists directories using a BFS approach, returning lines like:
  * d /absolute/path/to/folder
  * - /absolute/path/to/file.txt
+ * Will filter out directories/files that should be ignored across most programming languages.
  *
  * You can either pass a custom callback or rely on the default `formatListing`.
  *
@@ -727,6 +729,10 @@ export async function readDirectoryRecursively(
         }
 
         for (const [name, fileType] of entries) {
+            if (shouldIgnoreDirAndFile(name, fileType)) {
+                logger.debug(`Ignoring: ${name} in ${uri.fsPath}`)
+                continue
+            }
             const childUri = vscode.Uri.joinPath(uri, name)
             results.push(formatter(name, fileType, childUri.fsPath))
 
@@ -739,6 +745,54 @@ export async function readDirectoryRecursively(
     return results
 }
 
+export function shouldIgnoreDirAndFile(name: string, fileType: vscode.FileType): boolean {
+    for (const pattern of ignoredDirectoriesAndFiles) {
+        // Handle exact matches
+        if (name === pattern) {
+            return true
+        }
+        // Handle directory patterns that end with /
+        if (pattern.endsWith('/') && fileType === vscode.FileType.Directory) {
+            const dirName = pattern.slice(0, -1)
+            if (name === dirName) {
+                return true
+            }
+            continue
+        }
+        // Handle patterns with wildcards
+        if (pattern.includes('*')) {
+            // Handle patterns like "*.class" (wildcard at start)
+            if (pattern.startsWith('*') && !pattern.endsWith('*')) {
+                const suffix = pattern.slice(1)
+                if (name.endsWith(suffix)) {
+                    return true
+                }
+            }
+            // Handle patterns like "npm-debug.log*" (wildcard at end)
+            else if (!pattern.startsWith('*') && pattern.endsWith('*')) {
+                const prefix = pattern.slice(0, -1)
+                if (name.startsWith(prefix)) {
+                    return true
+                }
+            }
+            // Handle patterns like "*.env.*" or "*_credentials.*" (wildcards at both ends or middle)
+            else {
+                // Convert glob pattern to regex pattern
+                const regexPattern = pattern
+                    // Escape dots
+                    .replace(/\./g, '\\.')
+                    // Convert * to .*
+                    .replace(/\*/g, '.*')
+                const regex = new RegExp(`^${regexPattern}$`)
+                if (regex.test(name)) {
+                    return true
+                }
+            }
+        }
+    }
+    return false
+}
+
 export function getWorkspacePaths() {
     const workspaceFolders = vscode.workspace.workspaceFolders
     return workspaceFolders?.map((folder) => folder.uri.fsPath) ?? []

packages/core/src/test/codewhispererChat/tools/listDirectory.test.ts

@@ -43,9 +43,9 @@ describe('ListDirectory Tool', () => {
         const result = await listDirectory.invoke(process.stdout)
 
         const lines = result.output.content.split('\n')
-        const hasFileA = lines.some((line: string | string[]) => line.includes('[FILE] ') && line.includes('fileA.txt'))
+        const hasFileA = lines.some((line: string | string[]) => line.includes('[F] ') && line.includes('fileA.txt'))
         const hasSubfolder = lines.some(
-            (line: string | string[]) => line.includes('[DIR] ') && line.includes('subfolder')
+            (line: string | string[]) => line.includes('[D] ') && line.includes('subfolder')
         )
 
         assert.ok(hasFileA, 'Should list fileA.txt in the directory output')
@@ -62,17 +62,35 @@ describe('ListDirectory Tool', () => {
         const result = await listDirectory.invoke(process.stdout)
 
         const lines = result.output.content.split('\n')
-        const hasFileA = lines.some((line: string | string[]) => line.includes('[FILE] ') && line.includes('fileA.txt'))
+        const hasFileA = lines.some((line: string | string[]) => line.includes('[F] ') && line.includes('fileA.txt'))
         const hasSubfolder = lines.some(
-            (line: string | string[]) => line.includes('[DIR] ') && line.includes('subfolder')
+            (line: string | string[]) => line.includes('[D] ') && line.includes('subfolder')
         )
-        const hasFileB = lines.some((line: string | string[]) => line.includes('[FILE] ') && line.includes('fileB.md'))
+        const hasFileB = lines.some((line: string | string[]) => line.includes('[F] ') && line.includes('fileB.md'))
 
         assert.ok(hasFileA, 'Should list fileA.txt in the directory output')
         assert.ok(hasSubfolder, 'Should list the subfolder in the directory output')
         assert.ok(hasFileB, 'Should list fileB.md in the subfolder in the directory output')
     })
 
+    it('lists directory contents with ignored pattern', async () => {
+        await testFolder.mkdir('node_modules')
+        await testFolder.write(path.join('node_modules', 'fileC.md'), '# fileC')
+
+        const listDirectory = new ListDirectory({ path: testFolder.path })
+        await listDirectory.validate()
+        const result = await listDirectory.invoke(process.stdout)
+
+        const lines = result.output.content.split('\n')
+        const hasNodeModules = lines.some(
+            (line: string | string[]) => line.includes('[D] ') && line.includes('node_modules')
+        )
+        const hasFileC = lines.some((line: string | string[]) => line.includes('[F] ') && line.includes('fileC.md'))
+
+        assert.ok(!hasNodeModules, 'Should not list node_modules in the directory output')
+        assert.ok(!hasFileC, 'Should not list fileC.md under node_modules in the directory output')
+    })
+
     it('throws error if path does not exist', async () => {
         const missingPath = path.join(testFolder.path, 'no_such_file.txt')
         const listDirectory = new ListDirectory({ path: missingPath, maxDepth: 0 })

packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts

@@ -14,6 +14,7 @@ import {
     findStringInDirectory,
     getWorkspaceFoldersByPrefixes,
     getWorkspaceRelativePath,
+    shouldIgnoreDirAndFile,
 } from '../../../shared/utilities/workspaceUtils'
 import { getTestWorkspaceFolder } from '../../integrationTestsUtilities'
 import globals from '../../../shared/extensionGlobals'
@@ -591,6 +592,13 @@ describe('workspaceUtils', () => {
         })
     })
 
+    describe('shouldIgnoreDirAndFile', function () {
+        it('handles exact matches', function () {
+            assert.strictEqual(shouldIgnoreDirAndFile('node_modules', vscode.FileType.Directory), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('random_file.txt', vscode.FileType.File), false)
+        })
+    })
+
     describe('findStringInDirectory', function () {
         it('prints the line with the detected string to stdout', async () => {
             const fileAmount = 1