fix(chat): Ignore directories and files that are not relevant to the code analysis in listDirectory

jguoamz · jguoamz · commit a1a9e344414d · 2025-04-14T17:00:54.000-07:00
diff --git a/packages/core/src/codewhispererChat/constants.ts b/packages/core/src/codewhispererChat/constants.ts
@@ -55,3 +55,115 @@ export const defaultContextLengths: ContextLengths = {
 }
 
 export const defaultStreamingResponseTimeoutInMs = 180_000
+
+export const ignoredDirectoriesAndFiles = [
+    // Dependency directories
+    'node_modules',
+    '.venv',
+    'venv',
+    'bower_components',
+    'jspm_packages',
+    // Build outputs
+    'dist',
+    'build',
+    'out',
+    'target',
+    '.gradle',
+    '.pytest_cache',
+    '.tox',
+    '__snapshots__',
+    // Compiled files
+    '*.class',
+    '*.o',
+    '*.a',
+    '*.so',
+    '*.pyc',
+    '__pycache__',
+    '*.exe',
+    '*.dll',
+    // Package files
+    '*.jar',
+    '*.gem',
+    '*.vsix',
+    '*.zip',
+    '*.tar.gz',
+    '*.rar',
+    // IDE and editor files
+    '.idea/',
+    '.vscode/',
+    '*.sublime-*',
+    '*.swp',
+    '*.swo',
+    '.project',
+    '.classpath',
+    '*.iml',
+    // Log files
+    '*.log',
+    'logs/',
+    'npm-debug.log*',
+    // Coverage and test reports
+    'coverage/',
+    '.nyc_output/',
+    'test-results/',
+    '.test-reports/',
+    // Cache directories
+    '.cache/',
+    '.sass-cache/',
+    '.eslintcache',
+    '.parcel-cache',
+    // Environment and local configuration
+    '.env',
+    '.env.local',
+    '*.env.*',
+    '*.local.json',
+    '*.local.yml',
+    'config.local.*',
+    '.npmrc',
+    '.yarnrc',
+    '.dockerignore',
+    // OS specific files
+    '.DS_Store',
+    'Thumbs.db',
+    'desktop.ini',
+    // Temporary files
+    'tmp/',
+    'temp/',
+    '*.tmp',
+    '*.bak',
+    '*.bk',
+    // Generated documentation
+    'docs/_build/',
+    'site/',
+    'public/',
+    // Database files
+    '*.sqlite',
+    '*.db',
+    // Secrets and credentials
+    '*.pem',
+    '*.key',
+    'id_rsa',
+    'id_dsa',
+    '*.pfx',
+    '*.p12',
+    'credentials.json',
+    '*_credentials.*',
+    'aws-credentials.*',
+    'secrets.*',
+    // Version Control Directories
+    '.git/',
+    '.svn/',
+    '.hg/',
+    '.bzr/',
+    // Generated Code
+    '*.generated.*',
+    '*.auto.*',
+    '*.g.*',
+    // Cloud Provider Specific
+    '.terraform/',
+    '.serverless/',
+    'cdk.out/',
+    '.aws-sam/',
+    '.amplify/',
+    // Mobile Development
+    'Pods/',
+]
diff --git a/packages/core/src/codewhispererChat/tools/tool_index.json b/packages/core/src/codewhispererChat/tools/tool_index.json
@@ -83,7 +83,7 @@
     },
     "listDirectory": {
         "name": "listDirectory",
-        "description": "List the contents of a directory and its subdirectories.\n * Use this tool for discovery, before using more targeted tools like fsRead.\n *Useful to try to understand the file structure before diving deeper into specific files.\n *Can be used to explore the codebase.\n *Results clearly distinguish between files, directories or symlinks with [FILE], [DIR] and [LINK] prefixes.",
+        "description": "List the contents of a directory and its subdirectories, it will filter out build and artifacts directories and files such as `node_modules/`, `build/`, `.git/` or `*.log`.\n * Use this tool for discovery, before using more targeted tools like fsRead.\n *Useful to try to understand the file structure before diving deeper into specific files.\n *Can be used to explore the codebase.\n *Results clearly distinguish between files, directories or symlinks with [F], [D] and [L] prefixes.",
         "inputSchema": {
             "type": "object",
             "properties": {
diff --git a/packages/core/src/shared/utilities/workspaceUtils.ts b/packages/core/src/shared/utilities/workspaceUtils.ts
@@ -20,6 +20,7 @@ import fs from '../fs/fs'
 import { ChildProcess } from './processUtils'
 import { isWin } from '../vscode/env'
 import { maxRepoSizeBytes } from '../../amazonqFeatureDev/constants'
+import { ignoredDirectoriesAndFiles } from '../../codewhispererChat/constants'
 
 type GitIgnoreRelativeAcceptor = {
     folderPath: string
@@ -673,14 +674,14 @@ export async function findStringInDirectory(searchStr: string, dirPath: string)
 }
 
 /**
- * Returns a prefix for a directory ('[DIR]'), symlink ('[LINK]'), or file ('[FILE]').
+ * Returns a prefix for a directory ('[D]'), symlink ('[L]'), or file ('[F]').
  */
 export function formatListing(name: string, fileType: vscode.FileType, fullPath: string): string {
-    let typeChar = '[FILE]'
+    let typeChar = '[F]'
     if (fileType === vscode.FileType.Directory) {
-        typeChar = '[DIR]'
+        typeChar = '[D]'
     } else if (fileType === vscode.FileType.SymbolicLink) {
-        typeChar = '[LINK]'
+        typeChar = '[L]'
     }
     return `${typeChar} ${fullPath}`
 }
@@ -689,6 +690,7 @@ export function formatListing(name: string, fileType: vscode.FileType, fullPath:
  * Recursively lists directories using a BFS approach, returning lines like:
  * d /absolute/path/to/folder
  * - /absolute/path/to/file.txt
+ * Will filter out directories/files that should be ignored across most programming languages.
  *
  * You can either pass a custom callback or rely on the default `formatListing`.
  *
@@ -727,6 +729,10 @@ export async function readDirectoryRecursively(
         }
 
         for (const [name, fileType] of entries) {
+            if (shouldIgnoreDirAndFile(name, fileType)) {
+                logger.debug(`Ignoring: ${name} in ${uri.fsPath}`)
+                continue
+            }
             const childUri = vscode.Uri.joinPath(uri, name)
             results.push(formatter(name, fileType, childUri.fsPath))
 
@@ -739,6 +745,54 @@ export async function readDirectoryRecursively(
     return results
 }
 
+export function shouldIgnoreDirAndFile(name: string, fileType: vscode.FileType): boolean {
+    for (const pattern of ignoredDirectoriesAndFiles) {
+        // Handle exact matches
+        if (name === pattern) {
+            return true
+        }
+        // Handle directory patterns that end with /
+        if (pattern.endsWith('/') && fileType === vscode.FileType.Directory) {
+            const dirName = pattern.slice(0, -1)
+            if (name === dirName) {
+                return true
+            }
+            continue
+        }
+        // Handle patterns with wildcards
+        if (pattern.includes('*')) {
+            // Handle patterns like "*.class" (wildcard at start)
+            if (pattern.startsWith('*') && !pattern.endsWith('*')) {
+                const suffix = pattern.slice(1)
+                if (name.endsWith(suffix)) {
+                    return true
+                }
+            }
+            // Handle patterns like "npm-debug.log*" (wildcard at end)
+            else if (!pattern.startsWith('*') && pattern.endsWith('*')) {
+                const prefix = pattern.slice(0, -1)
+                if (name.startsWith(prefix)) {
+                    return true
+                }
+            }
+            // Handle patterns like "*.env.*" or "*_credentials.*" (wildcards at both ends or middle)
+            else {
+                // Convert glob pattern to regex pattern
+                const regexPattern = pattern
+                    // Escape dots
+                    .replace(/\./g, '\\.')
+                    // Convert * to .*
+                    .replace(/\*/g, '.*')
+                const regex = new RegExp(`^${regexPattern}$`)
+                if (regex.test(name)) {
+                    return true
+                }
+            }
+        }
+    }
+    return false
+}
+
 export function getWorkspacePaths() {
     const workspaceFolders = vscode.workspace.workspaceFolders
     return workspaceFolders?.map((folder) => folder.uri.fsPath) ?? []
diff --git a/packages/core/src/test/codewhispererChat/tools/listDirectory.test.ts b/packages/core/src/test/codewhispererChat/tools/listDirectory.test.ts
@@ -43,9 +43,9 @@ describe('ListDirectory Tool', () => {
         const result = await listDirectory.invoke(process.stdout)
 
         const lines = result.output.content.split('\n')
-        const hasFileA = lines.some((line: string | string[]) => line.includes('[FILE] ') && line.includes('fileA.txt'))
+        const hasFileA = lines.some((line: string | string[]) => line.includes('[F] ') && line.includes('fileA.txt'))
         const hasSubfolder = lines.some(
-            (line: string | string[]) => line.includes('[DIR] ') && line.includes('subfolder')
+            (line: string | string[]) => line.includes('[D] ') && line.includes('subfolder')
         )
 
         assert.ok(hasFileA, 'Should list fileA.txt in the directory output')
@@ -62,17 +62,35 @@ describe('ListDirectory Tool', () => {
         const result = await listDirectory.invoke(process.stdout)
 
         const lines = result.output.content.split('\n')
-        const hasFileA = lines.some((line: string | string[]) => line.includes('[FILE] ') && line.includes('fileA.txt'))
+        const hasFileA = lines.some((line: string | string[]) => line.includes('[F] ') && line.includes('fileA.txt'))
         const hasSubfolder = lines.some(
-            (line: string | string[]) => line.includes('[DIR] ') && line.includes('subfolder')
+            (line: string | string[]) => line.includes('[D] ') && line.includes('subfolder')
         )
-        const hasFileB = lines.some((line: string | string[]) => line.includes('[FILE] ') && line.includes('fileB.md'))
+        const hasFileB = lines.some((line: string | string[]) => line.includes('[F] ') && line.includes('fileB.md'))
 
         assert.ok(hasFileA, 'Should list fileA.txt in the directory output')
         assert.ok(hasSubfolder, 'Should list the subfolder in the directory output')
         assert.ok(hasFileB, 'Should list fileB.md in the subfolder in the directory output')
     })
 
+    it('lists directory contents with ignored pattern', async () => {
+        await testFolder.mkdir('node_modules')
+        await testFolder.write(path.join('node_modules', 'fileC.md'), '# fileC')
+
+        const listDirectory = new ListDirectory({ path: testFolder.path })
+        await listDirectory.validate()
+        const result = await listDirectory.invoke(process.stdout)
+
+        const lines = result.output.content.split('\n')
+        const hasNodeModules = lines.some(
+            (line: string | string[]) => line.includes('[D] ') && line.includes('node_modules')
+        )
+        const hasFileC = lines.some((line: string | string[]) => line.includes('[F] ') && line.includes('fileC.md'))
+
+        assert.ok(!hasNodeModules, 'Should not list node_modules in the directory output')
+        assert.ok(!hasFileC, 'Should not list fileC.md under node_modules in the directory output')
+    })
+
     it('throws error if path does not exist', async () => {
         const missingPath = path.join(testFolder.path, 'no_such_file.txt')
         const listDirectory = new ListDirectory({ path: missingPath, maxDepth: 0 })
diff --git a/packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts b/packages/core/src/testInteg/shared/utilities/workspaceUtils.test.ts
@@ -14,6 +14,7 @@ import {
     findStringInDirectory,
     getWorkspaceFoldersByPrefixes,
     getWorkspaceRelativePath,
+    shouldIgnoreDirAndFile,
 } from '../../../shared/utilities/workspaceUtils'
 import { getTestWorkspaceFolder } from '../../integrationTestsUtilities'
 import globals from '../../../shared/extensionGlobals'
@@ -591,6 +592,38 @@ describe('workspaceUtils', () => {
         })
     })
 
+    describe('shouldIgnoreDirAndFile', function () {
+        it('handles exact matches', function () {
+            assert.strictEqual(shouldIgnoreDirAndFile('node_modules', vscode.FileType.Directory), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('.env', vscode.FileType.File), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('random_file.txt', vscode.FileType.File), false)
+        })
+
+        it('handles directory patterns with trailing slash', function () {
+            assert.strictEqual(shouldIgnoreDirAndFile('.idea', vscode.FileType.Directory), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('logs', vscode.FileType.Directory), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('.idea', vscode.FileType.File), false)
+        })
+
+        it('handles wildcard patterns at the start', function () {
+            assert.strictEqual(shouldIgnoreDirAndFile('example.class', vscode.FileType.File), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('test.log', vscode.FileType.File), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('class.example', vscode.FileType.File), false)
+        })
+
+        it('handles wildcard patterns at the end', function () {
+            assert.strictEqual(shouldIgnoreDirAndFile('npm-debug.log123', vscode.FileType.File), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('npm-debug.txt', vscode.FileType.File), false)
+        })
+
+        it('handles complex wildcard patterns', function () {
+            assert.strictEqual(shouldIgnoreDirAndFile('test.env.local', vscode.FileType.File), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('my_credentials.json', vscode.FileType.File), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('config.local.json', vscode.FileType.File), true)
+            assert.strictEqual(shouldIgnoreDirAndFile('random.txt', vscode.FileType.File), false)
+        })
+    })
+
     describe('findStringInDirectory', function () {
         it('prints the line with the detected string to stdout', async () => {
             const fileAmount = 1
