fix(featureDev): Prevent crash on large repos or unsupported encoding during file collection

Author: jsalles

packages/core/src/amazonqFeatureDev/constants.ts

@@ -14,6 +14,9 @@ export const featureDevChat = 'featureDevChat'
 
 export const featureName = 'Amazon Q feature development'
 
+// Max allowed size for file collection
+export const maxRepoSize = 200 * 1024 * 1024
+
 // License text that's used in codewhisperer reference log
 export const referenceLogText = (reference: CodeReference) =>
     `[${new Date().toLocaleString()}] Accepted recommendation from Amazon Q. Code provided with reference under <a href="${LicenseUtil.getLicenseHtml(

packages/core/src/amazonqFeatureDev/util/files.ts

@@ -12,7 +12,7 @@ import { Uri } from 'vscode'
 import { GitIgnoreFilter } from './gitignore'
 
 import AdmZip from 'adm-zip'
-import { PrepareRepoFailedError } from '../errors'
+import { ContentLengthError, PrepareRepoFailedError } from '../errors'
 import { getLogger } from '../../shared/logger/logger'
 import { maxFileSizeBytes } from '../limits'
 import { createHash } from 'crypto'
@@ -21,6 +21,7 @@ import { ToolkitError } from '../../shared/errors'
 import { AmazonqCreateUpload, Metric } from '../../shared/telemetry/telemetry'
 import { TelemetryHelper } from './telemetryHelper'
 import { sanitizeFilename } from '../../shared/utilities/textUtilities'
+import { maxRepoSize } from '../constants'
 
 export function getExcludePattern(additionalPatterns: string[] = []) {
     const globAlwaysExcludedDirs = getGlobDirExcludedPatterns().map(pattern => `**/${pattern}/*`)
@@ -94,6 +95,8 @@ export async function collectFiles(
         return prefix === '' ? path : `${prefix}/${path}`
     }
 
+    let totalSize = 0
+    const decoder = new TextDecoder('utf8')
     for (const rootPath of sourcePaths) {
         const allFiles = await vscode.workspace.findFiles(
             new vscode.RelativePattern(rootPath, '**'),
@@ -103,7 +106,8 @@ export async function collectFiles(
 
         for (const file of files) {
             try {
-                const fileContent = await SystemUtilities.readFile(file, new TextDecoder('utf8', { fatal: true }))
+                const fileStat = await vscode.workspace.fs.stat(file)
+                const fileContent = await SystemUtilities.readFile(file, decoder)
                 const relativePath = getWorkspaceRelativePath(file.fsPath, { workspaceFolders })
 
                 if (relativePath) {
@@ -115,9 +119,13 @@ export async function collectFiles(
                         zipFilePath: prefixWithFolderPrefix(relativePath.workspaceFolder, relativePath.relativePath),
                     })
                 }
+                totalSize += fileStat.size
+                if (totalSize > maxRepoSize) {
+                    throw new ContentLengthError()
+                }
             } catch (error) {
                 getLogger().debug(
-                    `featureDev: Failed to read file ${file.fsPath} when collecting repository: ${error}. Skipping the file`
+                    `featureDev: Failed to read file ${file.fsPath} when collecting repository. Skipping the file`
                 )
             }
         }