fix(amazonq): incorrect project scan size limit #4924

Problem
`collectFiles` method uses a different max payload size value than security scans.

Solution
- Pass the desired max value to `collectFiles`
- Use 500MB as the max size

Author: ctlai95

packages/core/src/amazonqFeatureDev/controllers/chat/controller.ts

@@ -32,8 +32,9 @@ import { submitFeedback } from '../../../feedback/vue/submitFeedback'
 import { placeholder } from '../../../shared/vscode/commands2'
 import { EditorContentController } from '../../../amazonq/commons/controllers/contentController'
 import { openUrl } from '../../../shared/utilities/vsCodeUtils'
-import { getPathsFromZipFilePath, getWorkspaceFoldersByPrefixes } from '../../util/files'
+import { getPathsFromZipFilePath } from '../../util/files'
 import { examples, newTaskChanges, approachCreation, sessionClosed, updateCode } from '../../userFacingText'
+import { getWorkspaceFoldersByPrefixes } from '../../../shared/utilities/workspaceUtils'
 
 export interface ChatControllerEventEmitters {
     readonly processHumanChatMessage: EventEmitter<any>

packages/core/src/amazonqFeatureDev/session/sessionState.ts

@@ -26,14 +26,15 @@ import {
     SessionStateInteraction,
     SessionStatePhase,
 } from '../types'
-import { collectFiles, getWorkspaceFoldersByPrefixes, prepareRepoData } from '../util/files'
+import { prepareRepoData } from '../util/files'
 import { TelemetryHelper } from '../util/telemetryHelper'
 import { uploadCode } from '../util/upload'
 import { CodeReference } from '../../amazonq/webview/ui/connector'
 import { isPresent } from '../../shared/utilities/collectionUtils'
 import { encodeHTML } from '../../shared/utilities/textUtilities'
 import { AuthUtil } from '../../codewhisperer/util/authUtil'
 import { randomUUID } from '../../common/crypto'
+import { collectFiles, getWorkspaceFoldersByPrefixes } from '../../shared/utilities/workspaceUtils'
 
 export class ConversationNotStartedState implements Omit<SessionState, 'uploadId'> {
     public tokenSource: vscode.CancellationTokenSource

packages/core/src/amazonqFeatureDev/util/files.ts

@@ -5,11 +5,7 @@
 
 import * as vscode from 'vscode'
 import * as path from 'path'
-import { SystemUtilities } from '../../shared/systemUtilities'
-import { getGlobDirExcludedPatterns } from '../../shared/fs/watchedFiles'
-import { getWorkspaceRelativePath } from '../../shared/utilities/workspaceUtils'
-import { Uri } from 'vscode'
-import { GitIgnoreFilter } from './gitignore'
+import { collectFiles } from '../../shared/utilities/workspaceUtils'
 
 import AdmZip from 'adm-zip'
 import { ContentLengthError, PrepareRepoFailedError } from '../errors'
@@ -20,131 +16,6 @@ import { CurrentWsFolders } from '../types'
 import { ToolkitError } from '../../shared/errors'
 import { AmazonqCreateUpload, Metric } from '../../shared/telemetry/telemetry'
 import { TelemetryHelper } from './telemetryHelper'
-import { sanitizeFilename } from '../../shared/utilities/textUtilities'
-import { maxRepoSizeBytes } from '../constants'
-
-export function getExcludePattern(additionalPatterns: string[] = []) {
-    const globAlwaysExcludedDirs = getGlobDirExcludedPatterns().map(pattern => `**/${pattern}/*`)
-    const extraPatterns = [
-        '**/package-lock.json',
-        '**/yarn.lock',
-        '**/*.zip',
-        '**/*.bin',
-        '**/*.png',
-        '**/*.jpg',
-        '**/*.svg',
-        '**/*.pyc',
-        '**/license.txt',
-        '**/License.txt',
-        '**/LICENSE.txt',
-        '**/license.md',
-        '**/License.md',
-        '**/LICENSE.md',
-    ]
-    const allPatterns = [...globAlwaysExcludedDirs, ...extraPatterns, ...additionalPatterns]
-    return `{${allPatterns.join(',')}}`
-}
-
-/**
- * @param rootPath root folder to look for .gitignore files
- * @returns list of glob patterns extracted from .gitignore
- * These patterns are compatible with vscode exclude patterns
- */
-async function filterOutGitignoredFiles(rootPath: string, files: Uri[]): Promise<Uri[]> {
-    const gitIgnoreFiles = await vscode.workspace.findFiles(
-        new vscode.RelativePattern(rootPath, '**/.gitignore'),
-        getExcludePattern()
-    )
-    const gitIgnoreFilter = await GitIgnoreFilter.build(gitIgnoreFiles)
-    return gitIgnoreFilter.filterFiles(files)
-}
-
-/**
- * collects all files that are marked as source
- * @param sourcePaths the paths where collection starts
- * @param workspaceFolders the current workspace folders opened
- * @param respectGitIgnore whether to respect gitignore file
- * @returns all matched files
- */
-export async function collectFiles(
-    sourcePaths: string[],
-    workspaceFolders: CurrentWsFolders,
-    respectGitIgnore: boolean = true
-): Promise<
-    {
-        workspaceFolder: vscode.WorkspaceFolder
-        relativeFilePath: string
-        fileUri: vscode.Uri
-        fileContent: string
-        zipFilePath: string
-    }[]
-> {
-    const storage: Awaited<ReturnType<typeof collectFiles>> = []
-
-    const workspaceFoldersMapping = getWorkspaceFoldersByPrefixes(workspaceFolders)
-    const workspaceToPrefix = new Map<vscode.WorkspaceFolder, string>(
-        workspaceFoldersMapping === undefined
-            ? [[workspaceFolders[0], '']]
-            : Object.entries(workspaceFoldersMapping).map(value => [value[1], value[0]])
-    )
-    const prefixWithFolderPrefix = (folder: vscode.WorkspaceFolder, path: string) => {
-        const prefix = workspaceToPrefix.get(folder)
-        if (prefix === undefined) {
-            throw new ToolkitError(`Failed to find prefix for workspace folder ${folder.name}`)
-        }
-        return prefix === '' ? path : `${prefix}/${path}`
-    }
-
-    let totalSizeBytes = 0
-    for (const rootPath of sourcePaths) {
-        const allFiles = await vscode.workspace.findFiles(
-            new vscode.RelativePattern(rootPath, '**'),
-            getExcludePattern()
-        )
-        const files = respectGitIgnore ? await filterOutGitignoredFiles(rootPath, allFiles) : allFiles
-
-        for (const file of files) {
-            const relativePath = getWorkspaceRelativePath(file.fsPath, { workspaceFolders })
-            if (!relativePath) {
-                continue
-            }
-
-            const fileStat = await vscode.workspace.fs.stat(file)
-            if (totalSizeBytes + fileStat.size > maxRepoSizeBytes) {
-                throw new ContentLengthError()
-            }
-
-            const fileContent = await readFile(file)
-            if (fileContent === undefined) {
-                continue
-            }
-
-            // Now that we've read the file, increase our usage
-            totalSizeBytes += fileStat.size
-            storage.push({
-                workspaceFolder: relativePath.workspaceFolder,
-                relativeFilePath: relativePath.relativePath,
-                fileUri: file,
-                fileContent: fileContent,
-                zipFilePath: prefixWithFolderPrefix(relativePath.workspaceFolder, relativePath.relativePath),
-            })
-        }
-    }
-    return storage
-}
-
-const readFile = async (file: vscode.Uri) => {
-    try {
-        const fileContent = await SystemUtilities.readFile(file, new TextDecoder('utf8', { fatal: false }))
-        return fileContent
-    } catch (error) {
-        getLogger().debug(
-            `featureDev: Failed to read file ${file.fsPath} when collecting repository. Skipping the file`
-        )
-    }
-
-    return undefined
-}
 
 const getSha256 = (file: Buffer) => createHash('sha256').update(file).digest('base64')
 
@@ -191,123 +62,6 @@ export async function prepareRepoData(
     }
 }
 
-const workspaceFolderPrefixGuards = {
-    /**
-     * the maximum number of subfolders the method below takes into account when calculating a prefix
-     */
-    maximumFolderDepthConsidered: 500,
-    /**
-     * the maximum suffix that can be added to a folder prefix in case of full subfolder path matches
-     */
-    maximumFoldersWithMatchingSubfolders: 10_000,
-}
-
-/**
- * tries to determine the possible prefixes we will use for a given workspace folder in the zip file
- * We want to keep the folder names in the prefix, since they might convey useful information, for example
- * If both folders are just called cdk (no name specified for the ws folder), adding a prefix of cdk1 and cdk2 is much less context, than having app_cdk and canaries_cdk
- *
- * Input:
- * - packages/app/cdk
- * - packages/canaries/cdk
- * Output:
- * - {'app_cdk': packages/app/cdk, 'canaries_cdk': packages/canaries/cdk}
- *
- * @returns an object where workspace folders have a prefix, or undefined for single root workspace, as there is no mapping needed there
- */
-export function getWorkspaceFoldersByPrefixes(
-    folders: CurrentWsFolders
-): { [prefix: string]: vscode.WorkspaceFolder } | undefined {
-    if (folders.length <= 1) {
-        return undefined
-    }
-    let remainingWorkspaceFoldersToMap = folders.map(f => ({
-        folder: f,
-        preferredPrefixQueue: f.uri.fsPath
-            .split(path.sep)
-            .reverse()
-            .slice(0, workspaceFolderPrefixGuards.maximumFolderDepthConsidered)
-            .reduce(
-                (candidates, subDir) => {
-                    candidates.push(sanitizeFilename(path.join(subDir, candidates[candidates.length - 1])))
-                    return candidates
-                },
-                [f.name]
-            )
-            .reverse(),
-    }))
-    const results: ReturnType<typeof getWorkspaceFoldersByPrefixes> = {}
-
-    for (
-        let addParentFolderCount = 0;
-        remainingWorkspaceFoldersToMap.length > 0 &&
-        addParentFolderCount < workspaceFolderPrefixGuards.maximumFolderDepthConsidered;
-        addParentFolderCount++
-    ) {
-        const workspacesByPrefixes = remainingWorkspaceFoldersToMap.reduce((acc, wsFolder) => {
-            const prefix = wsFolder.preferredPrefixQueue.pop()
-            // this should never happen, as last candidates should be handled below, and the array starts non empty
-            if (prefix === undefined) {
-                throw new ToolkitError(
-                    `Encountered a folder with invalid prefix candidates (workspace folder ${wsFolder.folder.name})`
-                )
-            }
-            acc[prefix] = acc[prefix] ?? []
-            acc[prefix].push(wsFolder)
-            return acc
-        }, {} as { [key: string]: (typeof remainingWorkspaceFoldersToMap)[0][] })
-        remainingWorkspaceFoldersToMap = []
-        for (const [prefix, folders] of Object.entries(workspacesByPrefixes)) {
-            // if a folder has a unique prefix
-            if (folders.length === 1 && results[prefix] === undefined) {
-                results[prefix] = folders[0].folder
-                continue
-            }
-
-            // find the folders that do not have more parents
-            const foldersToSuffix: typeof folders = []
-            for (const folder of folders) {
-                if (folder.preferredPrefixQueue.length > 0) {
-                    remainingWorkspaceFoldersToMap.push(folder)
-                } else {
-                    foldersToSuffix.push(folder)
-                }
-            }
-            // for these last resort folders, suffix them with an increasing number until unique
-            if (foldersToSuffix.length === 1 && results[prefix] === undefined) {
-                results[prefix] = foldersToSuffix[0].folder
-            } else {
-                let suffix = 1
-                for (const folder of foldersToSuffix) {
-                    let newPrefix: string
-                    let safetyCounter = 0
-                    do {
-                        newPrefix = `${prefix}_${suffix}`
-                        suffix++
-                        safetyCounter++
-                    } while (
-                        results[newPrefix] !== undefined &&
-                        safetyCounter < workspaceFolderPrefixGuards.maximumFoldersWithMatchingSubfolders
-                    )
-                    if (safetyCounter >= workspaceFolderPrefixGuards.maximumFoldersWithMatchingSubfolders) {
-                        throw new ToolkitError(
-                            `Could not find a unique prefix for workspace folder ${folder.folder.name} in zip file.`
-                        )
-                    }
-                    results[newPrefix] = folder.folder
-                }
-            }
-        }
-    }
-    if (remainingWorkspaceFoldersToMap.length > 0) {
-        throw new ToolkitError(
-            `Could not find a unique prefix for workspace folder ${remainingWorkspaceFoldersToMap[0].folder.name} in zip file.`
-        )
-    }
-
-    return results
-}
-
 /**
  * gets the absolute path from a zip path
  * @param zipFilePath the path in the zip file

packages/core/src/amazonqFeatureDev/util/gitignore.ts

@@ -216,7 +216,7 @@ export const fileScanPayloadSizeLimitBytes = 200 * Math.pow(2, 10) // 200 KB
 
 export const fileScanUploadIntent = 'AUTOMATIC_FILE_SECURITY_SCAN'
 
-export const projectScanPayloadSizeLimitBytes = 5 * Math.pow(2, 30) // 5 GB
+export const projectScanPayloadSizeLimitBytes = 500 * Math.pow(2, 20) // 500 MB
 
 export const projectScanUploadIntent = 'FULL_PROJECT_SECURITY_SCAN'
 

packages/core/src/codewhisperer/models/constants.ts

@@ -10,11 +10,10 @@ import { getLogger } from '../../shared/logger'
 import * as CodeWhispererConstants from '../models/constants'
 import { ToolkitError } from '../../shared/errors'
 import { fsCommon } from '../../srcShared/fs'
-import { collectFiles } from '../../amazonqFeatureDev/util/files'
 import { getLoggerForScope } from '../service/securityScanHandler'
 import { runtimeLanguageContext } from './runtimeLanguageContext'
 import { CodewhispererLanguage } from '../../shared/telemetry/telemetry.gen'
-import { CurrentWsFolders } from '../../amazonqFeatureDev/types'
+import { CurrentWsFolders, collectFiles } from '../../shared/utilities/workspaceUtils'
 
 export interface ZipMetadata {
     rootDir: string
@@ -127,7 +126,12 @@ export class ZipUtil {
 
         const projectPaths = this.getProjectPaths()
 
-        const files = await collectFiles(projectPaths, vscode.workspace.workspaceFolders as CurrentWsFolders)
+        const files = await collectFiles(
+            projectPaths,
+            vscode.workspace.workspaceFolders as CurrentWsFolders,
+            true,
+            CodeWhispererConstants.projectScanPayloadSizeLimitBytes
+        )
         const languageCount = new Map<CodewhispererLanguage, number>()
         for (const file of files) {
             const isFileOpenAndDirty = this.isFileOpenAndDirty(file.fileUri)