Allow including binary files when execution is enabled

Author: Hamed Soleimani

packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts

@@ -16,23 +16,45 @@ import { MetricName, Span } from 'aws-core-vscode/telemetry'
 import sinon from 'sinon'
 import { CodeWhispererSettings } from 'aws-core-vscode/codewhisperer'
 
-const testDevfilePrepareRepo = async (expectedRepoSize: number, devfileEnabled: boolean) => {
+import AdmZip from 'adm-zip'
+
+const testDevfilePrepareRepo = async (devfileEnabled: boolean) => {
+    const files: Record<string, string> = {
+        'file.md': 'test content',
+        // only include when execution is enabled
+        'devfile.yaml': 'test',
+        // .git folder is always dropped (because of vscode global exclude rules)
+        '.git/ref': '####',
+        // .gitignore should always be included
+        '.gitignore': 'node_models/*',
+        // non code files only when dev execution is enabled
+        'abc.jar': 'jar-content',
+        'data/logo.ico': 'binary-content',
+    }
     const folder = await TestFolder.create()
-    await folder.write('devfile.yaml', 'test')
-    await folder.write('file.md', 'test content')
+
+    for (const [fileName, content] of Object.entries(files)) {
+        await folder.write(fileName, content)
+    }
+
+    const expectedFiles = !devfileEnabled
+        ? ['./file.md', './.gitignore']
+        : ['./devfile.yaml', './file.md', './.gitignore', './abc.jar', 'data/logo.ico']
+
     const workspace = getWorkspaceFolder(folder.path)
     sinon
         .stub(CodeWhispererSettings.instance, 'getDevCommandWorkspaceConfigurations')
         .returns(devfileEnabled ? { [workspace.uri.fsPath]: true } : {})
 
-    await testPrepareRepoData(workspace, expectedRepoSize)
+    await testPrepareRepoData(workspace, expectedFiles)
 }
 
 const testPrepareRepoData = async (
     workspace: vscode.WorkspaceFolder,
-    expectedRepoSize: number,
+    expectedFiles: string[],
     expectedTelemetryMetrics?: Array<{ metricName: MetricName; value: any }>
 ) => {
+    expectedFiles.sort((a, b) => a.localeCompare(b))
     const telemetry = new TelemetryHelper()
     const result = await prepareRepoData([workspace.uri.fsPath], [workspace], telemetry, {
         record: () => {},
@@ -41,13 +63,18 @@ const testPrepareRepoData = async (
     assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true)
     // checksum is not the same across different test executions because some unique random folder names are generated
     assert.strictEqual(result.zipFileChecksum.length, 44)
-    assert.strictEqual(telemetry.repositorySize, expectedRepoSize)
 
     if (expectedTelemetryMetrics) {
-        expectedTelemetryMetrics.forEach((metric) => {
+        for (const metric of expectedTelemetryMetrics) {
             assertTelemetry(metric.metricName, metric.value)
-        })
+        }
     }
+
+    // Unzip the buffer and compare the entry names
+    const zip = new AdmZip(result.zipFileBuffer)
+    const actualZipEntries = zip.getEntries().map((entry) => entry.entryName)
+    actualZipEntries.sort((a, b) => a.localeCompare(b))
+    assert.deepStrictEqual(actualZipEntries, expectedFiles)
 }
 
 describe('file utils', () => {
@@ -62,25 +89,27 @@ describe('file utils', () => {
             await folder.write('file2.md', 'test content')
             const workspace = getWorkspaceFolder(folder.path)
 
-            await testPrepareRepoData(workspace, 24)
+            await testPrepareRepoData(workspace, ['./file1.md', './file2.md'])
         })
 
         it('prepareRepoData ignores denied file extensions', async function () {
             const folder = await TestFolder.create()
             await folder.write('file.mp4', 'test content')
             const workspace = getWorkspaceFolder(folder.path)
 
-            await testPrepareRepoData(workspace, 0, [
-                { metricName: 'amazonq_bundleExtensionIgnored', value: { filenameExt: 'mp4', count: 1 } },
-            ])
+            await testPrepareRepoData(
+                workspace,
+                [],
+                [{ metricName: 'amazonq_bundleExtensionIgnored', value: { filenameExt: 'mp4', count: 1 } }]
+            )
         })
 
         it('should ignore devfile.yaml when setting is disabled', async function () {
-            await testDevfilePrepareRepo(12, false)
+            await testDevfilePrepareRepo(false)
         })
 
         it('should include devfile.yaml when setting is enabled', async function () {
-            await testDevfilePrepareRepo(16, true)
+            await testDevfilePrepareRepo(true)
         })
 
         // Test the logic that allows the customer to modify root source folder

packages/core/src/amazonqFeatureDev/util/files.ts

@@ -40,9 +40,10 @@ export async function prepareRepoData(
     zip: AdmZip = new AdmZip()
 ) {
     try {
-        const files = await collectFiles(repoRootPaths, workspaceFolders, true, maxRepoSizeBytes)
         const devCommandWorkspaceConfigurations = CodeWhispererSettings.instance.getDevCommandWorkspaceConfigurations()
         const useAutoBuildFeature = devCommandWorkspaceConfigurations[repoRootPaths[0]] ?? false
+        // We only respect gitignore file rules if useAutoBuildFeature is on, this is to avoid dropping necessary files for building the code (e.g. png files imported in js code)
+        const files = await collectFiles(repoRootPaths, workspaceFolders, true, maxRepoSizeBytes, !useAutoBuildFeature)
 
         let totalBytes = 0
         const ignoredExtensionMap = new Map<string, number>()
@@ -59,10 +60,10 @@ export async function prepareRepoData(
                 throw error
             }
             const isCodeFile_ = isCodeFile(file.relativeFilePath)
-            // exclude user's devfile if `useAutoBuildFeature` is set to false
-            const excludeDevFile = useAutoBuildFeature ? false : file.relativeFilePath === 'devfile.yaml'
-
-            if (fileSize >= maxFileSizeBytes || !isCodeFile_ || excludeDevFile) {
+            const isDevFile = file.relativeFilePath === 'devfile.yaml'
+            // When useAutoBuildFeature is on, only respect the gitignore rules filtered earlier and apply the size limit, otherwise, exclude all non code files and gitignore files
+            const isNonCodeFileAndIgnored = useAutoBuildFeature ? false : !isCodeFile_ || isDevFile
+            if (fileSize >= maxFileSizeBytes || isNonCodeFileAndIgnored) {
                 if (!isCodeFile_) {
                     const re = /(?:\.([^.]+))?$/
                     const extensionArray = re.exec(file.relativeFilePath)

packages/core/src/shared/filetypes.ts

@@ -233,7 +233,6 @@ export const codefileExtensions = new Set([
     '.idl',
     '.ini',
     '.io',
-    '.jar',
     '.java',
     '.jl',
     '.js',
@@ -362,7 +361,7 @@ export const codefileExtensions = new Set([
 ])
 
 // Code file names without an extension
-export const codefileNames = new Set(['Dockerfile', 'Dockerfile.build', 'gradlew', 'mvnw'])
+export const codefileNames = new Set(['Dockerfile', 'Dockerfile.build', 'gradlew', 'mvnw', '.gitignore'])
 
 // Build file names
 export const buildfileNames = new Set(['gradle/wrapper/gradle-wrapper.jar'])

packages/core/src/shared/utilities/workspaceUtils.ts

@@ -268,7 +268,7 @@ export function checkUnsavedChanges(): boolean {
     return vscode.workspace.textDocuments.some((doc) => doc.isDirty)
 }
 
-export function getExcludePattern(additionalPatterns: string[] = []) {
+export function getExcludePattern(additionalPatterns: string[] = [], nonGlobalExtraPatterns: boolean = true) {
     const globAlwaysExcludedDirs = getGlobDirExcludedPatterns().map((pattern) => `**/${pattern}/*`)
     const extraPatterns = [
         '**/package-lock.json',
@@ -290,19 +290,28 @@ export function getExcludePattern(additionalPatterns: string[] = []) {
         '**/License.md',
         '**/LICENSE.md',
     ]
-    const allPatterns = [...globAlwaysExcludedDirs, ...extraPatterns, ...additionalPatterns]
+    const allPatterns = [
+        ...globAlwaysExcludedDirs,
+        ...(nonGlobalExtraPatterns ? extraPatterns : []),
+        ...additionalPatterns,
+    ]
     return `{${allPatterns.join(',')}}`
 }
 
 /**
  * @param rootPath root folder to look for .gitignore files
+ * @param addExtraPatterns whether to add extra exclude patterns even if not in gitignore
  * @returns list of glob patterns extracted from .gitignore
  * These patterns are compatible with vscode exclude patterns
  */
-async function filterOutGitignoredFiles(rootPath: string, files: vscode.Uri[]): Promise<vscode.Uri[]> {
+async function filterOutGitignoredFiles(
+    rootPath: string,
+    files: vscode.Uri[],
+    addExtraPatterns: boolean = true
+): Promise<vscode.Uri[]> {
     const gitIgnoreFiles = await vscode.workspace.findFiles(
         new vscode.RelativePattern(rootPath, '**/.gitignore'),
-        getExcludePattern()
+        getExcludePattern([], addExtraPatterns)
     )
     const gitIgnoreFilter = await GitIgnoreFilter.build(gitIgnoreFiles)
     return gitIgnoreFilter.filterFiles(files)
@@ -313,13 +322,15 @@ async function filterOutGitignoredFiles(rootPath: string, files: vscode.Uri[]):
  * @param sourcePaths the paths where collection starts
  * @param workspaceFolders the current workspace folders opened
  * @param respectGitIgnore whether to respect gitignore file
+ * @param addExtraIgnorePatterns whether to add extra exclude patterns even if not in gitignore
  * @returns all matched files
  */
 export async function collectFiles(
     sourcePaths: string[],
     workspaceFolders: CurrentWsFolders,
     respectGitIgnore: boolean = true,
-    maxSize = 200 * 1024 * 1024 // 200 MB
+    maxSize = 200 * 1024 * 1024, // 200 MB
+    addExtraIgnorePatterns: boolean = true
 ): Promise<
     {
         workspaceFolder: vscode.WorkspaceFolder
@@ -356,10 +367,12 @@ export async function collectFiles(
     for (const rootPath of sourcePaths) {
         const allFiles = await vscode.workspace.findFiles(
             new vscode.RelativePattern(rootPath, '**'),
-            getExcludePattern()
+            getExcludePattern([], addExtraIgnorePatterns)
         )
 
-        const files = respectGitIgnore ? await filterOutGitignoredFiles(rootPath, allFiles) : allFiles
+        const files = respectGitIgnore
+            ? await filterOutGitignoredFiles(rootPath, allFiles, addExtraIgnorePatterns)
+            : allFiles
 
         for (const file of files) {
             const relativePath = getWorkspaceRelativePath(file.fsPath, { workspaceFolders })