diff --git a/packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts b/packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts index e18521de84e..a4e4a7b462c 100644 --- a/packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts +++ b/packages/amazonq/test/unit/amazonqFeatureDev/util/files.test.ts @@ -16,23 +16,45 @@ import { MetricName, Span } from 'aws-core-vscode/telemetry' import sinon from 'sinon' import { CodeWhispererSettings } from 'aws-core-vscode/codewhisperer' -const testDevfilePrepareRepo = async (expectedRepoSize: number, devfileEnabled: boolean) => { +import AdmZip from 'adm-zip' + +const testDevfilePrepareRepo = async (devfileEnabled: boolean) => { + const files: Record = { + 'file.md': 'test content', + // only include when execution is enabled + 'devfile.yaml': 'test', + // .git folder is always dropped (because of vscode global exclude rules) + '.git/ref': '####', + // .gitignore should always be included + '.gitignore': 'node_models/*', + // non code files only when dev execution is enabled + 'abc.jar': 'jar-content', + 'data/logo.ico': 'binary-content', + } const folder = await TestFolder.create() - await folder.write('devfile.yaml', 'test') - await folder.write('file.md', 'test content') + + for (const [fileName, content] of Object.entries(files)) { + await folder.write(fileName, content) + } + + const expectedFiles = !devfileEnabled + ? ['./file.md', './.gitignore'] + : ['./devfile.yaml', './file.md', './.gitignore', './abc.jar', 'data/logo.ico'] + const workspace = getWorkspaceFolder(folder.path) sinon .stub(CodeWhispererSettings.instance, 'getDevCommandWorkspaceConfigurations') .returns(devfileEnabled ? { [workspace.uri.fsPath]: true } : {}) - await testPrepareRepoData(workspace, expectedRepoSize) + await testPrepareRepoData(workspace, expectedFiles) } const testPrepareRepoData = async ( workspace: vscode.WorkspaceFolder, - expectedRepoSize: number, + expectedFiles: string[], expectedTelemetryMetrics?: Array<{ metricName: MetricName; value: any }> ) => { + expectedFiles.sort((a, b) => a.localeCompare(b)) const telemetry = new TelemetryHelper() const result = await prepareRepoData([workspace.uri.fsPath], [workspace], telemetry, { record: () => {}, @@ -41,13 +63,18 @@ const testPrepareRepoData = async ( assert.strictEqual(Buffer.isBuffer(result.zipFileBuffer), true) // checksum is not the same across different test executions because some unique random folder names are generated assert.strictEqual(result.zipFileChecksum.length, 44) - assert.strictEqual(telemetry.repositorySize, expectedRepoSize) if (expectedTelemetryMetrics) { - expectedTelemetryMetrics.forEach((metric) => { + for (const metric of expectedTelemetryMetrics) { assertTelemetry(metric.metricName, metric.value) - }) + } } + + // Unzip the buffer and compare the entry names + const zip = new AdmZip(result.zipFileBuffer) + const actualZipEntries = zip.getEntries().map((entry) => entry.entryName) + actualZipEntries.sort((a, b) => a.localeCompare(b)) + assert.deepStrictEqual(actualZipEntries, expectedFiles) } describe('file utils', () => { @@ -62,7 +89,7 @@ describe('file utils', () => { await folder.write('file2.md', 'test content') const workspace = getWorkspaceFolder(folder.path) - await testPrepareRepoData(workspace, 24) + await testPrepareRepoData(workspace, ['./file1.md', './file2.md']) }) it('prepareRepoData ignores denied file extensions', async function () { @@ -70,17 +97,19 @@ describe('file utils', () => { await folder.write('file.mp4', 'test content') const workspace = getWorkspaceFolder(folder.path) - await testPrepareRepoData(workspace, 0, [ - { metricName: 'amazonq_bundleExtensionIgnored', value: { filenameExt: 'mp4', count: 1 } }, - ]) + await testPrepareRepoData( + workspace, + [], + [{ metricName: 'amazonq_bundleExtensionIgnored', value: { filenameExt: 'mp4', count: 1 } }] + ) }) it('should ignore devfile.yaml when setting is disabled', async function () { - await testDevfilePrepareRepo(12, false) + await testDevfilePrepareRepo(false) }) it('should include devfile.yaml when setting is enabled', async function () { - await testDevfilePrepareRepo(16, true) + await testDevfilePrepareRepo(true) }) // Test the logic that allows the customer to modify root source folder diff --git a/packages/core/src/amazonqFeatureDev/util/files.ts b/packages/core/src/amazonqFeatureDev/util/files.ts index 8c74f419493..da355085ecd 100644 --- a/packages/core/src/amazonqFeatureDev/util/files.ts +++ b/packages/core/src/amazonqFeatureDev/util/files.ts @@ -40,9 +40,10 @@ export async function prepareRepoData( zip: AdmZip = new AdmZip() ) { try { - const files = await collectFiles(repoRootPaths, workspaceFolders, true, maxRepoSizeBytes) const devCommandWorkspaceConfigurations = CodeWhispererSettings.instance.getDevCommandWorkspaceConfigurations() const useAutoBuildFeature = devCommandWorkspaceConfigurations[repoRootPaths[0]] ?? false + // We only respect gitignore file rules if useAutoBuildFeature is on, this is to avoid dropping necessary files for building the code (e.g. png files imported in js code) + const files = await collectFiles(repoRootPaths, workspaceFolders, true, maxRepoSizeBytes, !useAutoBuildFeature) let totalBytes = 0 const ignoredExtensionMap = new Map() @@ -59,10 +60,10 @@ export async function prepareRepoData( throw error } const isCodeFile_ = isCodeFile(file.relativeFilePath) - // exclude user's devfile if `useAutoBuildFeature` is set to false - const excludeDevFile = useAutoBuildFeature ? false : file.relativeFilePath === 'devfile.yaml' - - if (fileSize >= maxFileSizeBytes || !isCodeFile_ || excludeDevFile) { + const isDevFile = file.relativeFilePath === 'devfile.yaml' + // When useAutoBuildFeature is on, only respect the gitignore rules filtered earlier and apply the size limit, otherwise, exclude all non code files and gitignore files + const isNonCodeFileAndIgnored = useAutoBuildFeature ? false : !isCodeFile_ || isDevFile + if (fileSize >= maxFileSizeBytes || isNonCodeFileAndIgnored) { if (!isCodeFile_) { const re = /(?:\.([^.]+))?$/ const extensionArray = re.exec(file.relativeFilePath) diff --git a/packages/core/src/shared/filetypes.ts b/packages/core/src/shared/filetypes.ts index 34d6f94f599..843397e6a17 100644 --- a/packages/core/src/shared/filetypes.ts +++ b/packages/core/src/shared/filetypes.ts @@ -233,7 +233,6 @@ export const codefileExtensions = new Set([ '.idl', '.ini', '.io', - '.jar', '.java', '.jl', '.js', @@ -362,17 +361,10 @@ export const codefileExtensions = new Set([ ]) // Code file names without an extension -export const codefileNames = new Set(['Dockerfile', 'Dockerfile.build', 'gradlew', 'mvnw']) - -// Build file names -export const buildfileNames = new Set(['gradle/wrapper/gradle-wrapper.jar']) +export const codefileNames = new Set(['Dockerfile', 'Dockerfile.build', 'gradlew', 'mvnw', '.gitignore']) /** Returns true if `filename` is a code file. */ export function isCodeFile(filename: string): boolean { const ext = path.extname(filename).toLowerCase() - return ( - codefileExtensions.has(ext) || - codefileNames.has(path.basename(filename)) || - buildfileNames.has(path.basename(filename)) - ) + return codefileExtensions.has(ext) || codefileNames.has(path.basename(filename)) } diff --git a/packages/core/src/shared/utilities/workspaceUtils.ts b/packages/core/src/shared/utilities/workspaceUtils.ts index 3894c3b56ff..c650b886016 100644 --- a/packages/core/src/shared/utilities/workspaceUtils.ts +++ b/packages/core/src/shared/utilities/workspaceUtils.ts @@ -268,7 +268,7 @@ export function checkUnsavedChanges(): boolean { return vscode.workspace.textDocuments.some((doc) => doc.isDirty) } -export function getExcludePattern(additionalPatterns: string[] = []) { +export function getExcludePattern(defaultExcludePatterns: boolean = true) { const globAlwaysExcludedDirs = getGlobDirExcludedPatterns().map((pattern) => `**/${pattern}/*`) const extraPatterns = [ '**/package-lock.json', @@ -290,19 +290,24 @@ export function getExcludePattern(additionalPatterns: string[] = []) { '**/License.md', '**/LICENSE.md', ] - const allPatterns = [...globAlwaysExcludedDirs, ...extraPatterns, ...additionalPatterns] + const allPatterns = [...globAlwaysExcludedDirs, ...(defaultExcludePatterns ? extraPatterns : [])] return `{${allPatterns.join(',')}}` } /** * @param rootPath root folder to look for .gitignore files + * @param addExtraPatterns whether to add extra exclude patterns even if not in gitignore * @returns list of glob patterns extracted from .gitignore * These patterns are compatible with vscode exclude patterns */ -async function filterOutGitignoredFiles(rootPath: string, files: vscode.Uri[]): Promise { +async function filterOutGitignoredFiles( + rootPath: string, + files: vscode.Uri[], + defaultExcludePatterns: boolean = true +): Promise { const gitIgnoreFiles = await vscode.workspace.findFiles( new vscode.RelativePattern(rootPath, '**/.gitignore'), - getExcludePattern() + getExcludePattern(defaultExcludePatterns) ) const gitIgnoreFilter = await GitIgnoreFilter.build(gitIgnoreFiles) return gitIgnoreFilter.filterFiles(files) @@ -313,13 +318,15 @@ async function filterOutGitignoredFiles(rootPath: string, files: vscode.Uri[]): * @param sourcePaths the paths where collection starts * @param workspaceFolders the current workspace folders opened * @param respectGitIgnore whether to respect gitignore file + * @param addExtraIgnorePatterns whether to add extra exclude patterns even if not in gitignore * @returns all matched files */ export async function collectFiles( sourcePaths: string[], workspaceFolders: CurrentWsFolders, respectGitIgnore: boolean = true, - maxSize = 200 * 1024 * 1024 // 200 MB + maxSize = 200 * 1024 * 1024, // 200 MB + defaultExcludePatterns: boolean = true ): Promise< { workspaceFolder: vscode.WorkspaceFolder @@ -356,10 +363,12 @@ export async function collectFiles( for (const rootPath of sourcePaths) { const allFiles = await vscode.workspace.findFiles( new vscode.RelativePattern(rootPath, '**'), - getExcludePattern() + getExcludePattern(defaultExcludePatterns) ) - const files = respectGitIgnore ? await filterOutGitignoredFiles(rootPath, allFiles) : allFiles + const files = respectGitIgnore + ? await filterOutGitignoredFiles(rootPath, allFiles, defaultExcludePatterns) + : allFiles for (const file of files) { const relativePath = getWorkspaceRelativePath(file.fsPath, { workspaceFolders }) diff --git a/packages/core/src/test/shared/filetypes.test.ts b/packages/core/src/test/shared/filetypes.test.ts index 2ae6e82ff8c..26e12631419 100644 --- a/packages/core/src/test/shared/filetypes.test.ts +++ b/packages/core/src/test/shared/filetypes.test.ts @@ -160,7 +160,6 @@ describe('isCodeFile', () => { 'mvnw', 'build.gradle', 'gradle/wrapper/gradle-wrapper.properties', - 'gradle/wrapper/gradle-wrapper.jar', ] for (const codeFilePath of codeFiles) { assert.strictEqual(isCodeFile(codeFilePath), true)