fix: move duplicate check to zipStream

Author: Hweinstock

packages/core/src/amazonq/util/files.ts

@@ -95,14 +95,7 @@ export async function prepareRepoData(
 
         let totalBytes = 0
         const ignoredExtensionMap = new Map<string, number>()
-        const addedFilePaths = new Set()
-
         for (const file of files) {
-            if (addedFilePaths.has(file.zipFilePath)) {
-                continue
-            }
-            addedFilePaths.add(file.zipFilePath)
-
             let fileSize
             try {
                 fileSize = (await fs.stat(file.fileUri)).size

packages/core/src/shared/utilities/zipStream.ts

@@ -57,6 +57,8 @@ export class ZipStream {
     private _filesToZip: [string, string][] = []
     private _filesBeingZipped: number = 0
     private _maxNumberOfFileStreams: number
+    private _targetFilesAdded: Set<string> = new Set()
+
     boundFileCompletionCallback: (computedSize: number) => Promise<void>
     boundFileStartCallback: (computedSize: number) => Promise<void>
 
@@ -123,6 +125,11 @@ export class ZipStream {
      * @param targetFilePath path to write data to in zip.
      */
     public writeFile(sourceFilePath: string, targetFilePath: string) {
+        if (this._targetFilesAdded.has(targetFilePath)) {
+            getLogger().debug(`Skipping duplicate file added to zip: ${targetFilePath}`)
+            return
+        }
+        this._targetFilesAdded.add(targetFilePath)
         // We use _numberOfFilesToStream to make sure we don't finalize too soon
         // (before the progress event has been fired for the last file)
         // The problem is that we can't rely on progress.entries.total,

packages/core/src/test/shared/utilities/zipStream.test.ts

@@ -11,6 +11,7 @@ import fs from '../../../shared/fs/fs'
 import crypto from 'crypto'
 // @ts-ignore
 import { BlobWriter } from '@zip.js/zip.js'
+import { assertLogsContain } from '../../globalSetup.test'
 
 describe('zipStream', function () {
     let tmpDir: string
@@ -67,6 +68,25 @@ describe('zipStream', function () {
 
         assert.strictEqual(result.sizeInBytes, (await fs.stat(zipPath)).size)
     })
+
+    it('only writes to target paths once', async function () {
+        const testFilePath = path.join(tmpDir, 'test.txt')
+        await fs.writeFile(testFilePath, 'foo bar')
+        const testFilePath2 = path.join(tmpDir, 'test2.txt')
+        await fs.writeFile(testFilePath2, 'foo bar')
+
+        const zipStream = new ZipStream()
+        zipStream.writeFile(testFilePath, 'file.txt')
+        zipStream.writeFile(testFilePath2, 'file.txt')
+        const result = await zipStream.finalize()
+
+        const zipBuffer = result.streamBuffer.getContents()
+        assert.ok(zipBuffer)
+        const zipEntries = await ZipStream.unzip(zipBuffer)
+
+        assert.strictEqual(zipEntries.length, 1)
+        assertLogsContain('duplicate file', false, 'debug')
+    })
 })
 
 async function verifyResult(result: ZipStreamResult, zipPath: string) {