Skip to content

Commit e9e2965

Browse files
authored
test(amazonq): generateZipTestGen is unreliable (#6290)
## Problem Flaky test `generateZipTestGen` sometimes fails with ``` 1) zipUtil generateZipTestGen Should generate zip for test generation successfully: Error: done() called multiple times in test <zipUtil generateZipTestGen Should generate zip for test generation successfully> of file /Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/amazonq/dist/test/unit/codewhisperer/util/zipUtil.test.js; in addition, done() received error: EntryNotFound (FileSystemError): Error: ENOENT: no such file or directory, scandir '/test/zip/utgRequiredArtifactsDir' at Function.e (/private/tmp/.vscode-test/vscode-darwin-arm64-1.83.0/Visual Studio Code.app/Contents/Resources/app/out/vs/workbench/api/node/extensionHostProcess.js:127:26740) at Object.readDirectory (/private/tmp/.vscode-test/vscode-darwin-arm64-1.83.0/Visual Studio Code.app/Contents/Resources/app/out/vs/workbench/api/node/extensionHostProcess.js:127:24744) at async processDirectory (/Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/core/src/codewhisperer/util/zipUtil.ts:190:29) at async ZipUtil.processMetadataDir (/Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/core/src/codewhisperer/util/zipUtil.ts:212:9) at async ZipUtil.zipProject (/Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/core/src/codewhisperer/util/zipUtil.ts:230:13) at async ZipUtil.generateZipTestGen (/Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/core/src/codewhisperer/util/zipUtil.ts:602:41) at async Context.<anonymous> (/Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/amazonq/test/unit/codewhisperer/util/zipUtil.test.ts:159:28) { code: 'FileNotFound' ``` Issue: #6160 ## Solution Based on the error, the test is trying to readdir on a mock dirpath even though the entire `zipProject` method should be stubbed. This can be confirmed by commenting out the following line: https://github.com/aws/aws-toolkit-vscode/blob/aa332da854c9d17ba258b459c622500911b1b0d9/packages/amazonq/test/unit/codewhisperer/util/zipUtil.test.ts#L135 which will give the same error ``` Error: ENOENT: no such file or directory, scandir '/test/zip/utgRequiredArtifactsDir' ``` So instead of stubbing which seems to be unreliable, we can let the test actually create the zip file. --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent 3d90772 commit e9e2965

File tree

1 file changed

+34
-89
lines changed

1 file changed

+34
-89
lines changed

packages/amazonq/test/unit/codewhisperer/util/zipUtil.test.ts

Lines changed: 34 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ import vscode from 'vscode'
88
import sinon from 'sinon'
99
import { join } from 'path'
1010
import { getTestWorkspaceFolder } from 'aws-core-vscode/test'
11-
import { CodeAnalysisScope, ZipUtil } from 'aws-core-vscode/codewhisperer'
11+
import { CodeAnalysisScope, CodeWhispererConstants, ZipUtil } from 'aws-core-vscode/codewhisperer'
1212
import { codeScanTruncDirPrefix } from 'aws-core-vscode/codewhisperer'
13-
import { ToolkitError } from 'aws-core-vscode/shared'
13+
import { tempDirPath, ToolkitError } from 'aws-core-vscode/shared'
1414
import { LspClient } from 'aws-core-vscode/amazonq'
1515
import { fs } from 'aws-core-vscode/shared'
1616
import path from 'path'
@@ -142,97 +142,58 @@ describe('zipUtil', function () {
142142

143143
describe('generateZipTestGen', function () {
144144
let zipUtil: ZipUtil
145-
let mockFs: sinon.SinonStubbedInstance<typeof fs>
146-
const projectPath = '/test/project'
147-
const zipDirPath = '/test/zip'
148-
const zipFilePath = '/test/zip/test.zip'
145+
let getZipDirPathStub: sinon.SinonStub
146+
let testTempDirPath: string
149147

150148
beforeEach(function () {
151149
zipUtil = new ZipUtil()
152-
mockFs = sinon.stub(fs)
153-
154-
const mockRepoMapPath = '/path/to/repoMapData.json'
155-
mockFs.exists.withArgs(mockRepoMapPath).resolves(true)
156-
sinon.stub(LspClient, 'instance').get(() => ({
157-
getRepoMapJSON: sinon.stub().resolves(mockRepoMapPath),
158-
}))
159-
160-
sinon.stub(zipUtil, 'getZipDirPath').returns(zipDirPath)
161-
sinon.stub(zipUtil as any, 'zipProject').resolves(zipFilePath)
150+
testTempDirPath = path.join(tempDirPath, CodeWhispererConstants.TestGenerationTruncDirPrefix)
151+
getZipDirPathStub = sinon.stub(zipUtil, 'getZipDirPath')
152+
getZipDirPathStub.callsFake(() => testTempDirPath)
162153
})
163154

164155
afterEach(function () {
165156
sinon.restore()
166157
})
167158

168-
it('Should generate zip for test generation successfully', async function () {
169-
mockFs.stat.resolves({
170-
type: vscode.FileType.File,
171-
size: 1000,
172-
ctime: Date.now(),
173-
mtime: Date.now(),
174-
} as vscode.FileStat)
175-
176-
mockFs.readFileBytes.resolves(Buffer.from('test content'))
159+
it('should generate zip for test generation successfully', async function () {
160+
const mkdirSpy = sinon.spy(fs, 'mkdir')
177161

178-
// Fix: Create a Set from the array
179-
zipUtil['_totalSize'] = 500
180-
zipUtil['_totalBuildSize'] = 200
181-
zipUtil['_totalLines'] = 100
182-
zipUtil['_language'] = 'typescript'
183-
zipUtil['_pickedSourceFiles'] = new Set(['file1.ts', 'file2.ts'])
162+
const result = await zipUtil.generateZipTestGen(appRoot, false)
184163

185-
const result = await zipUtil.generateZipTestGen(projectPath, false)
186-
187-
assert.ok(mockFs.mkdir.calledWith(path.join(zipDirPath, 'utgRequiredArtifactsDir')))
164+
assert.ok(mkdirSpy.calledWith(path.join(testTempDirPath, 'utgRequiredArtifactsDir')))
188165
assert.ok(
189-
mockFs.mkdir.calledWith(path.join(zipDirPath, 'utgRequiredArtifactsDir', 'buildAndExecuteLogDir'))
166+
mkdirSpy.calledWith(path.join(testTempDirPath, 'utgRequiredArtifactsDir', 'buildAndExecuteLogDir'))
190167
)
191-
assert.ok(mockFs.mkdir.calledWith(path.join(zipDirPath, 'utgRequiredArtifactsDir', 'repoMapData')))
192-
assert.ok(mockFs.mkdir.calledWith(path.join(zipDirPath, 'utgRequiredArtifactsDir', 'testCoverageDir')))
193-
194-
// assert.ok(
195-
// mockFs.copy.calledWith(
196-
// '/path/to/repoMapData.json',
197-
// path.join(zipDirPath, 'utgRequiredArtifactsDir', 'repoMapData', 'repoMapData.json')
198-
// )
199-
// )
200-
201-
assert.strictEqual(result.rootDir, zipDirPath)
202-
assert.strictEqual(result.zipFilePath, zipFilePath)
203-
assert.strictEqual(result.srcPayloadSizeInBytes, 500)
204-
assert.strictEqual(result.buildPayloadSizeInBytes, 200)
205-
assert.strictEqual(result.zipFileSizeInBytes, 1000)
206-
assert.strictEqual(result.lines, 100)
207-
assert.strictEqual(result.language, 'typescript')
208-
assert.deepStrictEqual(Array.from(result.scannedFiles), ['file1.ts', 'file2.ts'])
209-
})
210-
211-
// it('Should handle LSP client error', async function () {
212-
// // Override the default stub with one that rejects
213-
// sinon.stub(LspClient, 'instance').get(() => ({
214-
// getRepoMapJSON: sinon.stub().rejects(new Error('LSP error')),
215-
// }))
168+
assert.ok(mkdirSpy.calledWith(path.join(testTempDirPath, 'utgRequiredArtifactsDir', 'repoMapData')))
169+
assert.ok(mkdirSpy.calledWith(path.join(testTempDirPath, 'utgRequiredArtifactsDir', 'testCoverageDir')))
216170

217-
// await assert.rejects(() => zipUtil.generateZipTestGen(projectPath), /LSP error/)
218-
// })
171+
assert.strictEqual(result.rootDir, testTempDirPath)
172+
assert.strictEqual(result.zipFilePath, testTempDirPath + CodeWhispererConstants.codeScanZipExt)
173+
assert.ok(result.srcPayloadSizeInBytes > 0)
174+
assert.strictEqual(result.buildPayloadSizeInBytes, 0)
175+
assert.ok(result.zipFileSizeInBytes > 0)
176+
assert.strictEqual(result.lines, 150)
177+
assert.strictEqual(result.language, 'java')
178+
assert.strictEqual(result.scannedFiles.size, 4)
179+
})
219180

220181
it('Should handle file system errors during directory creation', async function () {
221182
sinon.stub(LspClient, 'instance').get(() => ({
222183
getRepoMapJSON: sinon.stub().resolves('{"mock": "data"}'),
223184
}))
224-
mockFs.mkdir.rejects(new Error('Directory creation failed'))
185+
sinon.stub(fs, 'mkdir').rejects(new Error('Directory creation failed'))
225186

226-
await assert.rejects(() => zipUtil.generateZipTestGen(projectPath, false), /Directory creation failed/)
187+
await assert.rejects(() => zipUtil.generateZipTestGen(appRoot, false), /Directory creation failed/)
227188
})
228189

229190
it('Should handle zip project errors', async function () {
230191
sinon.stub(LspClient, 'instance').get(() => ({
231192
getRepoMapJSON: sinon.stub().resolves('{"mock": "data"}'),
232193
}))
233-
;(zipUtil as any).zipProject.rejects(new Error('Zip failed'))
194+
sinon.stub(zipUtil, 'zipProject' as keyof ZipUtil).rejects(new Error('Zip failed'))
234195

235-
await assert.rejects(() => zipUtil.generateZipTestGen(projectPath, false), /Zip failed/)
196+
await assert.rejects(() => zipUtil.generateZipTestGen(appRoot, false), /Zip failed/)
236197
})
237198

238199
it('Should handle file copy to downloads folder error', async function () {
@@ -241,31 +202,15 @@ describe('zipUtil', function () {
241202
getRepoMapJSON: sinon.stub().resolves('{"mock": "data"}'),
242203
}))
243204

244-
// Mock file operations
245-
const mockFs = {
246-
mkdir: sinon.stub().resolves(),
247-
copy: sinon.stub().rejects(new Error('Copy failed')),
248-
exists: sinon.stub().resolves(true),
249-
stat: sinon.stub().resolves({
250-
type: vscode.FileType.File,
251-
size: 1000,
252-
ctime: Date.now(),
253-
mtime: Date.now(),
254-
} as vscode.FileStat),
255-
}
256-
257-
// Since the function now uses Promise.all for directory creation and file operations,
258-
// we need to ensure the mkdir succeeds but the copy fails
259-
fs.mkdir = mockFs.mkdir
260-
fs.copy = mockFs.copy
261-
fs.exists = mockFs.exists
262-
fs.stat = mockFs.stat
263-
264-
await assert.rejects(() => zipUtil.generateZipTestGen(projectPath, false), /Copy failed/)
205+
const mkdirSpy = sinon.spy(fs, 'mkdir')
206+
sinon.stub(fs, 'exists').resolves(true)
207+
sinon.stub(fs, 'copy').rejects(new Error('Copy failed'))
208+
209+
await assert.rejects(() => zipUtil.generateZipTestGen(appRoot, false), /Copy failed/)
265210

266211
// Verify mkdir was called for all directories
267-
assert(mockFs.mkdir.called, 'mkdir should have been called')
268-
assert.strictEqual(mockFs.mkdir.callCount, 4, 'mkdir should have been called 4 times')
212+
assert(mkdirSpy.called, 'mkdir should have been called')
213+
assert.strictEqual(mkdirSpy.callCount, 4, 'mkdir should have been called 4 times')
269214
})
270215
})
271216
})

0 commit comments

Comments
 (0)