Skip to content

Commit 9ff7c85

Browse files
KevinDing1KevinDing1
committed
test(amazonq): add retry mechanism for /doc e2e tests
1 parent 9c255dc commit 9ff7c85

File tree

1 file changed

+103
-50
lines changed

1 file changed

+103
-50
lines changed

‎packages/amazonq/test/e2e/amazonq/doc.test.ts‎

Lines changed: 103 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import { getTestWindow, registerAuthHook, toTextEditor, using } from 'aws-core-v
1010
import { loginToIdC } from './utils/setup'
1111
import { Messenger } from './framework/messenger'
1212
import { FollowUpTypes } from 'aws-core-vscode/amazonq'
13-
import { fs, i18n, workspaceUtils } from 'aws-core-vscode/shared'
13+
import { fs, i18n, sleep, workspaceUtils } from 'aws-core-vscode/shared'
1414
import {
1515
docGenerationProgressMessage,
1616
DocGenerationStep,
@@ -243,7 +243,47 @@ describe('Amazon Q Doc Generation', async function () {
243243
await tab.waitForChatFinishesLoading()
244244
},
245245
}
246-
246+
/**
247+
* Executes a test method with automatic retry capability for retryable errors.
248+
* Uses Promise.race to detect errors during test execution without hanging.
249+
*/
250+
async function retryIfRequired(testMethod: () => Promise<void>, maxAttempts: number = 3) {
251+
const errorMessages = {
252+
tooManyRequests: 'Too many requests',
253+
unexpectedError: 'Encountered an unexpected error when processing the request',
254+
}
255+
const hasRetryableError = () => {
256+
const lastTwoMessages = tab
257+
.getChatItems()
258+
.slice(-2)
259+
.map((item) => item.body)
260+
return lastTwoMessages.some(
261+
(body) => body?.includes(errorMessages.unexpectedError) || body?.includes(errorMessages.tooManyRequests)
262+
)
263+
}
264+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
265+
console.log(`Attempt ${attempt}/${maxAttempts}`)
266+
const errorDetectionPromise = new Promise((_, reject) => {
267+
const errorCheckInterval = setInterval(() => {
268+
if (hasRetryableError()) {
269+
clearInterval(errorCheckInterval)
270+
reject(new Error('Retryable error detected'))
271+
}
272+
}, 1000)
273+
})
274+
try {
275+
await Promise.race([testMethod(), errorDetectionPromise])
276+
return
277+
} catch (error) {
278+
if (attempt === maxAttempts) {
279+
assert.fail(`Test failed after ${maxAttempts} attempts`)
280+
}
281+
console.log(`Attempt ${attempt} failed, retrying...`)
282+
await sleep(1000 * attempt)
283+
await docUtils.setupTest()
284+
}
285+
}
286+
}
247287
before(async function () {
248288
/**
249289
* The tests are getting throttled, only run them on stable for now
@@ -332,65 +372,74 @@ describe('Amazon Q Doc Generation', async function () {
332372

333373
describe('README Creation', () => {
334374
let testProject: testProjectConfig
335-
this.retries(3)
336375
beforeEach(async function () {
337376
await docUtils.setupTest()
338377
testProject = docUtils.getRandomTestProject()
339378
})
340379

341380
it('Should create and save README in root folder when accepted', async () => {
342-
await docUtils.initializeDocOperation('create')
343-
await docUtils.executeDocumentationFlow('create')
344-
await docUtils.verifyResult(FollowUpTypes.AcceptChanges, rootReadmeFileUri, true)
381+
await retryIfRequired(async () => {
382+
await docUtils.initializeDocOperation('create')
383+
await docUtils.executeDocumentationFlow('create')
384+
await docUtils.verifyResult(FollowUpTypes.AcceptChanges, rootReadmeFileUri, true)
385+
})
345386
})
346387
it('Should create and save README in subfolder when accepted', async () => {
347-
await docUtils.initializeDocOperation('create')
348-
const readmeFileUri = await docUtils.handleFolderSelection(testProject)
349-
await docUtils.executeDocumentationFlow('create')
350-
await docUtils.verifyResult(FollowUpTypes.AcceptChanges, readmeFileUri, true)
388+
await retryIfRequired(async () => {
389+
await docUtils.initializeDocOperation('create')
390+
const readmeFileUri = await docUtils.handleFolderSelection(testProject)
391+
await docUtils.executeDocumentationFlow('create')
392+
await docUtils.verifyResult(FollowUpTypes.AcceptChanges, readmeFileUri, true)
393+
})
351394
})
352395

353396
it('Should discard README in subfolder when rejected', async () => {
354-
await docUtils.initializeDocOperation('create')
355-
const readmeFileUri = await docUtils.handleFolderSelection(testProject)
356-
await docUtils.executeDocumentationFlow('create')
357-
await docUtils.verifyResult(FollowUpTypes.RejectChanges, readmeFileUri, false)
397+
await retryIfRequired(async () => {
398+
await docUtils.initializeDocOperation('create')
399+
const readmeFileUri = await docUtils.handleFolderSelection(testProject)
400+
await docUtils.executeDocumentationFlow('create')
401+
await docUtils.verifyResult(FollowUpTypes.RejectChanges, readmeFileUri, false)
402+
})
358403
})
359404
})
360405

361406
describe('README Editing', () => {
362-
this.retries(3)
363407
beforeEach(async function () {
364408
await docUtils.setupTest()
365409
})
366410

367411
it('Should apply specific content changes when requested', async () => {
368-
await docUtils.initializeDocOperation('edit')
369-
await docUtils.executeDocumentationFlow('edit', 'remove the repository structure section')
370-
await docUtils.verifyResult(FollowUpTypes.AcceptChanges, rootReadmeFileUri, true)
412+
await retryIfRequired(async () => {
413+
await docUtils.initializeDocOperation('edit')
414+
await docUtils.executeDocumentationFlow('edit', 'remove the repository structure section')
415+
await docUtils.verifyResult(FollowUpTypes.AcceptChanges, rootReadmeFileUri, true)
416+
})
371417
})
372418

373419
it('Should handle unrelated prompts with appropriate error message', async () => {
374-
await docUtils.initializeDocOperation('edit')
375-
await tab.waitForButtons([FollowUpTypes.ProceedFolderSelection])
376-
tab.clickButton(FollowUpTypes.ProceedFolderSelection)
377-
tab.addChatMessage({ prompt: 'tell me about the weather' })
378-
await tab.waitForEvent(() =>
379-
tab.getChatItems().some(({ body }) => body?.startsWith(i18n('AWS.amazonq.doc.error.promptUnrelated')))
380-
)
381-
await tab.waitForEvent(() => {
382-
const store = tab.getStore()
383-
return (
384-
!store.promptInputDisabledState &&
385-
store.promptInputPlaceholder === i18n('AWS.amazonq.doc.placeholder.editReadme')
420+
await retryIfRequired(async () => {
421+
await docUtils.initializeDocOperation('edit')
422+
await tab.waitForButtons([FollowUpTypes.ProceedFolderSelection])
423+
tab.clickButton(FollowUpTypes.ProceedFolderSelection)
424+
tab.addChatMessage({ prompt: 'tell me about the weather' })
425+
await tab.waitForEvent(() =>
426+
tab
427+
.getChatItems()
428+
.some(({ body }) => body?.startsWith(i18n('AWS.amazonq.doc.error.promptUnrelated')))
386429
)
430+
await tab.waitForEvent(() => {
431+
const store = tab.getStore()
432+
return (
433+
!store.promptInputDisabledState &&
434+
store.promptInputPlaceholder === i18n('AWS.amazonq.doc.placeholder.editReadme')
435+
)
436+
})
387437
})
388438
})
389439
})
390440
describe('README Updates', () => {
391441
let testProject: testProjectConfig
392442
let mockFileUri: vscode.Uri
393-
this.retries(3)
394443

395444
beforeEach(async function () {
396445
await docUtils.setupTest()
@@ -405,31 +454,35 @@ describe('Amazon Q Doc Generation', async function () {
405454

406455
it('Should update README with code change in subfolder', async () => {
407456
mockFileUri = await docUtils.prepareMockFile(testProject)
408-
await docUtils.initializeDocOperation('update')
409-
const readmeFileUri = await docUtils.handleFolderSelection(testProject)
410-
await docUtils.executeDocumentationFlow('update')
411-
await docUtils.verifyResult(FollowUpTypes.AcceptChanges, readmeFileUri, true)
457+
await retryIfRequired(async () => {
458+
await docUtils.initializeDocOperation('update')
459+
const readmeFileUri = await docUtils.handleFolderSelection(testProject)
460+
await docUtils.executeDocumentationFlow('update')
461+
await docUtils.verifyResult(FollowUpTypes.AcceptChanges, readmeFileUri, true)
462+
})
412463
})
413464
it('Should update root README and incorporate additional changes', async () => {
414465
// Cleanup any existing README
415466
await fs.delete(rootReadmeFileUri, { force: true })
416467
mockFileUri = await docUtils.prepareMockFile(testProject)
417-
await docUtils.initializeDocOperation('update')
418-
await docUtils.executeDocumentationFlow('update')
419-
tab.clickButton(FollowUpTypes.MakeChanges)
420-
tab.addChatMessage({ prompt: 'remove the repository structure section' })
421-
422-
await tab.waitForText(docGenerationProgressMessage(DocGenerationStep.SUMMARIZING_FILES, Mode.SYNC))
423-
await tab.waitForText(
424-
`${docGenerationSuccessMessage(Mode.SYNC)} ${i18n('AWS.amazonq.doc.answer.codeResult')}`
425-
)
426-
await tab.waitForButtons([
427-
FollowUpTypes.AcceptChanges,
428-
FollowUpTypes.MakeChanges,
429-
FollowUpTypes.RejectChanges,
430-
])
468+
await retryIfRequired(async () => {
469+
await docUtils.initializeDocOperation('update')
470+
await docUtils.executeDocumentationFlow('update')
471+
tab.clickButton(FollowUpTypes.MakeChanges)
472+
tab.addChatMessage({ prompt: 'remove the repository structure section' })
473+
474+
await tab.waitForText(docGenerationProgressMessage(DocGenerationStep.SUMMARIZING_FILES, Mode.SYNC))
475+
await tab.waitForText(
476+
`${docGenerationSuccessMessage(Mode.SYNC)} ${i18n('AWS.amazonq.doc.answer.codeResult')}`
477+
)
478+
await tab.waitForButtons([
479+
FollowUpTypes.AcceptChanges,
480+
FollowUpTypes.MakeChanges,
481+
FollowUpTypes.RejectChanges,
482+
])
431483

432-
await docUtils.verifyResult(FollowUpTypes.AcceptChanges, rootReadmeFileUri, true)
484+
await docUtils.verifyResult(FollowUpTypes.AcceptChanges, rootReadmeFileUri, true)
485+
})
433486
})
434487
})
435488
})

0 commit comments

Comments
 (0)