Skip to content

Commit 5213237

Browse files
test(e2e): stabilize Amazon Q feature development tests (#6520)
## Problem Amazon Q feature dev e2e test was disabled due to flakiness. Issue: #6513 ## Solution - Modified test prompt to prevent no change required scenarios that were causing deadend. - Add `getSelectedTab` function in `qTestingFramework` to access newly created tab by quick action. --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent 2c12271 commit 5213237

File tree

2 files changed

+33
-19
lines changed

2 files changed

+33
-19
lines changed

packages/amazonq/test/e2e/amazonq/featureDev.test.ts

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,18 @@ import sinon from 'sinon'
99
import { registerAuthHook, using } from 'aws-core-vscode/test'
1010
import { loginToIdC } from './utils/setup'
1111
import { Messenger } from './framework/messenger'
12-
import { examples } from 'aws-core-vscode/amazonqFeatureDev'
1312
import { FollowUpTypes } from 'aws-core-vscode/amazonq'
1413
import { sleep } from 'aws-core-vscode/shared'
1514

1615
describe('Amazon Q Feature Dev', function () {
1716
let framework: qTestingFramework
1817
let tab: Messenger
1918

20-
const prompt = 'Add blank.txt file with empty content'
21-
const codegenApproachPrompt = `${prompt} and add a readme that describes the changes`
22-
const fileLevelAcceptPrompt = `${prompt} and add a license, and a contributing file`
19+
const prompt = 'Add current timestamp into blank.txt'
20+
const iteratePrompt = `Add a new section in readme to explain your change`
21+
const fileLevelAcceptPrompt = `${prompt} and ${iteratePrompt}`
22+
const informationCard =
23+
'After you provide a task, I will:\n1. Generate code based on your description and the code in your workspace\n2. Provide a list of suggestions for you to review and add to your workspace\n3. If needed, iterate based on your feedback\nTo learn more, visit the [user guide](https://docs.aws.amazon.com/amazonq/latest/qdeveloper-ug/software-dev.html)'
2324
const tooManyRequestsWaitTime = 100000
2425

2526
async function waitForText(text: string) {
@@ -144,34 +145,35 @@ describe('Amazon Q Feature Dev', function () {
144145
})
145146

146147
describe('/dev entry', () => {
147-
it('Clicks examples', async () => {
148-
const q = framework.createTab()
149-
q.addChatMessage({ command: '/dev' })
148+
before(async () => {
149+
tab = framework.createTab()
150+
tab.addChatMessage({ command: '/dev' }) // This would create a new tab for feature dev.
151+
tab = framework.getSelectedTab()
152+
})
153+
154+
it('should display information card', async () => {
150155
await retryIfRequired(
151156
async () => {
152-
await q.waitForChatFinishesLoading()
157+
await tab.waitForChatFinishesLoading()
153158
},
154159
() => {
155-
q.clickButton(FollowUpTypes.DevExamples)
156-
157-
const lastChatItems = q.getChatItems().pop()
158-
assert.deepStrictEqual(lastChatItems?.body, examples)
160+
const lastChatItems = tab.getChatItems().pop()
161+
assert.deepStrictEqual(lastChatItems?.body, informationCard)
159162
}
160163
)
161164
})
162165
})
163166

164-
// Disable failing tests while investigation. The tests are only failing in CI environments.
165-
describe.skip('/dev {msg} entry', async () => {
167+
describe('/dev {msg} entry', async () => {
166168
beforeEach(async function () {
169+
tab = framework.createTab()
167170
tab.addChatMessage({ command: '/dev', prompt })
171+
tab = framework.getSelectedTab()
168172
await retryIfRequired(
169173
async () => {
170174
await tab.waitForChatFinishesLoading()
171175
},
172-
() => {
173-
tab.addChatMessage({ prompt })
174-
}
176+
() => {}
175177
)
176178
})
177179

@@ -211,15 +213,17 @@ describe('Amazon Q Feature Dev', function () {
211213
})
212214
tab.clickButton(FollowUpTypes.ProvideFeedbackAndRegenerateCode)
213215
await tab.waitForChatFinishesLoading()
214-
await iterate(codegenApproachPrompt)
216+
await iterate(iteratePrompt)
215217
tab.clickButton(FollowUpTypes.InsertCode)
216218
await tab.waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
217219
})
218220
})
219221

220-
describe.skip('file-level accepts', async () => {
222+
describe('file-level accepts', async () => {
221223
beforeEach(async function () {
224+
tab = framework.createTab()
222225
tab.addChatMessage({ command: '/dev', prompt: fileLevelAcceptPrompt })
226+
tab = framework.getSelectedTab()
223227
await retryIfRequired(
224228
async () => {
225229
await tab.waitForChatFinishesLoading()

packages/amazonq/test/e2e/amazonq/framework/framework.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,16 @@ export class qTestingFramework {
105105
return Object.entries(tabs).map(([tabId]) => new Messenger(tabId, this.mynahUIProps, this.mynahUI))
106106
}
107107

108+
public getSelectedTab() {
109+
const selectedTabId = this.mynahUI.getSelectedTabId()
110+
const selectedTab = this.getTabs().find((tab) => tab.tabID === selectedTabId)
111+
112+
if (!selectedTab) {
113+
assert.fail('Selected tab not found')
114+
}
115+
return selectedTab
116+
}
117+
108118
public findTab(title: string) {
109119
return Object.values(this.getTabs()).find((tab) => tab.getStore().tabTitle === title)
110120
}

0 commit comments

Comments
 (0)