test(amazonq): E2E test for /doc (aws#6215)

avi-alpert · karanA-aws · commit 11b6c4f4d529 · 2025-01-17T14:45:23.000-08:00
Add an E2E test for Creating a README using /doc Moved reusable functions out of featureDev.test.ts to shared test framework --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). License: I confirm that my contribution is made under the terms of the Apache 2.0 license.
diff --git a/packages/amazonq/test/e2e/amazonq/doc.test.ts b/packages/amazonq/test/e2e/amazonq/doc.test.ts
@@ -0,0 +1,113 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import assert from 'assert'
+import { qTestingFramework } from './framework/framework'
+import sinon from 'sinon'
+import { registerAuthHook, using } from 'aws-core-vscode/test'
+import { loginToIdC } from './utils/setup'
+import { Messenger } from './framework/messenger'
+import { FollowUpTypes } from 'aws-core-vscode/amazonq'
+import { i18n } from 'aws-core-vscode/shared'
+import { docGenerationProgressMessage, DocGenerationStep, Mode } from 'aws-core-vscode/amazonqDoc'
+
+describe('Amazon Q Doc', async function () {
+    let framework: qTestingFramework
+    let tab: Messenger
+
+    before(async function () {
+        /**
+         * The tests are getting throttled, only run them on stable for now
+         *
+         * TODO: Re-enable for all versions once the backend can handle them
+         */
+        const testVersion = process.env['VSCODE_TEST_VERSION']
+        if (testVersion && testVersion !== 'stable') {
+            this.skip()
+        }
+
+        await using(registerAuthHook('amazonq-test-account'), async () => {
+            await loginToIdC()
+        })
+    })
+
+    beforeEach(() => {
+        registerAuthHook('amazonq-test-account')
+        framework = new qTestingFramework('doc', true, [])
+        tab = framework.createTab()
+    })
+
+    afterEach(() => {
+        framework.removeTab(tab.tabID)
+        framework.dispose()
+        sinon.restore()
+    })
+
+    describe('Quick action availability', () => {
+        it('Shows /doc when doc generation is enabled', async () => {
+            const command = tab.findCommand('/doc')
+            if (!command.length) {
+                assert.fail('Could not find command')
+            }
+
+            if (command.length > 1) {
+                assert.fail('Found too many commands with the name /doc')
+            }
+        })
+
+        it('Does NOT show /doc when doc generation is NOT enabled', () => {
+            // The beforeEach registers a framework which accepts requests. If we don't dispose before building a new one we have duplicate messages
+            framework.dispose()
+            framework = new qTestingFramework('doc', false, [])
+            const tab = framework.createTab()
+            const command = tab.findCommand('/doc')
+            if (command.length > 0) {
+                assert.fail('Found command when it should not have been found')
+            }
+        })
+    })
+
+    describe('/doc entry', () => {
+        beforeEach(async function () {
+            tab.addChatMessage({ command: '/doc' })
+            await tab.waitForChatFinishesLoading()
+        })
+
+        it('Checks for initial follow ups', async () => {
+            await tab.waitForButtons([FollowUpTypes.CreateDocumentation, FollowUpTypes.UpdateDocumentation])
+        })
+    })
+
+    describe('Creates a README', () => {
+        beforeEach(async function () {
+            tab.addChatMessage({ command: '/doc' })
+            await tab.waitForChatFinishesLoading()
+        })
+
+        it('Creates a README for root folder', async () => {
+            await tab.waitForButtons([FollowUpTypes.CreateDocumentation])
+
+            tab.clickButton(FollowUpTypes.CreateDocumentation)
+
+            await tab.waitForText(i18n('AWS.amazonq.doc.answer.createReadme'))
+
+            await tab.waitForButtons([FollowUpTypes.ProceedFolderSelection])
+
+            tab.clickButton(FollowUpTypes.ProceedFolderSelection)
+
+            await tab.waitForText(docGenerationProgressMessage(DocGenerationStep.SUMMARIZING_FILES, Mode.CREATE))
+
+            await tab.waitForText(
+                `${i18n('AWS.amazonq.doc.answer.readmeCreated')} ${i18n('AWS.amazonq.doc.answer.codeResult')}`
+            )
+
+            await tab.waitForButtons([
+                FollowUpTypes.AcceptChanges,
+                FollowUpTypes.MakeChanges,
+                FollowUpTypes.RejectChanges,
+            ])
+        })
+    })
+})
diff --git a/packages/amazonq/test/e2e/amazonq/featureDev.test.ts b/packages/amazonq/test/e2e/amazonq/featureDev.test.ts
@@ -22,22 +22,11 @@ describe('Amazon Q Feature Dev', function () {
     const fileLevelAcceptPrompt = `${prompt} and add a license, and a contributing file`
     const tooManyRequestsWaitTime = 100000
 
-    function waitForButtons(buttons: FollowUpTypes[]) {
-        return tab.waitForEvent(() => {
-            return buttons.every((value) => tab.hasButton(value))
-        })
-    }
-
     async function waitForText(text: string) {
-        await tab.waitForEvent(
-            () => {
-                return tab.getChatItems().some((chatItem) => chatItem.body === text)
-            },
-            {
-                waitIntervalInMs: 250,
-                waitTimeoutInMs: 2000,
-            }
-        )
+        await tab.waitForText(text, {
+            waitIntervalInMs: 250,
+            waitTimeoutInMs: 2000,
+        })
     }
 
     async function iterate(prompt: string) {
@@ -201,12 +190,12 @@ describe('Amazon Q Feature Dev', function () {
         it('Clicks accept code and click new task', async () => {
             await retryIfRequired(async () => {
                 await Promise.any([
-                    waitForButtons([FollowUpTypes.InsertCode, FollowUpTypes.ProvideFeedbackAndRegenerateCode]),
-                    waitForButtons([FollowUpTypes.Retry]),
+                    tab.waitForButtons([FollowUpTypes.InsertCode, FollowUpTypes.ProvideFeedbackAndRegenerateCode]),
+                    tab.waitForButtons([FollowUpTypes.Retry]),
                 ])
             })
             tab.clickButton(FollowUpTypes.InsertCode)
-            await waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
+            await tab.waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
             tab.clickButton(FollowUpTypes.NewTask)
             await waitForText('What new task would you like to work on?')
             assert.deepStrictEqual(tab.getChatItems().pop()?.body, 'What new task would you like to work on?')
@@ -215,15 +204,15 @@ describe('Amazon Q Feature Dev', function () {
         it('Iterates on codegen', async () => {
             await retryIfRequired(async () => {
                 await Promise.any([
-                    waitForButtons([FollowUpTypes.InsertCode, FollowUpTypes.ProvideFeedbackAndRegenerateCode]),
-                    waitForButtons([FollowUpTypes.Retry]),
+                    tab.waitForButtons([FollowUpTypes.InsertCode, FollowUpTypes.ProvideFeedbackAndRegenerateCode]),
+                    tab.waitForButtons([FollowUpTypes.Retry]),
                 ])
             })
             tab.clickButton(FollowUpTypes.ProvideFeedbackAndRegenerateCode)
             await tab.waitForChatFinishesLoading()
             await iterate(codegenApproachPrompt)
             tab.clickButton(FollowUpTypes.InsertCode)
-            await waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
+            await tab.waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
         })
     })
 
@@ -240,8 +229,8 @@ describe('Amazon Q Feature Dev', function () {
             )
             await retryIfRequired(async () => {
                 await Promise.any([
-                    waitForButtons([FollowUpTypes.InsertCode, FollowUpTypes.ProvideFeedbackAndRegenerateCode]),
-                    waitForButtons([FollowUpTypes.Retry]),
+                    tab.waitForButtons([FollowUpTypes.InsertCode, FollowUpTypes.ProvideFeedbackAndRegenerateCode]),
+                    tab.waitForButtons([FollowUpTypes.Retry]),
                 ])
             })
         })
@@ -271,7 +260,7 @@ describe('Amazon Q Feature Dev', function () {
 
             it('disables all action buttons when new task is clicked', async () => {
                 tab.clickButton(FollowUpTypes.InsertCode)
-                await waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
+                await tab.waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
                 tab.clickButton(FollowUpTypes.NewTask)
                 await waitForText('What new task would you like to work on?')
 
@@ -283,7 +272,7 @@ describe('Amazon Q Feature Dev', function () {
 
             it('disables all action buttons when close session is clicked', async () => {
                 tab.clickButton(FollowUpTypes.InsertCode)
-                await waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
+                await tab.waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
                 tab.clickButton(FollowUpTypes.CloseSession)
                 await waitForText(
                     "Okay, I've ended this chat session. You can open a new tab to chat or start another workflow."
@@ -335,7 +324,7 @@ describe('Amazon Q Feature Dev', function () {
                 for (const filePath of filePaths) {
                     await clickActionButton(filePath, 'accept-change')
                 }
-                await waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
+                await tab.waitForButtons([FollowUpTypes.NewTask, FollowUpTypes.CloseSession])
 
                 assert.ok(tab.hasButton(FollowUpTypes.InsertCode) === false)
                 assert.ok(tab.hasButton(FollowUpTypes.ProvideFeedbackAndRegenerateCode) === false)
diff --git a/packages/amazonq/test/e2e/amazonq/framework/messenger.ts b/packages/amazonq/test/e2e/amazonq/framework/messenger.ts
@@ -52,7 +52,7 @@ export class Messenger {
 
         const lastChatItem = this.getChatItems().pop()
         const option = lastChatItem?.followUp?.options?.filter((option) => option.type === type)
-        if (!option || option.length > 1) {
+        if (!option?.length || option.length > 1) {
             assert.fail('Could not find follow up option')
         }
 
@@ -153,17 +153,23 @@ export class Messenger {
         return this.getActionsByFilePath(filePath).some((action) => action.name === actionName)
     }
 
+    async waitForText(text: string, waitOverrides?: MessengerOptions) {
+        await this.waitForEvent(() => {
+            return this.getChatItems().some((chatItem) => chatItem.body === text)
+        }, waitOverrides)
+    }
+
+    async waitForButtons(buttons: FollowUpTypes[]) {
+        return this.waitForEvent(() => {
+            return buttons.every((value) => this.hasButton(value))
+        })
+    }
+
     async waitForChatFinishesLoading() {
         return this.waitForEvent(() => this.getStore().loadingChat === false || this.hasButton(FollowUpTypes.Retry))
     }
 
-    async waitForEvent(
-        event: () => boolean,
-        waitOverrides?: {
-            waitIntervalInMs: number
-            waitTimeoutInMs: number
-        }
-    ) {
+    async waitForEvent(event: () => boolean, waitOverrides?: MessengerOptions) {
         /**
          * Wait until the chat has finished loading. This happens when a backend request
          * has finished and responded in the chat
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -20,6 +20,7 @@
         "./auth": "./dist/src/auth/index.js",
         "./amazonqGumby": "./dist/src/amazonqGumby/index.js",
         "./amazonqFeatureDev": "./dist/src/amazonqFeatureDev/index.js",
+        "./amazonqDoc": "./dist/src/amazonqDoc/index.js",
         "./amazonqScan": "./dist/src/amazonqScan/index.js",
         "./amazonqTest": "./dist/src/amazonqTest/index.js",
         "./codewhispererChat": "./dist/src/codewhispererChat/index.js",
