tests: Introduce an e2e testing framework for Amazon Q (#4254)

jpinkney-aws · web-flow · commit 144dfccb9b19 · 2024-01-24T16:54:36.000-05:00
diff --git a/docs/TESTPLAN.md b/docs/TESTPLAN.md
@@ -41,6 +41,7 @@ The test suite has the following categories of tests:
 -   E2E Tests: **slow** tests
     -   Live in `src/testE2E`
     -   These tests are heavier than Integration tests.
+    -   Some E2E tests have a more complicated architecture, described in [TEST_E2E](./TEST_E2E.md)
 
 ## Test files
 
diff --git a/docs/TEST_E2E.md b/docs/TEST_E2E.md
@@ -0,0 +1,19 @@
+### Test E2E
+
+This document describes the testing architecture specifically for more complicated E2E tests running the toolkits.
+
+#### Amazon Q E2E Tests
+
+Since VSCode does not have native support for writing E2E/UI tests for components occuring inside of a webview, we have taken a slightly different approach to write E2E tests for Amazon Q.
+
+The E2E tests for Amazon Q follow the following flow: ![Amazon Q E2E flow](./images/amazon_q_e2e.png)
+
+This means instead of running the E2E tests inside of the webview, we hoist the Mynah UI package into the test structure using JSDOM. This allows us to directly interface with Mynah UI allowing us to not rely on any DOM elements, making tests a lot easier to create and simple to maintain.
+
+The downsides are that we technically aren't testing any of the actual contents of the webview to make sure things are behaving correctly. Instead, we are testing that when we programatically send a message to the UI that the Mynah UI <-> Amazon Q VSCode code interacts successfully.
+
+With this approach, the follow things can be tested:
+
+-   Whether or not certain features show/not show depending on the status of the users auth
+-   Run requests directly against the backend and see if we get results back
+-   Clicking any follow up buttons (including examples)
diff --git a/docs/images/amazon_q_e2e.png b/docs/images/amazon_q_e2e.png
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -4282,6 +4282,7 @@
         "@types/fs-extra": "^9.0.11",
         "@types/glob": "^8.1.0",
         "@types/js-yaml": "^4.0.5",
+        "@types/jsdom": "^21.1.6",
         "@types/lodash": "^4.14.180",
         "@types/markdown-it": "^13.0.2",
         "@types/mime-types": "^2.1.1",
@@ -4321,6 +4322,7 @@
         "file-loader": "^6.2.0",
         "husky": "^7.0.2",
         "json-schema-to-typescript": "^13.1.1",
+        "jsdom": "^23.0.1",
         "marked": "^11.1.1",
         "mocha": "^10.1.0",
         "mocha-junit-reporter": "^2.2.1",
diff --git a/src/amazonq/activation.ts b/src/amazonq/activation.ts
@@ -18,7 +18,7 @@ import { telemetry } from '../shared/telemetry/telemetry'
 import { focusAmazonQPanel } from '../auth/ui/vue/show'
 
 export async function activate(context: ExtensionContext) {
-    const appInitContext = new DefaultAmazonQAppInitContext()
+    const appInitContext = DefaultAmazonQAppInitContext.instance
 
     registerApps(appInitContext)
 
diff --git a/src/amazonq/apps/initContext.ts b/src/amazonq/apps/initContext.ts
@@ -21,6 +21,12 @@ export class DefaultAmazonQAppInitContext implements AmazonQAppInitContext {
     private readonly webViewToAppsMessagePublishers: Map<TabType, MessagePublisher<any>> = new Map()
     public readonly onDidChangeAmazonQVisibility = new EventEmitter<boolean>()
 
+    static #instance: DefaultAmazonQAppInitContext
+
+    public static get instance() {
+        return (this.#instance ??= new this())
+    }
+
     constructor() {}
 
     registerWebViewToAppMessagePublisher(messagePublisher: MessagePublisher<any>, tabType: TabType): void {
diff --git a/src/amazonq/messages/messageListener.ts b/src/amazonq/messages/messageListener.ts
@@ -9,6 +9,6 @@ export class MessageListener<T> {
     constructor(private readonly eventEmitter: EventEmitter<T>) {}
 
     public onMessage(listener: (e: T) => any) {
-        this.eventEmitter.event(listener)
+        return this.eventEmitter.event(listener)
     }
 }
diff --git a/src/amazonq/webview/generators/webViewContent.ts b/src/amazonq/webview/generators/webViewContent.ts
@@ -20,6 +20,14 @@ export class WebViewContentGenerator {
         return `<!DOCTYPE html>
         <html>
             <head>
+                <style>
+                body.vscode-dark,
+                body.vscode-high-contrast:not(.vscode-high-contrast-light) {
+                    --mynah-color-light: rgba(255, 255, 255, 0.05);
+                    --mynah-color-highlight: rgba(0, 137, 255, 0.2);
+                    --mynah-color-highlight-text: rgba(0, 137, 255, 1);
+                }                
+                </style>
                 <meta http-equiv="Content-Security-Policy" content="${contentPolicy}">
                 <title>Amazon Q (Preview)</title>                
                 ${await this.generateJS(extensionURI, webView)}                
diff --git a/src/amazonq/webview/ui/main.ts b/src/amazonq/webview/ui/main.ts
@@ -4,7 +4,6 @@
  */
 import { Connector } from './connector'
 import { ChatItem, ChatItemType, MynahUI, MynahUIDataModel, NotificationType } from '@aws/mynah-ui'
-import './styles/dark.scss'
 import { ChatPrompt } from '@aws/mynah-ui/dist/static'
 import { TabsStorage, TabType } from './storages/tabsStorage'
 import { WelcomeFollowupType } from './apps/amazonqCommonsConnector'
@@ -369,4 +368,9 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, gumby
         isFeatureDevEnabled,
         isGumbyEnabled,
     })
+
+    return {
+        mynahUI,
+        messageReceiver: connector.handleMessageReceive,
+    }
 }
diff --git a/src/amazonq/webview/ui/styles/dark.scss b/src/amazonq/webview/ui/styles/dark.scss
diff --git a/src/amazonqFeatureDev/controllers/chat/controller.ts b/src/amazonqFeatureDev/controllers/chat/controller.ts
@@ -23,9 +23,9 @@ import { AuthController } from '../../../amazonq/auth/controller'
 import { getLogger } from '../../../shared/logger'
 import { submitFeedback } from '../../../feedback/vue/submitFeedback'
 import { placeholder } from '../../../shared/vscode/commands2'
-import { userGuideURL } from '../../../amazonq/webview/ui/texts/constants'
 import { EditorContentController } from '../../../amazonq/commons/controllers/contentController'
 import { openUrl } from '../../../shared/utilities/vsCodeUtils'
+import { examples } from '../../userFacingText'
 
 export interface ChatControllerEventEmitters {
     readonly processHumanChatMessage: EventEmitter<any>
@@ -212,6 +212,7 @@ export class FeatureDevController {
         this.messenger.sendAsyncEventProgress(tabID, true, undefined)
 
         this.messenger.sendUpdatePlaceholder(tabID, 'Generating implementation plan ...')
+
         const interactions = await session.send(message)
         this.messenger.sendUpdatePlaceholder(tabID, 'Add more detail to iterate on the implementation plan')
 
@@ -326,13 +327,6 @@ export class FeatureDevController {
     }
 
     private initialExamples(message: any) {
-        const examples = `
-You can use /dev to:
-- Plan a code change
-- Coming soon: Generate code suggestions
-
-To learn more, visit the _[Amazon Q User Guide](${userGuideURL})_.
-`
         this.messenger.sendAnswer({
             type: 'answer',
             tabID: message.tabID,
diff --git a/src/amazonqFeatureDev/userFacingText.ts b/src/amazonqFeatureDev/userFacingText.ts
@@ -0,0 +1,14 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { userGuideURL } from '../amazonq/webview/ui/texts/constants'
+
+export const examples = `
+You can use /dev to:
+- Plan a code change
+- Coming soon: Generate code suggestions
+
+To learn more, visit the _[Amazon Q User Guide](${userGuideURL})_.
+`
diff --git a/src/testE2E/amazonq/featureDev.test.ts b/src/testE2E/amazonq/featureDev.test.ts
@@ -0,0 +1,139 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import assert from 'assert'
+import { qTestingFramework } from './framework/framework'
+import { FollowUpTypes } from '../../amazonqFeatureDev/types'
+import sinon from 'sinon'
+import { FeatureDevClient } from '../../amazonqFeatureDev/client/featureDev'
+import { verifyTextOrder } from './framework/text'
+import { examples } from '../../amazonqFeatureDev/userFacingText'
+import * as authUtil from '../../codewhisperer/util/authUtil'
+import request from '../../common/request'
+
+describe('Amazon Q Feature Dev', function () {
+    let framework: qTestingFramework
+
+    const samplePlanResponse = 'sample plan response'
+
+    beforeEach(() => {
+        /**
+         * TODO remove these stubs when we know the backend can handle all the test load + when we know the tests
+         * are working without any flakiness
+         */
+        sinon.stub(authUtil, 'getChatAuthState').resolves({
+            amazonQ: 'connected',
+            codewhispererChat: 'connected',
+            codewhispererCore: 'connected',
+        })
+        sinon.stub(FeatureDevClient.prototype, 'createConversation').resolves('1234')
+        sinon.stub(FeatureDevClient.prototype, 'createUploadUrl').resolves({
+            uploadId: '5678',
+            uploadUrl: 'foo',
+            $response: sinon.mock() as any,
+        })
+        sinon.stub(FeatureDevClient.prototype, 'generatePlan').resolves(samplePlanResponse)
+        sinon.stub(request, 'fetch').resolves({
+            response: {
+                status: 200,
+            },
+        })
+        framework = new qTestingFramework('featuredev', true, true)
+    })
+
+    afterEach(() => {
+        framework.dispose()
+        sinon.restore()
+    })
+
+    describe('quick action availability', () => {
+        it('Shows /dev when feature dev is enabled', () => {
+            const q = framework.createTab()
+            const command = q.findCommand('/dev')
+            if (!command) {
+                assert.fail('Could not find command')
+            }
+
+            if (command.length > 1) {
+                assert.fail('Found too many commands with the name /dev')
+            }
+        })
+
+        it('Does NOT show /dev when feature dev is NOT enabled', () => {
+            framework = new qTestingFramework('featuredev', false, true)
+            const q = framework.createTab()
+            const command = q.findCommand('/dev')
+            if (command.length > 0) {
+                assert.fail('Found command when it should not have been found')
+            }
+        })
+    })
+
+    describe('/dev {msg} entry', async () => {
+        it('Receives chat response', async () => {
+            this.timeout(60000)
+            const q = framework.createTab()
+            const prompt = 'Implement twosum in typescript'
+            q.addChatMessage({ command: '/dev', prompt })
+
+            // Wait for a backend response
+            await q.waitForChatFinishesLoading()
+
+            const chatItems = q.getChatItems()
+
+            // Verify that all the responses come back in the correct order
+            verifyTextOrder(chatItems, ['Welcome to /dev', prompt, samplePlanResponse])
+
+            // Check that the last UI message has the two buttons
+            assert.notStrictEqual(chatItems.pop()?.followUp?.options, [
+                {
+                    type: FollowUpTypes.NewPlan,
+                },
+                {
+                    type: FollowUpTypes.GenerateCode,
+                    disabled: true,
+                },
+            ])
+        })
+    })
+
+    describe('/dev entry', () => {
+        it('Clicks examples', async () => {
+            const q = framework.createTab()
+            q.addChatMessage({ command: '/dev' })
+            q.clickButton(FollowUpTypes.DevExamples)
+
+            const lastChatItems = q.getChatItems().pop()
+            assert.deepStrictEqual(lastChatItems?.body, examples)
+        })
+
+        it('Receives chat response', async () => {
+            this.timeout(60000)
+            const q = framework.createTab()
+            const prompt = 'Implement twosum in typescript'
+            q.addChatMessage({ command: '/dev' })
+            q.addChatMessage({ prompt })
+
+            // Wait for a backend response
+            await q.waitForChatFinishesLoading()
+
+            const chatItems = q.getChatItems()
+
+            // Verify that all the responses come back in the correct order
+            verifyTextOrder(chatItems, [prompt, samplePlanResponse])
+
+            // Check that the UI has the two buttons
+            assert.notStrictEqual(chatItems.pop()?.followUp?.options, [
+                {
+                    type: FollowUpTypes.NewPlan,
+                },
+                {
+                    type: FollowUpTypes.GenerateCode,
+                    disabled: true,
+                },
+            ])
+        })
+    })
+})
diff --git a/src/testE2E/amazonq/framework/framework.ts b/src/testE2E/amazonq/framework/framework.ts
@@ -0,0 +1,82 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { injectJSDOM } from './jsdomInjector'
+
+// This needs to be ran before all other imports so that mynah ui gets loaded inside of jsdom
+injectJSDOM()
+
+import * as vscode from 'vscode'
+import { createMynahUI } from '../../../amazonq/webview/ui/main'
+import { MynahUI, MynahUIProps } from '@aws/mynah-ui'
+import { DefaultAmazonQAppInitContext } from '../../../amazonq/apps/initContext'
+import { TabType } from '../../../amazonq/webview/ui/storages/tabsStorage'
+import { Messenger, MessengerOptions } from './messenger'
+
+/**
+ * Abstraction over Amazon Q to make e2e testing easier
+ */
+export class qTestingFramework {
+    private readonly mynahUI: MynahUI
+    private readonly mynahUIProps: MynahUIProps
+    private disposables: vscode.Disposable[] = []
+
+    constructor(featureName: TabType, featureDevEnabled: boolean, gumbyEnabled: boolean) {
+        /**
+         * Instantiate the UI and override the postMessage to publish using the app message
+         * publishers directly.
+         *
+         * The postMessage function implements the MynahUI -> VSCode flow
+         */
+        const ui = createMynahUI(
+            {
+                postMessage: (message: string) => {
+                    const appMessagePublisher = DefaultAmazonQAppInitContext.instance
+                        .getWebViewToAppsMessagePublishers()
+                        .get(featureName)
+                    if (appMessagePublisher === undefined) {
+                        return
+                    }
+                    appMessagePublisher.publish(message)
+                },
+            },
+            featureDevEnabled,
+            gumbyEnabled
+        )
+        this.mynahUI = ui.mynahUI
+        this.mynahUIProps = (this.mynahUI as any).props
+
+        /**
+         * Listen to incoming events coming from VSCode and redirect them to MynahUI
+         *
+         * This implements the VSCode -> Mynah UI flow
+         */
+        this.disposables.push(
+            DefaultAmazonQAppInitContext.instance.getAppsToWebViewMessageListener().onMessage(async message => {
+                // Emulate the json format of postMessage
+                const event = {
+                    data: JSON.stringify(message),
+                } as any
+                await ui.messageReceiver(event)
+            })
+        )
+    }
+
+    /**
+     * Create a new tab and then return a new encapsulated tab messenger that makes it easier to directly call
+     * functionality against a specific tab
+     */
+    public createTab(options?: MessengerOptions) {
+        const newTabID = this.mynahUI.updateStore('', {})
+        if (!newTabID) {
+            throw new Error('Could not create tab id')
+        }
+        return new Messenger(newTabID, this.mynahUIProps, this.mynahUI, options)
+    }
+
+    public dispose() {
+        vscode.Disposable.from(...this.disposables).dispose()
+    }
+}
diff --git a/src/testE2E/amazonq/framework/jsdomInjector.ts b/src/testE2E/amazonq/framework/jsdomInjector.ts
diff --git a/src/testE2E/amazonq/framework/messenger.ts b/src/testE2E/amazonq/framework/messenger.ts
diff --git a/src/testE2E/amazonq/framework/text.ts b/src/testE2E/amazonq/framework/text.ts

Original file line number	Diff line number	Diff line change
`@@ -9,6 +9,6 @@ export class MessageListener<T> {`
`9`	`9`	`constructor(private readonly eventEmitter: EventEmitter<T>) {}`
`10`	`10`
`11`	`11`	`public onMessage(listener: (e: T) => any) {`
`12`		`- this.eventEmitter.event(listener)`
	`12`	`+ return this.eventEmitter.event(listener)`
`13`	`13`	`}`
`14`	`14`	`}`