tests: Introduce an e2e testing framework for Amazon Q

jpinkney-aws · jpinkney-aws · commit 50e27f5c73fa · 2024-01-24T12:14:34.000-05:00
Problem:
- Amazon Q has to be manually tested on each release for everything e2e because we don't have any automation

Solution:
- Introduce a testing framework for Amazon Q that will allow teams to easily test Amazon Q e2e.
- Since VSCode doesn't have a built in way to do e2e tests against a webview, we instead hoist everything that would run inside of the webview into the current test instance, allowing us to test directly against the mynah UI instance programatically. This makes it so that we don't have to maintain any DOM elements + makes writing tests simple/easy to maintain while still testing all the same code flow e2e (minus the code that loads mynah ui into the webview)
diff --git a/docs/E2E_TESTING_ARCHITECTURE.md b/docs/E2E_TESTING_ARCHITECTURE.md
@@ -0,0 +1,19 @@
+### E2E Testing
+
+This document describes the testing architecture specifically for more complicated E2E tests running the toolkits.
+
+#### Amazon Q E2E Tests
+
+Since VSCode does not have native support for writing E2E/UI tests for components occuring inside of a webview, we have taken a slightly different approach to write E2E tests for Amazon Q.
+
+The E2E tests for Amazon Q follow the following flow: ![Amazon Q E2E flow](./images/amazon_q_e2e.png)
+
+This means instead of running the E2E tests inside of the webview, we hoist the Mynah UI package into the test structure using JSDOM. This allows us to directly interface with Mynah UI allowing us to not rely on any DOM elements, making tests a lot easier to create and simple to maintain.
+
+The downsides are that we technically aren't testing any of the actual contents of the webview to make sure things are behaving correctly. Instead, we are testing that when we programatically send a message to the UI that the Mynah UI <-> Amazon Q VSCode code interacts successfully.
+
+With this approach, the follow things can be tested:
+
+-   Whether or not certain features show/not show depending on the status of the users auth
+-   Run requests directly against the backend and see if we get results back
+-   Clicking any follow up buttons (including examples)
diff --git a/docs/TESTPLAN.md b/docs/TESTPLAN.md
@@ -41,6 +41,7 @@ The test suite has the following categories of tests:
 -   E2E Tests: **slow** tests
     -   Live in `src/testE2E`
     -   These tests are heavier than Integration tests.
+    -   Some E2E tests have a more complicated architecture, described in [E2E_TESTING](./E2E_TESTING_ARCHITECTURE.md)
 
 ## Test files
 
diff --git a/docs/images/amazon_q_e2e.png b/docs/images/amazon_q_e2e.png
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -4282,6 +4282,7 @@
         "@types/fs-extra": "^9.0.11",
         "@types/glob": "^8.1.0",
         "@types/js-yaml": "^4.0.5",
+        "@types/jsdom": "^21.1.6",
         "@types/lodash": "^4.14.180",
         "@types/markdown-it": "^13.0.2",
         "@types/mime-types": "^2.1.1",
@@ -4321,6 +4322,7 @@
         "file-loader": "^6.2.0",
         "husky": "^7.0.2",
         "json-schema-to-typescript": "^13.1.1",
+        "jsdom": "^23.0.1",
         "marked": "^11.1.1",
         "mocha": "^10.1.0",
         "mocha-junit-reporter": "^2.2.1",
diff --git a/src/amazonq/activation.ts b/src/amazonq/activation.ts
@@ -18,7 +18,7 @@ import { telemetry } from '../shared/telemetry/telemetry'
 import { focusAmazonQPanel } from '../auth/ui/vue/show'
 
 export async function activate(context: ExtensionContext) {
-    const appInitContext = new DefaultAmazonQAppInitContext()
+    const appInitContext = DefaultAmazonQAppInitContext.instance
 
     registerApps(appInitContext)
 
diff --git a/src/amazonq/apps/initContext.ts b/src/amazonq/apps/initContext.ts
@@ -21,6 +21,12 @@ export class DefaultAmazonQAppInitContext implements AmazonQAppInitContext {
     private readonly webViewToAppsMessagePublishers: Map<TabType, MessagePublisher<any>> = new Map()
     public readonly onDidChangeAmazonQVisibility = new EventEmitter<boolean>()
 
+    static #instance: DefaultAmazonQAppInitContext
+
+    public static get instance() {
+        return (this.#instance ??= new this())
+    }
+
     constructor() {}
 
     registerWebViewToAppMessagePublisher(messagePublisher: MessagePublisher<any>, tabType: TabType): void {
diff --git a/src/amazonq/messages/messageListener.ts b/src/amazonq/messages/messageListener.ts
@@ -9,6 +9,6 @@ export class MessageListener<T> {
     constructor(private readonly eventEmitter: EventEmitter<T>) {}
 
     public onMessage(listener: (e: T) => any) {
-        this.eventEmitter.event(listener)
+        return this.eventEmitter.event(listener)
     }
 }
diff --git a/src/amazonq/webview/generators/webViewContent.ts b/src/amazonq/webview/generators/webViewContent.ts
@@ -20,6 +20,14 @@ export class WebViewContentGenerator {
         return `<!DOCTYPE html>
         <html>
             <head>
+                <style>
+                body.vscode-dark,
+                body.vscode-high-contrast:not(.vscode-high-contrast-light) {
+                    --mynah-color-light: rgba(255, 255, 255, 0.05);
+                    --mynah-color-highlight: rgba(0, 137, 255, 0.2);
+                    --mynah-color-highlight-text: rgba(0, 137, 255, 1);
+                }                
+                </style>
                 <meta http-equiv="Content-Security-Policy" content="${contentPolicy}">
                 <title>Amazon Q (Preview)</title>                
                 ${await this.generateJS(extensionURI, webView)}                
diff --git a/src/amazonq/webview/ui/main.ts b/src/amazonq/webview/ui/main.ts
@@ -4,7 +4,6 @@
  */
 import { Connector } from './connector'
 import { ChatItem, ChatItemType, MynahUI, MynahUIDataModel, NotificationType } from '@aws/mynah-ui'
-import './styles/dark.scss'
 import { ChatPrompt } from '@aws/mynah-ui/dist/static'
 import { TabsStorage, TabType } from './storages/tabsStorage'
 import { WelcomeFollowupType } from './apps/amazonqCommonsConnector'
@@ -369,4 +368,9 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, gumby
         isFeatureDevEnabled,
         isGumbyEnabled,
     })
+
+    return {
+        mynahUI,
+        messageReceiver: connector.handleMessageReceive,
+    }
 }
diff --git a/src/amazonq/webview/ui/styles/dark.scss b/src/amazonq/webview/ui/styles/dark.scss
diff --git a/src/amazonqFeatureDev/controllers/chat/controller.ts b/src/amazonqFeatureDev/controllers/chat/controller.ts
@@ -23,9 +23,9 @@ import { AuthController } from '../../../amazonq/auth/controller'
 import { getLogger } from '../../../shared/logger'
 import { submitFeedback } from '../../../feedback/vue/submitFeedback'
 import { placeholder } from '../../../shared/vscode/commands2'
-import { userGuideURL } from '../../../amazonq/webview/ui/texts/constants'
 import { EditorContentController } from '../../../amazonq/commons/controllers/contentController'
 import { openUrl } from '../../../shared/utilities/vsCodeUtils'
+import { examples } from '../../text'
 
 export interface ChatControllerEventEmitters {
     readonly processHumanChatMessage: EventEmitter<any>
@@ -212,6 +212,7 @@ export class FeatureDevController {
         this.messenger.sendAsyncEventProgress(tabID, true, undefined)
 
         this.messenger.sendUpdatePlaceholder(tabID, 'Generating implementation plan ...')
+
         const interactions = await session.send(message)
         this.messenger.sendUpdatePlaceholder(tabID, 'Add more detail to iterate on the implementation plan')
 
@@ -326,13 +327,6 @@ export class FeatureDevController {
     }
 
     private initialExamples(message: any) {
-        const examples = `
-You can use /dev to:
-- Plan a code change
-- Coming soon: Generate code suggestions
-
-To learn more, visit the _[Amazon Q User Guide](${userGuideURL})_.
-`
         this.messenger.sendAnswer({
             type: 'answer',
             tabID: message.tabID,
diff --git a/src/amazonqFeatureDev/text.ts b/src/amazonqFeatureDev/text.ts
@@ -0,0 +1,14 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { userGuideURL } from '../amazonq/webview/ui/texts/constants'
+
+export const examples = `
+You can use /dev to:
+- Plan a code change
+- Coming soon: Generate code suggestions
+
+To learn more, visit the _[Amazon Q User Guide](${userGuideURL})_.
+`
diff --git a/src/testE2E/amazonq/featureDev.test.ts b/src/testE2E/amazonq/featureDev.test.ts
@@ -0,0 +1,135 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import * as got from 'got'
+import assert from 'assert'
+import { qTestingFramework } from './framework/framework'
+import { FollowUpTypes } from '../../amazonqFeatureDev/types'
+import sinon from 'sinon'
+import { FeatureDevClient } from '../../amazonqFeatureDev/client/featureDev'
+import { verifyTextOrder } from './framework/text'
+import { examples } from '../../amazonqFeatureDev/text'
+import * as authUtil from '../../codewhisperer/util/authUtil'
+
+describe('Amazon Q Feature Dev', function () {
+    let framework: qTestingFramework
+
+    const samplePlanResponse = 'sample plan response'
+
+    beforeEach(() => {
+        /**
+         * TODO remove these stubs when we know the backend can handle all the test load + when we know the tests
+         * are working without any flakiness
+         */
+        sinon.stub(authUtil, 'getChatAuthState').resolves({
+            amazonQ: 'connected',
+            codewhispererChat: 'connected',
+            codewhispererCore: 'connected',
+        })
+        sinon.stub(FeatureDevClient.prototype, 'createConversation').resolves('1234')
+        sinon.stub(FeatureDevClient.prototype, 'createUploadUrl').resolves({
+            uploadId: '5678',
+            uploadUrl: 'foo',
+            $response: sinon.mock() as any,
+        })
+        sinon.stub(FeatureDevClient.prototype, 'generatePlan').resolves(samplePlanResponse)
+        sinon.stub(got, 'default').resolves({ statusCode: 200 })
+        framework = new qTestingFramework('featuredev', true, true)
+    })
+
+    afterEach(() => {
+        framework.dispose()
+        sinon.restore()
+    })
+
+    describe('quick action availability', () => {
+        it('Shows /dev when feature dev is enabled', () => {
+            const q = framework.createTab()
+            const command = q.findCommand('/dev')
+            if (!command) {
+                assert.fail('Could not find command')
+            }
+
+            if (command.length > 1) {
+                assert.fail('Found too many commands with the name /dev')
+            }
+        })
+
+        it('Does NOT show /dev when feature dev is NOT enabled', () => {
+            framework = new qTestingFramework('featuredev', false, true)
+            const q = framework.createTab()
+            const command = q.findCommand('/dev')
+            if (command.length > 0) {
+                assert.fail('Found command when it should not have been found')
+            }
+        })
+    })
+
+    describe('/dev {msg} entry', async () => {
+        it('Receives chat response', async () => {
+            this.timeout(60000)
+            const q = framework.createTab()
+            const prompt = 'Implement twosum in typescript'
+            q.addChatMessage({ command: '/dev', prompt })
+
+            // Wait for a backend response
+            await q.waitForChatFinishesLoading()
+
+            const chatItems = q.getChatItems()
+
+            // Verify that all the responses come back in the correct order
+            verifyTextOrder(chatItems, ['Welcome to /dev', prompt, samplePlanResponse])
+
+            // Check that the last UI message has the two buttons
+            assert.notStrictEqual(chatItems.pop()?.followUp?.options, [
+                {
+                    type: FollowUpTypes.NewPlan,
+                },
+                {
+                    type: FollowUpTypes.GenerateCode,
+                    disabled: true,
+                },
+            ])
+        })
+    })
+
+    describe('/dev entry', () => {
+        it('Clicks examples', async () => {
+            const q = framework.createTab()
+            q.addChatMessage({ command: '/dev' })
+            q.clickButton(FollowUpTypes.DevExamples)
+
+            const lastChatItems = q.getChatItems().pop()
+            assert.deepStrictEqual(lastChatItems?.body, examples)
+        })
+
+        it('Receives chat response', async () => {
+            this.timeout(60000)
+            const q = framework.createTab()
+            const prompt = 'Implement twosum in typescript'
+            q.addChatMessage({ command: '/dev' })
+            q.addChatMessage({ prompt })
+
+            // Wait for a backend response
+            await q.waitForChatFinishesLoading()
+
+            const chatItems = q.getChatItems()
+
+            // Verify that all the responses come back in the correct order
+            verifyTextOrder(chatItems, [prompt, samplePlanResponse])
+
+            // Check that the UI has the two buttons
+            assert.notStrictEqual(chatItems.pop()?.followUp?.options, [
+                {
+                    type: FollowUpTypes.NewPlan,
+                },
+                {
+                    type: FollowUpTypes.GenerateCode,
+                    disabled: true,
+                },
+            ])
+        })
+    })
+})
diff --git a/src/testE2E/amazonq/framework/framework.ts b/src/testE2E/amazonq/framework/framework.ts
@@ -0,0 +1,101 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { JSDOM } from 'jsdom'
+/**
+ * JSDOM is used to help hoist MynahUI to running in a node environment vs in the browser (which is what it's made for)
+ */
+const dom = new JSDOM(undefined, {
+    pretendToBeVisual: true,
+    includeNodeLocations: true,
+})
+global.window = dom.window as unknown as Window & typeof globalThis
+global.document = dom.window.document
+global.self = dom.window as unknown as Window & typeof globalThis
+global.Element = dom.window.Element
+global.HTMLElement = dom.window.HTMLElement
+
+// jsdom doesn't have support for innerText: https://github.com/jsdom/jsdom/issues/1245 which mynah ui uses
+Object.defineProperty(global.Element.prototype, 'innerText', {
+    get() {
+        return this.textContent
+    },
+})
+
+// jsdom doesn't have support for structuredClone. See https://github.com/jsdom/jsdom/issues/3363
+global.structuredClone = val => JSON.parse(JSON.stringify(val))
+
+import * as vscode from 'vscode'
+import { createMynahUI } from '../../../amazonq/webview/ui/main'
+import { MynahUI, MynahUIProps } from '@aws/mynah-ui'
+import { DefaultAmazonQAppInitContext } from '../../../amazonq/apps/initContext'
+import { TabType } from '../../../amazonq/webview/ui/storages/tabsStorage'
+import { Messenger, MessengerOptions } from './messenger'
+
+/**
+ * Abstraction over Amazon Q to make e2e testing easier
+ */
+export class qTestingFramework {
+    private readonly mynahUI: MynahUI
+    private readonly mynahUIProps: MynahUIProps
+    private disposables: vscode.Disposable[] = []
+
+    constructor(featureName: TabType, featureDevEnabled: boolean, gumbyEnabled: boolean) {
+        /**
+         * Instantiate the UI and override the postMessage to publish using the app message
+         * publishers directly.
+         *
+         * The postMessage function implements the MynahUI -> VSCode flow
+         */
+        const ui = createMynahUI(
+            {
+                postMessage: (message: string) => {
+                    const appMessagePublisher = DefaultAmazonQAppInitContext.instance
+                        .getWebViewToAppsMessagePublishers()
+                        .get(featureName)
+                    if (appMessagePublisher === undefined) {
+                        return
+                    }
+                    appMessagePublisher.publish(message)
+                },
+            },
+            featureDevEnabled,
+            gumbyEnabled
+        )
+        this.mynahUI = ui.mynahUI
+        this.mynahUIProps = (this.mynahUI as any).props
+
+        /**
+         * Listen to incoming events coming from VSCode and redirect them to MynahUI
+         *
+         * This implements the VSCode -> Mynah UI flow
+         */
+        this.disposables.push(
+            DefaultAmazonQAppInitContext.instance.getAppsToWebViewMessageListener().onMessage(async message => {
+                // Emulate the json format of postMessage
+                const event = {
+                    data: JSON.stringify(message),
+                } as any
+                await ui.messageReceiver(event)
+            })
+        )
+    }
+
+    /**
+     * Create a new tab and then return a new encapsulated tab messenger that makes it easier to directly call
+     * functionality against a specific tab
+     */
+    public createTab(options?: MessengerOptions) {
+        const newTabID = this.mynahUI.updateStore('', {})
+        if (!newTabID) {
+            throw new Error('Could not create tab id')
+        }
+        return new Messenger(newTabID, this.mynahUIProps, this.mynahUI, options)
+    }
+
+    public dispose() {
+        vscode.Disposable.from(...this.disposables).dispose()
+    }
+}
diff --git a/src/testE2E/amazonq/framework/messenger.ts b/src/testE2E/amazonq/framework/messenger.ts
diff --git a/src/testE2E/amazonq/framework/text.ts b/src/testE2E/amazonq/framework/text.ts

Original file line number	Diff line number	Diff line change
`@@ -9,6 +9,6 @@ export class MessageListener<T> {`
`9`	`9`	`constructor(private readonly eventEmitter: EventEmitter<T>) {}`
`10`	`10`
`11`	`11`	`public onMessage(listener: (e: T) => any) {`
`12`		`- this.eventEmitter.event(listener)`
	`12`	`+ return this.eventEmitter.event(listener)`
`13`	`13`	`}`
`14`	`14`	`}`