Merge pull request #4405 from jpinkney-aws/jpinkney-aws/sso-testing

tests: Allow Amazon Q e2e tests to call the backend

Author: jpinkney-aws

CONTRIBUTING.md

@@ -420,7 +420,7 @@ Unlike the user setting overrides, not all of these environment variables have t
 
 #### Lambda
 
--   `AUTH_UTIL_LAMBDA_ARN`: The Amazon Resource Name (ARN) of the lambda function
+-   `AUTH_UTIL_LAMBDA_ARN`: The Auth Util Lambda is used to log into using Builder ID/IdC automatically when running e2e tests. This is the arn that points to the auth util lambda.
 
 #### ECS
 
@@ -436,8 +436,10 @@ Unlike the user setting overrides, not all of these environment variables have t
 -   `GITHUB_ACTION`: The name of the current GitHub Action workflow step that is running
 -   `CODEBUILD_BUILD_ID`: The unique ID of the current CodeBuild build that is executing
 -   `AWS_TOOLKIT_AUTOMATION`: If tests are currently being ran
--   `DEVELOPMENT_PATH`: The path to the aws toolkit vscode project
+-   `TEST_SSO_STARTURL`: The start url you want to use on E2E tests
+-   `TEST_SSO_REGION`: The region for the start url you want to use on E2E tests
 -   `AWS_TOOLKIT_TEST_NO_COLOR`: If the tests should include colour in their output
+-   `DEVELOPMENT_PATH`: The path to the aws toolkit vscode project
 -   `TEST_DIR` - The directory where the test runner should find the tests
 
 ### SAM/CFN ("goformation") JSON schema

packages/toolkit/src/testE2E/amazonq/featureDev.test.ts

@@ -7,39 +7,22 @@ import assert from 'assert'
 import { qTestingFramework } from './framework/framework'
 import { FollowUpTypes } from '../../amazonqFeatureDev/types'
 import sinon from 'sinon'
-import { FeatureDevClient } from '../../amazonqFeatureDev/client/featureDev'
 import { verifyTextOrder } from './framework/text'
 import { examples } from '../../amazonqFeatureDev/userFacingText'
-import * as authUtil from '../../codewhisperer/util/authUtil'
-import request from '../../common/request'
+import { registerAuthHook, using } from '../../test/setupUtil'
+import { loginToIdC } from './utils/setup'
 
 describe('Amazon Q Feature Dev', function () {
     let framework: qTestingFramework
 
-    const samplePlanResponse = 'sample plan response'
+    before(async function () {
+        await using(registerAuthHook('amazonq-test-account'), async () => {
+            await loginToIdC()
+        })
+    })
 
     beforeEach(() => {
-        /**
-         * TODO remove these stubs when we know the backend can handle all the test load + when we know the tests
-         * are working without any flakiness
-         */
-        sinon.stub(authUtil, 'getChatAuthState').resolves({
-            amazonQ: 'connected',
-            codewhispererChat: 'connected',
-            codewhispererCore: 'connected',
-        })
-        sinon.stub(FeatureDevClient.prototype, 'createConversation').resolves('1234')
-        sinon.stub(FeatureDevClient.prototype, 'createUploadUrl').resolves({
-            uploadId: '5678',
-            uploadUrl: 'foo',
-            $response: sinon.mock() as any,
-        })
-        sinon.stub(FeatureDevClient.prototype, 'generatePlan').resolves(samplePlanResponse)
-        sinon.stub(request, 'fetch').resolves({
-            response: {
-                status: 200,
-            },
-        })
+        registerAuthHook('amazonq-test-account')
         framework = new qTestingFramework('featuredev', true, true)
     })
 
@@ -85,8 +68,12 @@ describe('Amazon Q Feature Dev', function () {
 
             const chatItems = q.getChatItems()
 
-            // Verify that all the responses come back in the correct order
-            verifyTextOrder(chatItems, ['Welcome to /dev', prompt, samplePlanResponse])
+            /**
+             * Verify that all the responses come back in the correct order and that a response
+             * after the prompt is non empty (represents a response from the backend, since the same response isn't
+             * guarenteed we can't verify direct responses)
+             */
+            verifyTextOrder(chatItems, [/Welcome to \/dev/, new RegExp(prompt), /.\S/])
 
             // Check that the last UI message has the two buttons
             assert.notStrictEqual(chatItems.pop()?.followUp?.options, [
@@ -123,8 +110,12 @@ describe('Amazon Q Feature Dev', function () {
 
             const chatItems = q.getChatItems()
 
-            // Verify that all the responses come back in the correct order
-            verifyTextOrder(chatItems, [prompt, samplePlanResponse])
+            /**
+             * Verify that all the responses come back in the correct order and that a response
+             * after the prompt is non empty (represents a response from the backend, since the same response isn't
+             * guarenteed we can't verify direct responses)
+             */
+            verifyTextOrder(chatItems, [/Welcome to \/dev/, new RegExp(prompt), /.\S/])
 
             // Check that the UI has the two buttons
             assert.notStrictEqual(chatItems.pop()?.followUp?.options, [

packages/toolkit/src/testE2E/amazonq/framework/text.ts

@@ -12,10 +12,11 @@ import { ChatItem } from '@aws/mynah-ui'
  * @param expectedText An array of expected text items in order
  * @returns true if the items in expectedText are found in the correct order in chatItems otherwise false
  */
-export function verifyTextOrder(chatItems: ChatItem[], expectedText: string[]) {
+export function verifyTextOrder(chatItems: ChatItem[], expectedText: RegExp[]) {
     let currInd = 0
     for (const item of chatItems) {
-        if (item.body?.includes(expectedText[currInd])) {
+        const currentExpected = expectedText[currInd]
+        if (currentExpected && item.body?.match(currentExpected)) {
             currInd++
         }
     }

packages/toolkit/src/testE2E/amazonq/utils/setup.ts

@@ -0,0 +1,26 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { AuthUtil, getChatAuthState } from '../../../codewhisperer/util/authUtil'
+
+export async function loginToIdC() {
+    const authState = await getChatAuthState()
+    if (process.env['AWS_TOOLKIT_AUTOMATION'] === 'local') {
+        if (authState.amazonQ !== 'connected') {
+            throw new Error('You will need to login manually before running tests.')
+        }
+        return
+    }
+
+    const startUrl = process.env['TEST_SSO_STARTURL']
+    const region = process.env['TEST_SSO_REGION']
+
+    if (!startUrl || !region) {
+        throw new Error(
+            'TEST_SSO_STARTURL and TEST_SSO_REGION are required environment variables when running Amazon Q E2E tests'
+        )
+    }
+
+    await AuthUtil.instance.connectToEnterpriseSso(startUrl, region)
+}