fix(amazonq): Reducing error propagation flakiness in UI E2E Tests in utils and Helpers (#7882)

surajrdy-aws · web-flow · commit 9db3a91439c4 · 2025-08-15T10:07:31.000-07:00
## Problem Our tests are currently experiencing flakiness due to our usage of try-catch blocks which allow errors to pass through our tests. One of our more egregious examples of this is our `quickActions.test.ts` file which is unable to find the proper CSS element of our quickAction command overlay, but still allows our test to pass as this error is non-critical due to our try-catch block. This leaves an inherent problem in our tests making them flaky while still containing UI errors. Additionally, many of our helper functions return Booleans, and this logic of booleans is used in some instances like our `chat.test.ts`. ## Solution We will remove the majority of try-catch blocks that are used in our helpers to have more stricter errors in our tests. Our tests should fail if an error occurs while finding a UI element. Additionally, we update our `chat.test.ts` so that it can properly stop using the Boolean returned from the chat helper function. We have updated: - generalUtils - cleanupUtils - chat.test.ts - quickActionsHelper - switchModelHelper --- - Treat all work as PUBLIC. Private `feature/x` branches will not be squash-merged at release time. - Your code changes must meet the guidelines in [CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines). - License: I confirm that my contribution is made under the terms of the Apache 2.0 license.
diff --git a/packages/amazonq/test/e2e_new/amazonq/helpers/quickActionsHelper.ts b/packages/amazonq/test/e2e_new/amazonq/helpers/quickActionsHelper.ts
@@ -13,62 +13,39 @@ import { sleep, waitForElements } from '../utils/generalUtils'
  * @returns Promise<{items: WebElement[], texts: string[]}> Array of menu items and their text labels
  */
 export async function getQuickActionsCommands(webview: WebviewView): Promise<{ items: WebElement[]; texts: string[] }> {
-    try {
-        await writeToChat('/', webview, false)
-        await sleep(2000)
+    await writeToChat('/', webview, false)
+    await sleep(2000)
 
-        const menuItems = await waitForElements(
-            webview,
-            By.css('.mynah-detailed-list-item.mynah-ui-clickable-item.target-command'),
-            10000
-        )
+    const menuItems = await waitForElements(
+        webview,
+        By.css('.mynah-detailed-list-item.mynah-ui-clickable-item.target-command'),
+        10000
+    )
 
-        const menuTexts = []
-        for (let i = 0; i < menuItems.length; i++) {
-            try {
-                const text = await menuItems[i].getText()
-                menuTexts.push(text)
-                console.log(`Command ${i + 1}: ${text}`)
-            } catch (e) {
-                menuTexts.push('')
-                console.log(`Could not get text for command ${i + 1}`)
-            }
-        }
-
-        console.log(`Found ${menuItems.length} quick action command items`)
-        return { items: menuItems, texts: menuTexts }
-    } catch (e) {
-        console.error('Error getting quick action commands:', e)
-        return { items: [], texts: [] }
+    const menuTexts = []
+    for (let i = 0; i < menuItems.length; i++) {
+        const text = await menuItems[i].getText()
+        menuTexts.push(text)
     }
+
+    return { items: menuItems, texts: menuTexts }
 }
 
 /**
  * Clicks a specific quick action command by name
  * @param webview The WebviewView instance
  * @param commandName The name of the command to click
- * @returns Promise<boolean> True if command was found and clicked, false otherwise
  */
-export async function clickQuickActionsCommand(webview: WebviewView, commandName: string): Promise<boolean> {
-    try {
-        const { items, texts } = await getQuickActionsCommands(webview)
-        if (items.length === 0) {
-            console.log('No quick action commands found to click')
-            return false
-        }
-        const indexToClick = texts.findIndex((text) => text === commandName)
+export async function clickQuickActionsCommand(webview: WebviewView, commandName: string): Promise<void> {
+    const { items, texts } = await getQuickActionsCommands(webview)
+    if (items.length === 0) {
+        throw new Error('No quick action commands found')
+    }
+    const indexToClick = texts.findIndex((text) => text === commandName)
 
-        if (indexToClick === -1) {
-            console.log(`Command "${commandName}" not found`)
-            return false
-        }
-        console.log(`Clicking on command: ${commandName}`)
-        await items[indexToClick].click()
-        await sleep(3000)
-        console.log('Command clicked successfully')
-        return true
-    } catch (e) {
-        console.error('Error clicking quick action command:', e)
-        return false
+    if (indexToClick === -1) {
+        throw new Error(`Command "${commandName}" not found`)
     }
+    await items[indexToClick].click()
+    await sleep(3000)
 }
diff --git a/packages/amazonq/test/e2e_new/amazonq/helpers/switchModelHelper.ts b/packages/amazonq/test/e2e_new/amazonq/helpers/switchModelHelper.ts
@@ -10,16 +10,10 @@ import { waitForElement } from '../utils/generalUtils'
  * @param webviewView The WebviewView instance
  */
 export async function listModels(webviewView: WebviewView): Promise<void> {
-    try {
-        const selectElement = await waitForElement(webviewView, By.css('.mynah-form-input.auto-width'))
-        const options = await selectElement.findElements(By.css('option'))
-        const optionTexts = await Promise.all(options.map(async (option) => await option.getText()))
-
-        console.log('Available model options:', optionTexts)
-    } catch (e) {
-        console.error('Error listing model options:', e)
-        throw e
-    }
+    const selectElement = await waitForElement(webviewView, By.css('.mynah-form-input.auto-width'))
+    const options = await selectElement.findElements(By.css('option'))
+    const optionTexts = await Promise.all(options.map(async (option) => await option.getText()))
+    console.log('Available model options:', optionTexts)
 }
 
 /**
@@ -28,25 +22,19 @@ export async function listModels(webviewView: WebviewView): Promise<void> {
  * @param modelName The exact name of the model to select
  */
 export async function selectModel(webviewView: WebviewView, modelName: string): Promise<void> {
-    try {
-        const selectElement = await waitForElement(webviewView, By.css('.mynah-form-input.auto-width'))
-        await selectElement.click()
-        const options = await selectElement.findElements(By.css('option'))
-        let targetOption: WebElement | undefined
-        for (const option of options) {
-            const optionText = await option.getText()
-            if (optionText === modelName) {
-                targetOption = option
-                break
-            }
+    const selectElement = await waitForElement(webviewView, By.css('.mynah-form-input.auto-width'))
+    await selectElement.click()
+    const options = await selectElement.findElements(By.css('option'))
+    let targetOption: WebElement | undefined
+    for (const option of options) {
+        const optionText = await option.getText()
+        if (optionText === modelName) {
+            targetOption = option
+            break
         }
-        if (!targetOption) {
-            throw new Error(`Model option "${modelName}" not found`)
-        }
-        await targetOption.click()
-        console.log(`Selected model option: ${modelName}`)
-    } catch (e) {
-        console.error('Error selecting model option:', e)
-        throw e
     }
+    if (!targetOption) {
+        throw new Error(`Model option "${modelName}" not found`)
+    }
+    await targetOption.click()
 }
diff --git a/packages/amazonq/test/e2e_new/amazonq/tests/chat.test.ts b/packages/amazonq/test/e2e_new/amazonq/tests/chat.test.ts
@@ -23,11 +23,7 @@ describe('Amazon Q Chat Basic Functionality', function () {
 
     it('Allows User to Chat with AmazonQ', async () => {
         await writeToChat('Hello, Amazon Q!', webviewView)
-        const responseReceived = await waitForChatResponse(webviewView)
-        if (!responseReceived) {
-            throw new Error('Chat response not received within timeout')
-        }
-        console.log('Chat response detected successfully')
+        await waitForChatResponse(webviewView)
     })
     it('Allows User to Add Multiple Chat Tabs', async () => {
         console.log('Starting Multiple Chat Test')
diff --git a/packages/amazonq/test/e2e_new/amazonq/utils/cleanupUtils.ts b/packages/amazonq/test/e2e_new/amazonq/utils/cleanupUtils.ts
@@ -8,55 +8,41 @@ import { sleep } from './generalUtils'
 /**
  * Closes all open chat tabs
  * @param webview The WebviewView instance
- * @returns Promise<boolean> True if all tabs were successfully closed
  * @throws Error if tabs could not be closed
  */
-export async function closeAllTabs(webview: WebviewView): Promise<boolean> {
-    try {
-        const closeButtons = await webview.findWebElements(By.css('.mynah-tabs-close-button'))
+export async function closeAllTabs(webview: WebviewView): Promise<void> {
+    const closeButtons = await webview.findWebElements(By.css('.mynah-tabs-close-button'))
 
-        for (const button of closeButtons) {
-            await button.click()
-            await sleep(500)
-        }
+    for (const button of closeButtons) {
+        await button.click()
+        await sleep(500)
+    }
 
-        const tabsContainer = await webview.findWebElements(By.css('.mynah-tabs-container'))
-        const allClosed =
-            tabsContainer.length === 1 ||
-            (await tabsContainer[0].findElements(By.css('.mynah-tab-item-label'))).length === 0
+    const tabsContainer = await webview.findWebElements(By.css('.mynah-tabs-container'))
+    const allClosed =
+        tabsContainer.length === 1 ||
+        (await tabsContainer[0].findElements(By.css('.mynah-tab-item-label'))).length === 0
 
-        if (allClosed) {
-            console.log('All chat tabs successfully closed')
-            return true
-        } else {
-            throw new Error('Failed to close all tabs')
-        }
-    } catch (error) {
-        console.error('Error closing tabs:', error)
-        throw error
+    if (!allClosed) {
+        throw new Error('Failed to close all tabs')
     }
 }
 
 /**
  * Attempts to dismiss any open overlays
  * @param webview The WebviewView instance
- * @returns Promise<boolean> True if overlay was dismissed or none was present, false if dismissal failed
+ * @throws Error if overlay dismissal failed
  */
-export async function dismissOverlayIfPresent(webview: WebviewView): Promise<boolean> {
-    try {
-        const overlays = await webview.findWebElements(By.css('.mynah-overlay.mynah-overlay-open'))
-        if (overlays.length > 0) {
-            console.log('Overlay detected, attempting to dismiss...')
-            const driver = webview.getDriver()
-            await driver.executeScript('document.body.click()')
+export async function dismissOverlayIfPresent(webview: WebviewView): Promise<void> {
+    const overlays = await webview.findWebElements(By.css('.mynah-overlay.mynah-overlay-open'))
+    if (overlays.length > 0) {
+        const driver = webview.getDriver()
+        await driver.executeScript('document.body.click()')
 
-            await sleep(1000)
-            const overlaysAfter = await webview.findWebElements(By.css('.mynah-overlay.mynah-overlay-open'))
-            return overlaysAfter.length === 0
+        await sleep(1000)
+        const overlaysAfter = await webview.findWebElements(By.css('.mynah-overlay.mynah-overlay-open'))
+        if (overlaysAfter.length > 0) {
+            throw new Error('Failed to dismiss overlay')
         }
-        return true
-    } catch (e) {
-        console.log('Error while trying to dismiss overlay:', e)
-        return false
     }
 }
diff --git a/packages/amazonq/test/e2e_new/amazonq/utils/generalUtils.ts b/packages/amazonq/test/e2e_new/amazonq/utils/generalUtils.ts
@@ -132,25 +132,23 @@ export async function pressShortcut(driver: WebDriver, ...keys: (string | keyof
  * @param prompt The text to write in the chat input
  * @param webview The WebviewView instance
  * @param send Whether to click the send button (defaults to true)
- * @returns Promise<boolean> True if successful
  */
-export async function writeToChat(prompt: string, webview: WebviewView, send = true): Promise<boolean> {
+export async function writeToChat(prompt: string, webview: WebviewView, send = true): Promise<void> {
     const chatInput = await waitForElement(webview, By.css('.mynah-chat-prompt-input'))
     await chatInput.sendKeys(prompt)
-    if (send === true) {
+    if (send) {
         const sendButton = await waitForElement(webview, By.css('.mynah-chat-prompt-button'))
         await sendButton.click()
     }
-    return true
 }
 
 /**
  * Waits for a chat response to be generated
  * @param webview The WebviewView instance
  * @param timeout The timeout in milliseconds
- * @returns Promise<boolean> True if a response was detected, false if timeout occurred
+ * @throws Error if timeout occurs before response is detected
  */
-export async function waitForChatResponse(webview: WebviewView, timeout = 8000): Promise<boolean> {
+export async function waitForChatResponse(webview: WebviewView, timeout = 15000): Promise<void> {
     const startTime = Date.now()
 
     while (Date.now() - startTime < timeout) {
@@ -162,34 +160,27 @@ export async function waitForChatResponse(webview: WebviewView, timeout = 8000):
             const chatItems = await latestContainer.findElements(By.css('*'))
 
             if (chatItems.length >= 2) {
-                return true
+                return
             }
         }
         await sleep(500)
     }
 
-    return false
+    throw new Error('Timeout waiting for chat response')
 }
 
 /**
  * Clears the text in the chat input field
  * @param webview The WebviewView instance
- * @returns Promise<boolean> True if successful, false if an error occurred
  */
-export async function clearChatInput(webview: WebviewView): Promise<boolean> {
-    try {
-        const chatInput = await waitForElement(webview, By.css('.mynah-chat-prompt-input'))
-        await chatInput.sendKeys(
-            process.platform === 'darwin'
-                ? '\uE03D\u0061' // Command+A on macOS
-                : '\uE009\u0061' // Ctrl+A on Windows/Linux
-        )
-        await chatInput.sendKeys('\uE003') // Backspace
-        return true
-    } catch (e) {
-        console.error('Error clearing chat input:', e)
-        return false
-    }
+export async function clearChatInput(webview: WebviewView): Promise<void> {
+    const chatInput = await waitForElement(webview, By.css('.mynah-chat-prompt-input'))
+    await chatInput.sendKeys(
+        process.platform === 'darwin'
+            ? '\uE03D\u0061' // Command+A on macOS
+            : '\uE009\u0061' // Ctrl+A on Windows/Linux
+    )
+    await chatInput.sendKeys('\uE003') // Backspace
 }
 
 /**
