diff --git a/ui-tests-starter/tst-243+/software/aws/toolkits/jetbrains/uitests/featureDevTests/FeatureDevTest.kt b/ui-tests-starter/tst-243+/software/aws/toolkits/jetbrains/uitests/featureDevTests/FeatureDevTest.kt new file mode 100644 index 00000000000..038d8593fac --- /dev/null +++ b/ui-tests-starter/tst-243+/software/aws/toolkits/jetbrains/uitests/featureDevTests/FeatureDevTest.kt @@ -0,0 +1,231 @@ +// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.jetbrains.uitests.featureDevTests + +import com.intellij.driver.sdk.waitForProjectOpen +import com.intellij.ide.starter.ci.CIServer +import com.intellij.ide.starter.config.ConfigurationStorage +import com.intellij.ide.starter.di.di +import com.intellij.ide.starter.driver.engine.runIdeWithDriver +import com.intellij.ide.starter.ide.IdeProductProvider +import com.intellij.ide.starter.junit5.hyphenateWithClass +import com.intellij.ide.starter.models.TestCase +import com.intellij.ide.starter.project.LocalProjectInfo +import com.intellij.ide.starter.runner.CurrentTestMethod +import com.intellij.ide.starter.runner.Starter +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.kodein.di.DI +import org.kodein.di.bindSingleton +import software.aws.toolkits.jetbrains.uitests.TestCIServer +import software.aws.toolkits.jetbrains.uitests.clearAwsXmlFile +import software.aws.toolkits.jetbrains.uitests.executePuppeteerScript +import software.aws.toolkits.jetbrains.uitests.setupTestEnvironment +import software.aws.toolkits.jetbrains.uitests.useExistingConnectionForTest +import java.io.File +import java.io.FileOutputStream +import java.nio.file.Path +import java.nio.file.Paths + +class FeatureDevTest { + init { + di = DI { + extend(di) + bindSingleton(overrides = true) { TestCIServer } + val defaults = ConfigurationStorage.instance().defaults.toMutableMap().apply { + put("LOG_ENVIRONMENT_VARIABLES", (!System.getenv("CI").toBoolean()).toString()) + } + + bindSingleton(overrides = true) { + ConfigurationStorage(this, defaults) + } + } + } + + @BeforeEach + fun setUp() { + // Setup test environment + setupTestEnvironment() + } + + @AfterEach + fun resetTestProject() { + val operationFile = Paths.get("tstData", "FeatureDevE2ETestFolder", "operation.js").toFile() + FileOutputStream(operationFile).channel.truncate(0) + + val changelogFile = Paths.get("tstData", "FeatureDevE2ETestFolder", "CHANGELOG.md").toFile() + FileOutputStream(changelogFile).channel.truncate(0) + } + + @Test + fun `Accept initial code generation`() { + val testCase = TestCase( + IdeProductProvider.IC, + LocalProjectInfo( + Paths.get("tstData", "FeatureDevE2ETestFolder") + ) + ).useRelease(System.getProperty("org.gradle.project.ideProfileName")) + + // inject connection + useExistingConnectionForTest() + + Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply { + System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path -> + pluginConfigurator.installPluginFromPath( + Path.of(path) + ) + } + + copyExistingConfig(Paths.get("tstData", "configAmazonQTests")) + updateGeneralSettings() + }.runIdeWithDriver() + .useDriverAndCloseIde { + waitForProjectOpen() + // required wait time for the system to be fully ready + Thread.sleep(30000) + + val result = executePuppeteerScript(testAcceptInitalCode) + assertTrue(result.contains("Success: /dev ends the conversation successfully.")) + } + } + + @Test + fun `Iterate code generation`() { + val testCase = TestCase( + IdeProductProvider.IC, + LocalProjectInfo( + Paths.get("tstData", "FeatureDevE2ETestFolder") + ) + ).useRelease(System.getProperty("org.gradle.project.ideProfileName")) + + // inject connection + useExistingConnectionForTest() + + Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply { + System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path -> + pluginConfigurator.installPluginFromPath( + Path.of(path) + ) + } + + copyExistingConfig(Paths.get("tstData", "configAmazonQTests")) + updateGeneralSettings() + }.runIdeWithDriver() + .useDriverAndCloseIde { + waitForProjectOpen() + // required wait time for the system to be fully ready + Thread.sleep(30000) + + val result = executePuppeteerScript(testIterateCodeGen) + assertTrue(result.contains("Success: /dev ends the conversation successfully.")) + } + } + + @Test + fun `Start new code generation`() { + val testCase = TestCase( + IdeProductProvider.IC, + LocalProjectInfo( + Paths.get("tstData", "FeatureDevE2ETestFolder") + ) + ).useRelease(System.getProperty("org.gradle.project.ideProfileName")) + + // inject connection + useExistingConnectionForTest() + + Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply { + System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path -> + pluginConfigurator.installPluginFromPath( + Path.of(path) + ) + } + + copyExistingConfig(Paths.get("tstData", "configAmazonQTests")) + updateGeneralSettings() + }.runIdeWithDriver() + .useDriverAndCloseIde { + waitForProjectOpen() + // required wait time for the system to be fully ready + Thread.sleep(30000) + + val result = executePuppeteerScript(testNewCodeGen) + assertTrue(result.contains("Success: /dev ends the conversation successfully.")) + } + } + + @Test + fun `Accept partial code generation`() { + val testCase = TestCase( + IdeProductProvider.IC, + LocalProjectInfo( + Paths.get("tstData", "FeatureDevE2ETestFolder") + ) + ).useRelease(System.getProperty("org.gradle.project.ideProfileName")) + + // inject connection + useExistingConnectionForTest() + + Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply { + System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path -> + pluginConfigurator.installPluginFromPath( + Path.of(path) + ) + } + + copyExistingConfig(Paths.get("tstData", "configAmazonQTests")) + updateGeneralSettings() + }.runIdeWithDriver() + .useDriverAndCloseIde { + waitForProjectOpen() + // required wait time for the system to be fully ready + Thread.sleep(30000) + + val result = executePuppeteerScript(testPartialCodeGen) + assertTrue(result.contains("Success: /dev ends the conversation successfully.")) + } + } + + @Test + fun `Stop and restart code generation`() { + val testCase = TestCase( + IdeProductProvider.IC, + LocalProjectInfo( + Paths.get("tstData", "FeatureDevE2ETestFolder") + ) + ).useRelease(System.getProperty("org.gradle.project.ideProfileName")) + + // inject connection + useExistingConnectionForTest() + + Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply { + System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path -> + pluginConfigurator.installPluginFromPath( + Path.of(path) + ) + } + + copyExistingConfig(Paths.get("tstData", "configAmazonQTests")) + updateGeneralSettings() + }.runIdeWithDriver() + .useDriverAndCloseIde { + waitForProjectOpen() + // required wait time for the system to be fully ready + Thread.sleep(30000) + + val result = executePuppeteerScript(testStopAndRestartCodeGen) + assertTrue(result.contains("Success: /dev ends the conversation successfully.")) + } + } + + companion object { + @JvmStatic + @AfterAll + fun clearAwsXml() { + clearAwsXmlFile() + } + } +} diff --git a/ui-tests-starter/tst-243+/software/aws/toolkits/jetbrains/uitests/featureDevTests/TestCaseScripts.kt b/ui-tests-starter/tst-243+/software/aws/toolkits/jetbrains/uitests/featureDevTests/TestCaseScripts.kt new file mode 100644 index 00000000000..fad222fea87 --- /dev/null +++ b/ui-tests-starter/tst-243+/software/aws/toolkits/jetbrains/uitests/featureDevTests/TestCaseScripts.kt @@ -0,0 +1,253 @@ +// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package software.aws.toolkits.jetbrains.uitests.featureDevTests + +import software.aws.toolkits.jetbrains.uitests.testScriptPrefix + +val testAcceptInitalCode = """ +$testScriptPrefix + +async function testAcceptInitalCode() { + const browser = await puppeteer.connect({ + browserURL: "http://localhost:9222", + protocolTimeout: 750_000, + }); + + const page = (await browser.pages()).find((p) => p.url().startsWith('file:')); + + try { + const contents = await page.evaluate(el => el.innerHTML, await page.${'$'}(':root')); + await page.waitForSelector('.mynah-chat-prompt-input'); + await page.type('.mynah-chat-prompt-input', '/dev implement a debounce function in operation.js') + await page.keyboard.press('Enter'); + + await retryIfRequired(page, async () => { + await page.waitForSelector('.mynah-chat-item-followup-question-option ::-p-text(Accept all changes)', {timeout: 600_000}); + const [acceptCode, iterateCode] = await page.$$('.mynah-chat-item-followup-question-option'); + await acceptCode.click(); + + await page.waitForSelector('.mynah-chat-item-followup-question-option'); + const [newTask, noThanks, generateDevFile] = await page.$$('.mynah-chat-item-followup-question-option'); + await noThanks.click(); + }); + + // Validate if /dev ends the conversation successfully. + await page.waitForSelector('p ::-p-text(Okay, I\'ve ended this chat session. You can open a new tab to chat or start another workflow.)', {timeout: 3000}); + console.log('Success: /dev ends the conversation successfully.'); + + } finally { + await closeSelectedTab(page); + await browser.close(); + } +} + +testAcceptInitalCode().catch(console.error); + +""".trimIndent() + +val testIterateCodeGen = """ +$testScriptPrefix + +async function testIterateCodeGen() { + const browser = await puppeteer.connect({ + browserURL: "http://localhost:9222", + protocolTimeout: 750_000, + }); + + const page = (await browser.pages()).find((p) => p.url().startsWith('file:')); + + try { + await page.evaluate(el => el.innerHTML, await page.${'$'}(':root')); + await page.waitForSelector('.mynah-chat-prompt-input'); + await page.type('.mynah-chat-prompt-input', '/dev') + await page.keyboard.press('Enter'); + + await page.type('.mynah-chat-prompt-input', 'Add debounce function in operation.js.') + await page.keyboard.press('Enter'); + + // First iteration + await retryIfRequired(page, async () => { + await page.waitForSelector('.mynah-chat-item-followup-question-option ::-p-text(Accept all changes)', {timeout: 600_000}); + const [acceptCode, iterateCode] = await page.$$('.mynah-chat-item-followup-question-option'); + await iterateCode.click(); + }); + + // Second iteration + await page.type('.mynah-chat-prompt-input', 'Also add throttle function in operation.js.'); + await page.keyboard.press('Enter'); + + await retryIfRequired(page, async () => { + await page.waitForSelector('.mynah-chat-item-followup-question-option ::-p-text(Accept all changes)', {timeout: 600_000}); + const [acceptCode, iterateCode] = await page.$$('.mynah-chat-item-followup-question-option'); + await acceptCode.click(); + + await page.waitForSelector('.mynah-chat-item-followup-question-option'); + const [newTask, noThanks, generateDevFile] = await page.$$('.mynah-chat-item-followup-question-option'); + await noThanks.click(); + }); + + // Validate if /dev ends the conversation successfully. + await page.waitForSelector('p ::-p-text(Okay, I\'ve ended this chat session. You can open a new tab to chat or start another workflow.)', {timeout: 3000}); + console.log('Success: /dev ends the conversation successfully.'); + + } finally { + await closeSelectedTab(page); + await browser.close(); + } +} + +testIterateCodeGen().catch(console.error); +""".trimIndent() + +val testNewCodeGen = """ +$testScriptPrefix + +async function testNewCodeGen() { + const browser = await puppeteer.connect({ + browserURL: "http://localhost:9222", + protocolTimeout: 750_000, + }); + + const page = (await browser.pages()).find((p) => p.url().startsWith('file:')); + + try { + await page.evaluate(el => el.innerHTML, await page.${'$'}(':root')); + await page.waitForSelector('.mynah-chat-prompt-input'); + await page.type('.mynah-chat-prompt-input', '/dev') + await page.keyboard.press('Enter'); + + // Initial task + await page.type('.mynah-chat-prompt-input', 'Add debounce function in operation.js.') + await page.keyboard.press('Enter'); + + await retryIfRequired(page, async () => { + await page.waitForSelector('.mynah-chat-item-followup-question-option ::-p-text(Accept all changes)', {timeout: 600_000}); + const [acceptCode, iterateCode] = await page.$$('.mynah-chat-item-followup-question-option'); + await acceptCode.click(); + await page.waitForSelector('.mynah-chat-item-followup-question-option'); + const [newTask, noThanks, generateDevFile] = await page.$$('.mynah-chat-item-followup-question-option'); + await newTask.click(); + }); + + // New task + await page.type('.mynah-chat-prompt-input', 'Add throttle function in operation.js.') + await page.keyboard.press('Enter'); + + await retryIfRequired(page, async () => { + await page.waitForSelector('.mynah-chat-item-followup-question-option ::-p-text(Accept all changes)', {timeout: 600_000}); + const [acceptCode, iterateCode] = await page.$$('.mynah-chat-item-followup-question-option'); + await acceptCode.click(); + await page.waitForSelector('.mynah-chat-item-followup-question-option'); + const [newTask, noThanks, generateDevFile] = await page.$$('.mynah-chat-item-followup-question-option'); + await noThanks.click(); + }); + + // Validate if /dev ends the conversation successfully. + await page.waitForSelector('p ::-p-text(Okay, I\'ve ended this chat session. You can open a new tab to chat or start another workflow.)', {timeout: 3000}); + console.log('Success: /dev ends the conversation successfully.'); + } + finally { + await closeSelectedTab(page); + await browser.close(); + } +} +testNewCodeGen().catch(console.error); +""".trimIndent() + +val testPartialCodeGen = """ +$testScriptPrefix + +async function testPartialCodeGen() { + const browser = await puppeteer.connect({ + browserURL: "http://localhost:9222", + protocolTimeout: 750_000, + }); + + const page = (await browser.pages()).find((p) => p.url().startsWith('file:')); + + try { + // Ensure page is ready to evaluate + await page.evaluate(el => el.innerHTML, await page.${'$'}(':root')); + await page.waitForSelector('.mynah-chat-prompt-input'); + + // Ensure prompt input is visible + await page.waitForSelector('.mynah-chat-prompt-input'); + + // Enter initial prompt + await page.type('.mynah-chat-prompt-input', '/dev Add debounce function in operation.js and add explain what you did in CHANGELOG.md') + await page.keyboard.press('Enter'); + + await retryIfRequired(page, async () => { + // Ensure tree view is visiable and de-select a modified file + await page.waitForSelector('.mynah-chat-item-tree-view-file-item-actions .error', {timeout: 600_000}); + await page.locator('.mynah-chat-item-tree-view-file-item-actions .error').click(); + + // Accept code change + await page.waitForSelector('.mynah-chat-item-followup-question-option ::-p-text(Accept remaining changes)', {timeout: 3000}); + const [acceptCode, iterateCode] = await page.$$('.mynah-chat-item-followup-question-option'); + await acceptCode.click(); + + // End the conversation + await page.waitForSelector('.mynah-chat-item-followup-question-option'); + const [newTask, noThanks, generateDevFile] = await page.$$('.mynah-chat-item-followup-question-option'); + await noThanks.click(); + }); + + // Validate if /dev ends the conversation successfully + await page.waitForSelector('p ::-p-text(Okay, I\'ve ended this chat session. You can open a new tab to chat or start another workflow.)', {timeout: 3000}); + console.log('Success: /dev ends the conversation successfully.'); + + } finally { + // Close current tab before disconnect browser + await closeSelectedTab(page); + await browser.close(); + } +} + +testPartialCodeGen().catch(console.error); +""".trimIndent() + +val testStopAndRestartCodeGen = """ +$testScriptPrefix + +async function testStopAndRestartCodeGen() { + const browser = await puppeteer.connect({ + browserURL: "http://localhost:9222", + protocolTimeout: 750_000, + }); + + const page = (await browser.pages()).find((p) => p.url().startsWith('file:')); + + try { + const contents = await page.evaluate(el => el.innerHTML, await page.${'$'}(':root')); + await page.waitForSelector('.mynah-chat-prompt-input'); + await page.type('.mynah-chat-prompt-input', '/dev Add debounce function in operation.js.') + await page.keyboard.press('Enter'); + + const stopBtn = await page.${'$'}('.loading .mynah-chat-stop-chat-response-button', {visible: true}); + await stopBtn.click(); + + await retryIfRequired(page, async () => { + await page.waitForSelector('.mynah-chat-item-followup-question-option ::-p-text(Accept all changes)', {timeout: 600_000}); + const [acceptCode, iterateCode] = await page.$$('.mynah-chat-item-followup-question-option'); + await acceptCode.click(); + + await page.waitForSelector('.mynah-chat-item-followup-question-option'); + const [newTask, noThanks, generateDevFile] = await page.$$('.mynah-chat-item-followup-question-option'); + await noThanks.click(); + }); + + // Validate if /dev ends the conversation successfully. + await page.waitForSelector('p ::-p-text(Okay, I\'ve ended this chat session. You can open a new tab to chat or start another workflow.)', {timeout: 3000}); + console.log('Success: /dev ends the conversation successfully.'); + + } finally { + await closeSelectedTab(page); + await browser.close(); + } +} + +testStopAndRestartCodeGen().catch(console.error); + +""".trimIndent() diff --git a/ui-tests-starter/tst/software/aws/toolkits/jetbrains/uitests/TestUtils.kt b/ui-tests-starter/tst/software/aws/toolkits/jetbrains/uitests/TestUtils.kt index 65de953f7ee..9705df5d735 100644 --- a/ui-tests-starter/tst/software/aws/toolkits/jetbrains/uitests/TestUtils.kt +++ b/ui-tests-starter/tst/software/aws/toolkits/jetbrains/uitests/TestUtils.kt @@ -102,3 +102,43 @@ fun writeToAwsXml(configContent: String) { StandardOpenOption.TRUNCATE_EXISTING ) } + +val testScriptPrefix = """ +const puppeteer = require('puppeteer'); + +async function retryIfRequired(page, request) { + let retryCounts = 0; + const maxRetries = 3; + + async function retry() { + + await page.waitForSelector('::-p-text(Retry)', {timeout: 600_000}) + + const [retry] = await page.$$('.mynah-chat-item-followup-question-option'); + await retry.click(); + + console.log(`Retrying ${'$'}{++retryCounts} time(s)`); + } + + while (retryCounts < maxRetries) { + const currRetryCounts = retryCounts; + await Promise.race([ + retry(), + request() + ]); + + // This indicates request proceed successfully. There is no more need to retry. + if (retryCounts === currRetryCounts) { + return; + } + } +} + +async function closeSelectedTab(page) { + const selectedTabId = await page.${'$'}eval('.mynah-nav-tabs-wrapper', (el) => el.getAttribute('selected-tab')); + + const closeBtn = await page.${'$'}(`span[key=mynah-main-tabs-${'$'}{selectedTabId}] label button`); + await closeBtn.click(); +} + +""".trimIndent() diff --git a/ui-tests-starter/tstData/FeatureDevE2ETestFolder/CHANGELOG.md b/ui-tests-starter/tstData/FeatureDevE2ETestFolder/CHANGELOG.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ui-tests-starter/tstData/FeatureDevE2ETestFolder/operation.js b/ui-tests-starter/tstData/FeatureDevE2ETestFolder/operation.js new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ui-tests-starter/tstData/configAmazonQTests/options/ide.general.xml b/ui-tests-starter/tstData/configAmazonQTests/options/ide.general.xml index 9cd270cc445..99ab6758018 100644 --- a/ui-tests-starter/tstData/configAmazonQTests/options/ide.general.xml +++ b/ui-tests-starter/tstData/configAmazonQTests/options/ide.general.xml @@ -1,6 +1,6 @@ - +