ci: UI test for feature availability in chat panel (#5393)

* Using puppeteer for basic tests

* test with auth

* UI tests with Auth

* handle env vars

* feedback 1

* Switch test package

* Modify import

* enable jcef

* more settings

* install apparmor

* install apparmor

* install apparmor

* install apparmor

* install apparmor

* Fix test

* try this setting

* modify 241 config

* detekt

* assertions

* revert to thread sleep

* removed print statement

Author: manodnyab

buildspec/linuxUiTests.yml

@@ -47,7 +47,7 @@ phases:
       - chmod +x gradlew
 
       - ffmpeg -loglevel quiet -nostdin -f x11grab -video_size ${SCREEN_WIDTH}x${SCREEN_HEIGHT} -i ${DISPLAY} -codec:v libx264 -pix_fmt yuv420p -vf drawtext="fontsize=48:box=1:[email protected]:boxborderw=5:fontcolor=white:x=0:y=h-text_h:text='%{gmtime\:%H\\\\\:%M\\\\\:%S}'" -framerate 12 -g 12 /tmp/screen_recording.mp4 &
-      - ./gradlew -PideProfileName=$ALTERNATIVE_IDE_PROFILE_NAME :ui-tests-starter:test coverageReport --console plain --info
+      - ./gradlew -PideProfileName=$ALTERNATIVE_IDE_PROFILE_NAME :ui-tests-starter:uiTest coverageReport --console plain --info
 
   post_build:
     commands:

noop/build.gradle.kts

@@ -3,3 +3,4 @@
 
 // project that does nothing
 tasks.register("test")
+tasks.register("uiTest")

settings.gradle.kts

@@ -100,10 +100,12 @@ include("sandbox-all")
 include("ui-tests-starter")
 when (providers.gradleProperty("ideProfileName").get()) {
     // FIX_WHEN_MIN_IS_242: `tmp-all` test module no longer needed in 242+
-    "2023.3", "2024.1" -> {
+    "2024.1" -> {
         include("tmp-all")
-
-        // only available 242+
+        project(":ui-tests-starter").projectDir = file("noop")
+    }
+    "2024.2" -> {
+        // only available 243+
         project(":ui-tests-starter").projectDir = file("noop")
     }
 }

ui-tests-starter/build.gradle.kts

@@ -9,16 +9,26 @@ plugins {
     id("toolkit-kotlin-conventions")
     id("toolkit-intellij-plugin")
 
-    id("org.jetbrains.intellij.platform.base")
+    id("org.jetbrains.intellij.platform")
 }
 
 val ideProfile = IdeVersions.ideProfile(project)
+val testPlugins by configurations.registering
 
-// Add our source sets per IDE profile version (i.e. src-211)
 sourceSets {
     test {
-        java.srcDirs(findFolders(project, "tst", ideProfile))
-        resources.srcDirs(findFolders(project, "tst-resources", ideProfile))
+        java.setSrcDirs(findFolders(project, "tst-prep", ideProfile))
+        resources.setSrcDirs(findFolders(project, "tst-resources", ideProfile))
+    }
+}
+
+val uiTestSource = sourceSets.create("uiTest") {
+    java.setSrcDirs(findFolders(project, "tst", ideProfile))
+}
+
+idea {
+    module {
+        testSources.from(uiTestSource.allSource.srcDirs)
     }
 }
 
@@ -27,35 +37,57 @@ intellijPlatform {
     instrumentCode = false
 }
 
-tasks.initializeIntellijPlatformPlugin {
-    enabled = false
-}
+val uiTestImplementation by configurations.getting
 
-tasks.verifyPluginProjectConfiguration {
-    runtimeDirectory.set(null as File?)
-    enabled = false
+configurations.getByName(uiTestSource.compileClasspathConfigurationName) {
+    extendsFrom(uiTestImplementation)
 }
 
-val testPlugins by configurations.registering
+configurations.getByName(uiTestSource.runtimeClasspathConfigurationName) {
+    extendsFrom(uiTestImplementation)
+}
 
 dependencies {
     // should really be set by the BOM, but too much work to figure out right now
-    testImplementation("org.kodein.di:kodein-di-jvm:7.20.2")
+    uiTestImplementation("org.kodein.di:kodein-di-jvm:7.20.2")
+    uiTestImplementation(platform(libs.junit5.bom))
+    uiTestImplementation(libs.junit5.jupiter)
+
     intellijPlatform {
-        // shouldn't be needed? but IsolationException
         val version = ideProfile.community.sdkVersion
         intellijIdeaCommunity(version, !version.contains("SNAPSHOT"))
-        testFramework(TestFrameworkType.Starter)
+
+        localPlugin(project(":plugin-core"))
+        testImplementation(project(":plugin-core:core"))
+        testImplementation(project(":plugin-core:jetbrains-community"))
+        testImplementation(testFixtures(project(":plugin-core:jetbrains-community")))
+
+        testFramework(TestFrameworkType.Bundled)
+        testFramework(TestFrameworkType.JUnit5)
+
+        testFramework(TestFrameworkType.Starter, configurationName = uiTestImplementation.name)
     }
 
     testPlugins(project(":plugin-amazonq", "pluginZip"))
     testPlugins(project(":plugin-core", "pluginZip"))
 }
 
 tasks.test {
-    dependsOn(testPlugins)
+    enabled = false
+}
 
-    useJUnitPlatform()
+val prepareAmazonQTest by intellijPlatformTesting.testIde.registering {
+    task {
+        useJUnitPlatform()
+    }
+}
+
+tasks.register<Test>("uiTest") {
+    testClassesDirs = uiTestSource.output.classesDirs
+    classpath = uiTestSource.runtimeClasspath
+
+    dependsOn(prepareAmazonQTest)
+    dependsOn(testPlugins)
 
     systemProperty("ui.test.plugins", testPlugins.get().asPath)
     systemProperty("org.gradle.project.ideProfileName", ideProfile.name)

ui-tests-starter/tst-243+/software/aws/toolkits/jetbrains/uitests/chatTests/AmazonQChatTest.kt

@@ -0,0 +1,99 @@
+// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package software.aws.toolkits.jetbrains.uitests.chatTests
+
+import com.intellij.driver.sdk.waitForProjectOpen
+import com.intellij.ide.starter.ci.CIServer
+import com.intellij.ide.starter.config.ConfigurationStorage
+import com.intellij.ide.starter.di.di
+import com.intellij.ide.starter.driver.engine.runIdeWithDriver
+import com.intellij.ide.starter.ide.IdeProductProvider
+import com.intellij.ide.starter.junit5.hyphenateWithClass
+import com.intellij.ide.starter.models.TestCase
+import com.intellij.ide.starter.project.LocalProjectInfo
+import com.intellij.ide.starter.runner.CurrentTestMethod
+import com.intellij.ide.starter.runner.Starter
+import org.junit.jupiter.api.AfterAll
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import org.kodein.di.DI
+import org.kodein.di.bindSingleton
+import software.aws.toolkits.jetbrains.uitests.TestCIServer
+import software.aws.toolkits.jetbrains.uitests.clearAwsXmlFile
+import software.aws.toolkits.jetbrains.uitests.executePuppeteerScript
+import software.aws.toolkits.jetbrains.uitests.setupTestEnvironment
+import software.aws.toolkits.jetbrains.uitests.useExistingConnectionForTest
+import java.io.File
+import java.nio.file.Path
+import java.nio.file.Paths
+
+class AmazonQChatTest {
+
+    init {
+        di = DI {
+            extend(di)
+            bindSingleton<CIServer>(overrides = true) { TestCIServer }
+            val defaults = ConfigurationStorage.instance().defaults.toMutableMap().apply {
+                put("LOG_ENVIRONMENT_VARIABLES", (!System.getenv("CI").toBoolean()).toString())
+            }
+
+            bindSingleton<ConfigurationStorage>(overrides = true) {
+                ConfigurationStorage(this, defaults)
+            }
+        }
+    }
+
+    @BeforeEach
+    fun setUp() {
+        // Setup test environment
+        setupTestEnvironment()
+    }
+
+    @Test
+    fun `Ensure feature availability on slash`() {
+        val testCase = TestCase(
+            IdeProductProvider.IC,
+            LocalProjectInfo(
+                Paths.get("tstData", "Hello")
+            )
+        ).useRelease(System.getProperty("org.gradle.project.ideProfileName"))
+
+        // inject connection
+        useExistingConnectionForTest()
+
+        Starter.newContext(CurrentTestMethod.hyphenateWithClass(), testCase).apply {
+            System.getProperty("ui.test.plugins").split(File.pathSeparator).forEach { path ->
+                pluginConfigurator.installPluginFromPath(
+                    Path.of(path)
+                )
+            }
+
+            copyExistingConfig(Paths.get("tstData", "configAmazonQTests"))
+            updateGeneralSettings()
+        }.runIdeWithDriver()
+            .useDriverAndCloseIde {
+                waitForProjectOpen()
+                // required wait time for the system to be fully ready
+                Thread.sleep(30000)
+
+                val result = executePuppeteerScript(testFeatureAvailabilityOnSlash)
+                assertTrue(result.contains("/doc"))
+                assertTrue(result.contains("/dev"))
+                assertTrue(result.contains("/transform"))
+                assertTrue(result.contains("/help"))
+                assertTrue(result.contains("/clear"))
+                assertTrue(result.contains("/review"))
+                assertTrue(result.contains("/test"))
+            }
+    }
+
+    companion object {
+        @JvmStatic
+        @AfterAll
+        fun clearAwsXml() {
+            clearAwsXmlFile()
+        }
+    }
+}

ui-tests-starter/tst-243+/software/aws/toolkits/jetbrains/uitests/chatTests/TestCaseScripts.kt

@@ -0,0 +1,36 @@
+// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package software.aws.toolkits.jetbrains.uitests.chatTests
+
+// language=JS
+val testFeatureAvailabilityOnSlash = """
+const puppeteer = require('puppeteer');
+
+async function testNavigation() {
+    const browser = await puppeteer.connect({
+        browserURL: "http://localhost:9222"
+    })
+    try {
+        const pages = await browser.pages()
+        for(const page of pages) {
+            const contents = await page.evaluate(el => el.innerHTML, await page.${'$'}(':root'));
+            const element = await page.$('.mynah-chat-prompt-input')
+            if(element) {
+                await page.type('.mynah-chat-prompt-input', '/')
+                const elements = await page.$$(".mynah-chat-command-selector-command");
+                const attr = await Promise.all(
+                    elements.map(async element => {
+                        return element.evaluate(el => el.getAttribute("command"));
+                    })
+                );
+                console.log(JSON.stringify(attr, null, 2))
+            }
+        }
+    } finally {
+        await browser.close();
+    }
+}
+testNavigation().catch(console.error);
+
+""".trimIndent()

ui-tests-starter/tst-prep/PreAmazonQUiTest.kt

@@ -0,0 +1,51 @@
+// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import com.intellij.openapi.Disposable
+import com.intellij.testFramework.ApplicationExtension
+import com.intellij.testFramework.junit5.TestDisposable
+import org.junit.Rule
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.extension.ExtendWith
+import software.aws.toolkits.core.rules.SystemPropertyHelper
+import software.aws.toolkits.jetbrains.core.credentials.LegacyManagedBearerSsoConnection
+import software.aws.toolkits.jetbrains.core.credentials.ManagedBearerSsoConnection
+import software.aws.toolkits.jetbrains.core.credentials.pinning.ConnectionPinningManager
+import software.aws.toolkits.jetbrains.core.credentials.pinning.QConnection
+import software.aws.toolkits.jetbrains.core.credentials.sono.Q_SCOPES
+import software.aws.toolkits.jetbrains.core.credentials.sso.bearer.BearerTokenProvider
+import software.aws.toolkits.jetbrains.utils.extensions.SsoLogin
+import software.aws.toolkits.jetbrains.utils.extensions.SsoLoginExtension
+
+@ExtendWith(ApplicationExtension::class, SsoLoginExtension::class)
+@SsoLogin("amazonq-test-account")
+class PreAmazonQUiTest {
+
+    @TestDisposable
+    lateinit var disposable: Disposable
+
+    @Rule
+    @JvmField
+    val systemPropertyHelper = SystemPropertyHelper()
+
+    private lateinit var connection: ManagedBearerSsoConnection
+
+    @BeforeEach
+    fun setUp() {
+        System.setProperty("aws.dev.useDAG", "true")
+    }
+
+    @Test
+    fun `can set up Connection`() {
+        try {
+            val startUrl = System.getenv("TEST_START_URL")
+            val region = System.getenv("TEST_REGION")
+            connection = LegacyManagedBearerSsoConnection(startUrl, region, Q_SCOPES)
+            ConnectionPinningManager.getInstance().setPinnedConnection(QConnection.getInstance(), connection)
+            (connection.getConnectionSettings().tokenProvider.delegate as BearerTokenProvider).reauthenticate()
+        } catch (e: Exception) {
+            error("Could not connect to Idc.")
+        }
+    }
+}