Use Gradle retry plugin to reduce pipeline flake (#2472)

- Since we have 16 builds in the pipeline even with pretty stable tests, it's pretty easy to have a failure. We also observe this in PR builds, so:
- Use the Gradle retry plugin to retry twice on test failure
- Our test flake monitoring still works since it still outputs to the log that the test fails

Author: hunterwerlla

build.gradle.kts

@@ -52,6 +52,7 @@ val publishChannel: String by project
 plugins {
     java
     id("de.undercouch.download") apply false
+    id("org.gradle.test-retry") version "1.2.0"
 }
 
 group = "software.aws.toolkits"
@@ -71,6 +72,7 @@ allprojects {
     apply(plugin = "com.adarshr.test-logger")
     apply(plugin = "java")
     apply(plugin = "jacoco")
+    apply(plugin = "org.gradle.test-retry")
 
     java.sourceCompatibility = JavaVersion.VERSION_1_8
     java.targetCompatibility = JavaVersion.VERSION_1_8
@@ -180,6 +182,13 @@ subprojects {
             junitXml.isEnabled = true
             html.isEnabled = true
         }
+
+        retry {
+            failOnPassedAfterRetry.set(false)
+            // If there are 5 failures, don't even attempt a retry
+            maxFailures.set(5)
+            maxRetries.set(2)
+        }
     }
 
     plugins.withType<IdeaPlugin> {

jetbrains-core/src/software/aws/toolkits/jetbrains/core/explorer/DeleteResourceDialog.kt

@@ -19,7 +19,10 @@ class DeleteResourceDialog(
     private val resourceType: String,
     private val resourceName: String
 ) : DialogWrapper(project) {
-    private val deleteResourceConfirmation = JBTextField().apply { emptyText.text = message("delete_resource.confirmation_text") }
+    private val deleteResourceConfirmation = JBTextField().apply {
+        emptyText.text = message("delete_resource.confirmation_text")
+        accessibleContext.accessibleName = message("general.delete_accessible_name")
+    }
     private val warningIcon = JBLabel(Messages.getWarningIcon())
     private val component by lazy {
         panel {

resources/resources/software/aws/toolkits/resources/MessagesBundle.properties

@@ -426,6 +426,7 @@ general.configure_button=Configure
 general.create_button=Create
 general.create_in_progress=Creating...
 general.delete=Delete
+general.delete_accessible_name=Delete confirmation box
 general.execution.canceled=canceled
 general.execution.cli_error=Command did not exist successfully, exit code: {0}\n
 general.execution.failed=failed

ui-tests/build.gradle.kts

@@ -56,4 +56,12 @@ tasks.register<Test>("uiTestCore") {
     useJUnitPlatform {
         includeTags("core")
     }
+
+    // uiTestCore needs its own version of this since it's not part of normal test tasks
+    retry {
+        failOnPassedAfterRetry.set(false)
+        // If there are 5 failures, don't even attempt a retry
+        maxFailures.set(5)
+        maxRetries.set(2)
+    }
 }

ui-tests/tst/software/aws/toolkits/jetbrains/uitests/fixtures/FixtureExtensions.kt

@@ -28,6 +28,11 @@ fun ContainerFixture.fillSingleTextField(text: String) = step("Fill single text
     find<JTextFieldFixture>(byXpath("//div[@class='JTextField']"), Duration.ofSeconds(5)).text = text
 }
 
+fun ContainerFixture.fillDeletionAndConfirm() = step("Fill in delete me and delete") {
+    find<JTextFieldFixture>(byXpath("//div[@accessiblename='Delete confirmation box']"), Duration.ofSeconds(5)).text = "delete me"
+    pressOk()
+}
+
 // There is no function to write text to this directly :(
 fun ContainerFixture.fillSingleJBTextArea(text: String) = step("Fill single JBTextArea with $text") {
     find<JTextAreaFixture>(byXpath("//div[@class='JBTextArea']")).click()

ui-tests/tst/software/aws/toolkits/jetbrains/uitests/tests/CloudFormationBrowserTest.kt

@@ -17,10 +17,9 @@ import software.aws.toolkits.jetbrains.uitests.CoreTest
 import software.aws.toolkits.jetbrains.uitests.extensions.uiTest
 import software.aws.toolkits.jetbrains.uitests.fixtures.IdeaFrame
 import software.aws.toolkits.jetbrains.uitests.fixtures.awsExplorer
-import software.aws.toolkits.jetbrains.uitests.fixtures.fillAllJBTextFields
+import software.aws.toolkits.jetbrains.uitests.fixtures.fillDeletionAndConfirm
 import software.aws.toolkits.jetbrains.uitests.fixtures.findAndClick
 import software.aws.toolkits.jetbrains.uitests.fixtures.idea
-import software.aws.toolkits.jetbrains.uitests.fixtures.pressOk
 import software.aws.toolkits.jetbrains.uitests.fixtures.welcomeFrame
 import java.nio.file.Path
 import java.nio.file.Paths
@@ -91,8 +90,7 @@ class CloudFormationBrowserTest {
                     openExplorerActionMenu(cloudFormation, stack)
                 }
                 findAndClick("//div[@text='$deleteStackText']")
-                fillAllJBTextFields("delete me")
-                pressOk()
+                fillDeletionAndConfirm()
             }
 
             waitForStackDeletion()
@@ -107,7 +105,7 @@ class CloudFormationBrowserTest {
 
     @AfterAll
     fun cleanup() {
-        // Make sure that we delete the stack even if it fails in the UIs
+        log.info("Running final cleanup")
         try {
             cloudFormationClient.deleteStack { it.stackName(stack) }
             waitForStackDeletion()
@@ -124,7 +122,9 @@ class CloudFormationBrowserTest {
 
     private fun waitForStackDeletion() {
         log.info("Waiting for the deletion of stack $stack")
-        cloudFormationClient.waiter().waitUntilStackDeleteComplete { it.stackName(stack) }
+        if (cloudFormationClient.describeStacks { it.stackName(stack) }.hasStacks()) {
+            cloudFormationClient.waiter().waitUntilStackDeleteComplete { it.stackName(stack) }
+        }
         log.info("Finished deleting stack $stack")
     }
 }

ui-tests/tst/software/aws/toolkits/jetbrains/uitests/tests/S3BrowserTest.kt

@@ -28,7 +28,7 @@ import software.aws.toolkits.jetbrains.uitests.fixtures.actionButton
 import software.aws.toolkits.jetbrains.uitests.fixtures.actionMenuItem
 import software.aws.toolkits.jetbrains.uitests.fixtures.awsExplorer
 import software.aws.toolkits.jetbrains.uitests.fixtures.fileBrowser
-import software.aws.toolkits.jetbrains.uitests.fixtures.fillAllJBTextFields
+import software.aws.toolkits.jetbrains.uitests.fixtures.fillDeletionAndConfirm
 import software.aws.toolkits.jetbrains.uitests.fixtures.fillSingleTextField
 import software.aws.toolkits.jetbrains.uitests.fixtures.findAndClick
 import software.aws.toolkits.jetbrains.uitests.fixtures.idea
@@ -193,15 +193,15 @@ class S3BrowserTest {
                     openExplorerActionMenu(S3, bucket)
                 }
                 findAndClick("//div[@text='$deleteBucketText']")
-                fillAllJBTextFields("delete me")
-                pressOk()
+                fillDeletionAndConfirm()
                 waitForS3BucketDeletion()
             }
         }
     }
 
     @AfterAll
     fun cleanup() {
+        log.info("Running final cleanup")
         try {
             s3Client.deleteBucketAndContents(bucket)
             waitForS3BucketDeletion()

ui-tests/tst/software/aws/toolkits/jetbrains/uitests/tests/SQSTest.kt

@@ -28,13 +28,13 @@ import software.aws.toolkits.jetbrains.uitests.fixtures.actionMenuItem
 import software.aws.toolkits.jetbrains.uitests.fixtures.awsExplorer
 import software.aws.toolkits.jetbrains.uitests.fixtures.dialog
 import software.aws.toolkits.jetbrains.uitests.fixtures.fillAllJBTextFields
+import software.aws.toolkits.jetbrains.uitests.fixtures.fillDeletionAndConfirm
 import software.aws.toolkits.jetbrains.uitests.fixtures.fillSingleJBTextArea
 import software.aws.toolkits.jetbrains.uitests.fixtures.fillSingleTextField
 import software.aws.toolkits.jetbrains.uitests.fixtures.findAndClick
 import software.aws.toolkits.jetbrains.uitests.fixtures.findByXpath
 import software.aws.toolkits.jetbrains.uitests.fixtures.idea
 import software.aws.toolkits.jetbrains.uitests.fixtures.pressCreate
-import software.aws.toolkits.jetbrains.uitests.fixtures.pressOk
 import software.aws.toolkits.jetbrains.uitests.fixtures.pressSave
 import software.aws.toolkits.jetbrains.uitests.fixtures.pressYes
 import software.aws.toolkits.jetbrains.uitests.fixtures.rightClick
@@ -220,17 +220,15 @@ class SQSTest {
                         openExplorerActionMenu(sqsNodeLabel, queueName)
                     }
                     actionMenuItem(deleteQueueText).click()
-                    fillAllJBTextFields("delete me")
-                    pressOk()
+                    fillDeletionAndConfirm()
                     client.waitForDeletion(queueName)
                 }
                 step("Delete queue $fifoQueueName") {
                     awsExplorer {
                         openExplorerActionMenu(sqsNodeLabel, fifoQueueName)
                     }
                     actionMenuItem(deleteQueueText).click()
-                    fillAllJBTextFields("delete me")
-                    pressOk()
+                    fillDeletionAndConfirm()
                     client.waitForDeletion(fifoQueueName)
                 }
             }
@@ -240,6 +238,7 @@ class SQSTest {
     @AfterAll
     // Make sure the two queues and sns topic are deleted, and if not, delete them
     fun cleanup() {
+        log.info("Running final cleanup")
         try {
             SqsClient.create().use { client ->
                 client.verifyDeleted(queueName)