Fix UI test coverage (#2549)

Create a jacoco server for the life of our ui tests that listens to coverage from the IDE

Author: abrooksv

build.gradle.kts

@@ -42,6 +42,7 @@ tasks.createRelease.configure {
 
 dependencies {
     aggregateCoverage(project(":intellij"))
+    aggregateCoverage(project(":ui-tests"))
 }
 
 tasks.register("runIde") {

buildSrc/build.gradle.kts

@@ -30,6 +30,7 @@ dependencies {
     testImplementation("com.pinterest.ktlint:ktlint-core:$ktlintVersion")
     testImplementation("com.pinterest.ktlint:ktlint-test:$ktlintVersion")
 
+    implementation("org.jacoco:org.jacoco.core:${JacocoPlugin.DEFAULT_JACOCO_VERSION}")
     implementation("org.gradle:test-retry-gradle-plugin:$gradleRetryPluginVersion")
     implementation("com.adarshr:gradle-test-logger-plugin:$gradleTestLoggerPlugin")
 

buildSrc/src/main/kotlin/software/aws/toolkits/gradle/jacoco/RemoteCoverage.kt

@@ -0,0 +1,109 @@
+// Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package software.aws.toolkits.gradle.jacoco
+
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.services.BuildService
+import org.gradle.api.services.BuildServiceParameters
+import org.gradle.api.tasks.testing.Test
+import org.gradle.testing.jacoco.plugins.JacocoTaskExtension
+import org.jacoco.core.data.ExecutionData
+import org.jacoco.core.data.ExecutionDataWriter
+import org.jacoco.core.data.IExecutionDataVisitor
+import org.jacoco.core.data.ISessionInfoVisitor
+import org.jacoco.core.data.SessionInfo
+import org.jacoco.core.runtime.RemoteControlReader
+import org.jacoco.core.runtime.RemoteControlWriter
+import java.net.ServerSocket
+import java.net.Socket
+import java.util.concurrent.atomic.AtomicBoolean
+
+class RemoteCoverage private constructor(task: Test) {
+    companion object {
+        fun enableRemoteCoverage(task: Test) = RemoteCoverage(task)
+
+        private const val DEFAULT_JACOCO_PORT = 6300
+    }
+
+    init {
+        task.extensions.findByType(JacocoTaskExtension::class.java)?.let {
+            val execFile = it.destinationFile ?: return@let
+
+            val jacocoServer = task.project.gradle.sharedServices.registerIfAbsent("jacocoServer", JacocoServer::class.java) {
+                if (!execFile.exists()) {
+                    task.project.mkdir(execFile.parentFile)
+                    execFile.createNewFile()
+                }
+
+                parameters.execFile.set(execFile)
+            }
+
+            task.doFirst {
+                jacocoServer.get().start()
+            }
+        } ?: task.logger.warn("$task does not have Jacoco enabled on it")
+    }
+
+    abstract class JacocoServer : BuildService<JacocoServer.Params>, AutoCloseable {
+        interface Params : BuildServiceParameters {
+            val execFile: RegularFileProperty
+        }
+
+        private val serverSocket = ServerSocket(DEFAULT_JACOCO_PORT)
+        private val signalShutdown = AtomicBoolean(false)
+
+        private val outputStream = parameters.execFile.asFile.get().outputStream()
+        private val fileWriter = ExecutionDataWriter(outputStream)
+        private val serverThread = Thread {
+            while (!signalShutdown.get()) {
+                val clientSocket = serverSocket.accept()
+                JacocoHandler(clientSocket, fileWriter).start()
+            }
+        }
+
+        fun start() {
+            serverThread.start()
+        }
+
+        override fun close() {
+            signalShutdown.set(true)
+            serverThread.interrupt()
+
+            outputStream.close()
+        }
+    }
+
+    private class JacocoHandler(private val socket: Socket, private val fileWriter: ExecutionDataWriter) : Thread(), ISessionInfoVisitor,
+        IExecutionDataVisitor {
+
+        override fun run() {
+            socket.use {
+                socket.getInputStream().use { input ->
+                    socket.getOutputStream().use { output ->
+
+                        val reader = RemoteControlReader(input)
+                        reader.setSessionInfoVisitor(this)
+                        reader.setExecutionDataVisitor(this)
+
+                        RemoteControlWriter(output)
+
+                        @Suppress("ControlFlowWithEmptyBody")
+                        // Read all the data from jacoco
+                        while (reader.read()) {
+                        }
+                        synchronized(fileWriter) { fileWriter.flush() }
+                    }
+                }
+            }
+        }
+
+        override fun visitSessionInfo(info: SessionInfo) {
+            synchronized(fileWriter) { fileWriter.visitSessionInfo(info) }
+        }
+
+        override fun visitClassExecution(data: ExecutionData) {
+            synchronized(fileWriter) { fileWriter.visitClassExecution(data) }
+        }
+    }
+}

buildSrc/src/main/kotlin/toolkit-intellij-subplugin.gradle.kts

@@ -1,6 +1,7 @@
 // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import org.gradle.testing.jacoco.plugins.JacocoTaskExtension.Output
 import org.jetbrains.intellij.tasks.DownloadRobotServerPluginTask
 import org.jetbrains.intellij.tasks.RunIdeForUiTestTask
 import software.aws.toolkits.gradle.IdeVersions
@@ -9,7 +10,6 @@ import software.aws.toolkits.gradle.findFolders
 import software.aws.toolkits.gradle.intellij
 import software.aws.toolkits.gradle.intellij.ToolkitIntelliJExtension
 import software.aws.toolkits.gradle.intellij.ToolkitIntelliJExtension.IdeFlavor
-import java.time.Instant
 
 val toolkitIntelliJ = project.extensions.create<ToolkitIntelliJExtension>("intellijToolkit")
 
@@ -142,7 +142,7 @@ afterEvaluate {
 
         configure<JacocoTaskExtension> {
             includes = listOf("software.aws.toolkits.*")
-            setDestinationFile(file("$buildDir/jacoco/${Instant.now()}-jacocoUiTests.exec"))
+            output = Output.TCP_CLIENT // Dump to our jacoco server instead of to a file
         }
     }
 }

ui-tests/build.gradle.kts

@@ -1,3 +1,5 @@
+import software.aws.toolkits.gradle.jacoco.RemoteCoverage.Companion.enableRemoteCoverage
+
 // Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
@@ -55,4 +57,12 @@ tasks.register<Test>("uiTestCore") {
     useJUnitPlatform {
         includeTags("core")
     }
+
+    // We disable coverage for the JVM running our UI tests, we are running a TCP server that the sandbox IDE dumps to when it exits
+    // This is transparent to coverageReport creation since the coverage gets associated with this tasks jacoco output
+    configure<JacocoTaskExtension> {
+        isEnabled = false
+    }
+
+    enableRemoteCoverage(this)
 }