Fix: Coverage ignored when executeAndroidTests is false and running on Gradle Managed Devices (#104)

When `executeAndroidTests` is set to  false and the Android integration tests have run manually using Gradle Managed Devices the coverage results are not picked-up by the `rootCoverageReport` and/or `coverageReport` tasks, this is now fixed.

---------

Co-authored-by: pavel.krylov <[email protected]>
Co-authored-by: Rolf Smit <[email protected]>

Author: 3 people

plugin/src/main/kotlin/org/neotech/plugin/rootcoverage/RootCoveragePlugin.kt

@@ -11,6 +11,7 @@ import org.gradle.api.GradleException
 import org.gradle.api.NamedDomainObjectContainer
 import org.gradle.api.Plugin
 import org.gradle.api.Project
+import org.gradle.api.Task
 import org.gradle.api.tasks.TaskProvider
 import org.gradle.api.tasks.testing.Test
 import org.gradle.testing.jacoco.plugins.JacocoPlugin
@@ -64,7 +65,7 @@ class RootCoveragePlugin : Plugin<Project> {
     }
 
     private fun createCoverageTaskForRoot(project: Project) {
-        val task = project.createJacocoReportTask(
+        val rootCoverageTask = project.createJacocoReportTask(
             taskName = "rootCoverageReport",
             taskGroup = "reporting",
             taskDescription = "Generates a Jacoco report with combined results from all the subprojects.",
@@ -76,11 +77,12 @@ class RootCoveragePlugin : Plugin<Project> {
         }
 
         // Configure the root task with sub-tasks for the sub-projects.
-        task.project.subprojects.forEach {
+        rootCoverageTask.project.subprojects.forEach {
             it.afterEvaluate { subProject ->
                 subProject.applyConfiguration()
             }
-            task.addSubProject(it)
+            rootCoverageTask.addSubProject(it)
+
             createSubProjectCoverageTask(it)
         }
 
@@ -136,47 +138,28 @@ class RootCoveragePlugin : Plugin<Project> {
     }
 
     private fun JacocoReport.addSubProjectVariant(subProject: Project, variant: Variant, buildType: BuildType) {
-        val name = variant.name.replaceFirstChar(Char::titlecase)
+        val variantName = variant.name.replaceFirstChar(Char::titlecase)
 
         // Gets the relative path from this task to the subProject
         val path = project.relativeProjectPath(subProject.path)
 
         // Add dependencies to the test tasks of the subProject
         if (rootProjectExtension.shouldExecuteUnitTests() && (buildType.enableUnitTestCoverage || buildType.isTestCoverageEnabled)) {
-            dependsOn("$path:test${name}UnitTest")
+            dependsOn("$path:test${variantName}UnitTest")
         }
 
-        var runsOnGradleManagedDevices = false
+        val androidTestTask = getAndroidTestTask(subProject, variant)
+        val runsOnGradleManagedDevices = androidTestTask.runsOnGradleManagedDevices
 
         if (rootProjectExtension.shouldExecuteAndroidTests() && (buildType.enableAndroidTestCoverage || buildType.isTestCoverageEnabled)) {
-
-            // Attempt to run on instrumented tests, giving priority to the following devices in this order:
-            // - A user provided Gradle Managed Device.
-            // - All Gradle Managed Devices if any is available.
-            // - All through ADB connected devices.
-            val gradleManagedDevices = subProject.extensions.getByType(BaseExtension::class.java).testOptions.managedDevices.devices
-            if (rootProjectExtension.runOnGradleManagedDevices && !rootProjectExtension.gradleManagedDeviceName.isNullOrEmpty()) {
-                runsOnGradleManagedDevices = true
-                dependsOn("$path:${rootProjectExtension.gradleManagedDeviceName}${name}AndroidTest")
-            } else if (rootProjectExtension.runOnGradleManagedDevices && gradleManagedDevices.isNotEmpty()) {
-                runsOnGradleManagedDevices = true
-                dependsOn("$path:allDevices${name}AndroidTest")
-            } else {
-                dependsOn("$path:connected${name}AndroidTest")
-            }
+            dependsOn(androidTestTask.taskPath)
         } else {
             // If this plugin should not run instrumented tests on it's own, at least make sure it runs after those tasks (if they are
             // selected to run as well and exists).
             //
             // In theory we don't need to do this if `rootProjectExtension.includeAndroidTestResults` is false, so we could check that, but
             // it also does not hurt.
-
-            val executeAndroidTestsOnGradleManagedDevicesTask = project.tasks.findByPath("$path:allDevices${name}AndroidTest")
-            if(executeAndroidTestsOnGradleManagedDevicesTask != null) {
-                // This task only exists if a Gradle Managed Device is configured, which may not be the case.
-                mustRunAfter("$path:allDevices${name}AndroidTest")
-            }
-            mustRunAfter("$path:connected${name}AndroidTest")
+            mustRunAfter(androidTestTask.taskPath)
         }
 
         sourceDirectories.from(variant.sources.java?.all)
@@ -200,6 +183,41 @@ class RootCoveragePlugin : Plugin<Project> {
         )
     }
 
+    private class AndroidTestTask(
+        val taskPath: String,
+        val runsOnGradleManagedDevices: Boolean
+    )
+
+    private fun JacocoReport.getAndroidTestTask(subProject: Project, variant: Variant): AndroidTestTask {
+        // Gets the relative path from this task to the subProject
+        val path = project.relativeProjectPath(subProject.path)
+        val variantName = variant.name.replaceFirstChar(Char::titlecase)
+
+
+        // Attempt to run on instrumented tests, giving priority to the following devices in this order:
+        // - A user provided Gradle Managed Device.
+        // - All Gradle Managed Devices if any is available.
+        // - All through ADB connected devices.
+        val gradleManagedDevices = subProject.extensions.getByType(BaseExtension::class.java).testOptions.managedDevices.devices
+
+        if (rootProjectExtension.runOnGradleManagedDevices && !rootProjectExtension.gradleManagedDeviceName.isNullOrEmpty()) {
+            return AndroidTestTask(
+                taskPath = "$path:${rootProjectExtension.gradleManagedDeviceName}${variantName}AndroidTest",
+                runsOnGradleManagedDevices = true
+            )
+        } else if (rootProjectExtension.runOnGradleManagedDevices && gradleManagedDevices.isNotEmpty()) {
+            return AndroidTestTask(
+                taskPath = "$path:allDevices${variantName}AndroidTest",
+                runsOnGradleManagedDevices = true
+            )
+        } else {
+            return AndroidTestTask(
+                taskPath = "$path:connected${variantName}AndroidTest",
+                runsOnGradleManagedDevices = false
+            )
+        }
+    }
+
     /**
      * Apply configuration from [RootCoveragePluginExtension] to the project.
      */

plugin/src/test/kotlin/org/neotech/plugin/rootcoverage/IntegrationTest.kt

@@ -11,6 +11,7 @@ import org.junit.runners.Parameterized
 import org.neotech.plugin.rootcoverage.util.SimpleTemplate
 import org.neotech.plugin.rootcoverage.util.SystemOutputWriter
 import org.neotech.plugin.rootcoverage.util.assertSuccessful
+import org.neotech.plugin.rootcoverage.util.assertTaskNotExecuted
 import org.neotech.plugin.rootcoverage.util.assertTaskSuccess
 import org.neotech.plugin.rootcoverage.util.createGradlePropertiesFile
 import org.neotech.plugin.rootcoverage.util.createLocalPropertiesFile
@@ -86,28 +87,36 @@ class IntegrationTest(
             })
         })
 
+        val executeAndroidTests = configuration.pluginConfiguration.getPropertyValue("executeAndroidTests", "true").toBoolean()
+
         // Note: rootCodeCoverageReport is the old and deprecated name of the rootCoverageReport task, it is
         // used to check whether the old name properly aliases to the new task name.
-        val gradleCommands = if (configuration.pluginConfiguration.getPropertyValue("executeAndroidTests") == "false") {
-            listOf("clean", "connectedDebugAndroidTest", "coverageReport", "rootCodeCoverageReport", "--stacktrace")
+        val gradleCommands = if (!executeAndroidTests) {
+            val runOnGradleManagedDevices = configuration.pluginConfiguration.getPropertyValue("runOnGradleManagedDevices") ?: "false"
+
+            // Execute Android tests completely separately (as if run on some external service,
+            // after which the resulting files have been imported)
+            if (runOnGradleManagedDevices == "false") {
+                executeGradleTasks(listOf("clean", "connectedDebugAndroidTest"))
+            } else {
+                executeGradleTasks(listOf("clean", "nexusoneapi30DebugAndroidTest"))
+            }
+
+            listOf("coverageReport", "rootCodeCoverageReport", "--stacktrace")
         } else {
             listOf("clean", "coverageReport", "rootCodeCoverageReport", "--stacktrace")
         }
 
-        val runner = GradleRunner.create()
-            .withProjectDir(projectRoot)
-            .withGradleVersion(gradleVersion)
-            .withPluginClasspath()
-            .forwardStdOutput(SystemOutputWriter.out())
-            .forwardStdError(SystemOutputWriter.err())
-            .withArguments(gradleCommands)
-
-        val result = runner.build()
+        val result = executeGradleTasks(gradleCommands)
 
         result.assertSuccessful()
 
         // Assert whether the correct Android Test tasks are executed
-        result.assertCorrectAndroidTestTasksAreExecuted()
+        if(executeAndroidTests) {
+            result.assertCorrectAndroidTestTasksAreExecuted()
+        } else {
+            result.assertCorrectAndroidTestTasksAreNotExecuted()
+        }
 
         // Assert whether the combined coverage report is what we expected
         result.assertRootCoverageReport()
@@ -121,18 +130,28 @@ class IntegrationTest(
 
     private fun BuildResult.assertCorrectAndroidTestTasksAreExecuted() {
         if (configuration.pluginConfiguration.getPropertyValue("runOnGradleManagedDevices", "false").toBoolean()) {
-            // Assert that the tests have been run on Gradle Managed Devices
             val device = configuration.pluginConfiguration.getPropertyValue("gradleManagedDeviceName", "allDevices")
             assertTaskSuccess(":app:${device}DebugAndroidTest")
             assertTaskSuccess(":library_android:${device}DebugAndroidTest")
 
         } else {
-            // Assert that the tests have been run on connected devices
             assertTaskSuccess(":app:connectedDebugAndroidTest")
             assertTaskSuccess(":library_android:connectedDebugAndroidTest")
         }
     }
 
+    private fun BuildResult.assertCorrectAndroidTestTasksAreNotExecuted() {
+        if (configuration.pluginConfiguration.getPropertyValue("runOnGradleManagedDevices", "false").toBoolean()) {
+            val device = configuration.pluginConfiguration.getPropertyValue("gradleManagedDeviceName", "allDevices")
+            assertTaskNotExecuted(":app:${device}DebugAndroidTest")
+            assertTaskNotExecuted(":library_android:${device}DebugAndroidTest")
+
+        } else {
+            assertTaskNotExecuted(":app:connectedDebugAndroidTest")
+            assertTaskNotExecuted(":library_android:connectedDebugAndroidTest")
+        }
+    }
+
     private fun BuildResult.assertRootCoverageReport() {
         assertTaskSuccess(":rootCoverageReport")
 
@@ -191,6 +210,17 @@ class IntegrationTest(
             }
     }
 
+    private fun executeGradleTasks(tasks: List<String>): BuildResult {
+        return GradleRunner.create()
+            .withProjectDir(projectRoot)
+            .withGradleVersion(gradleVersion)
+            .withPluginClasspath()
+            .forwardStdOutput(SystemOutputWriter.out())
+            .forwardStdError(SystemOutputWriter.err())
+            .withArguments(tasks)
+            .build()
+    }
+
     companion object {
 
         @Suppress("unused") // This method is used by the JVM (Parameterized JUnit Runner)

plugin/src/test/kotlin/org/neotech/plugin/rootcoverage/util/GradleBuildExtensions.kt

@@ -10,4 +10,8 @@ fun BuildResult.assertSuccessful() {
 
 fun BuildResult.assertTaskSuccess(taskPath: String) {
     assertThat(task(taskPath)!!.outcome).isEqualTo(TaskOutcome.SUCCESS)
+}
+
+fun BuildResult.assertTaskNotExecuted(taskPath: String) {
+    assertThat(task(taskPath)).isNull()
 }

plugin/src/test/test-fixtures/multi-module/configurations/gradle-managed-device-dont-execute-android-tests.yaml

@@ -0,0 +1,30 @@
+projectConfiguration:
+  addGradleManagedDevice: true
+pluginConfiguration:
+  properties:
+    - name: generateHtml
+      value: true
+    - name: generateXml
+      value: false
+    - name: generateCsv
+      value: true
+
+    - name: buildVariant
+      value: debug
+
+    - name: executeUnitTests
+      value: true
+    - name: executeAndroidTests
+      value: false
+
+    - name: includeUnitTestResults
+      value: true
+    - name: includeAndroidTestResults
+      value: true
+    - name: includeNoLocationClasses
+      value: true
+
+    - name: runOnGradleManagedDevices
+      value: true
+    - name: gradleManagedDeviceName
+      value: nexusoneapi30