Add: Experimental support for running instrumented tests on Gradle Managed Devices (#84)

This adds experimental support for running instrumented tests on Gradle Managed Devices. Some credit has to go to @bddckr for his initial PR on this (see: #73).

Author: Rolf-Smit

.github/workflows/build.yml

@@ -8,16 +8,17 @@ jobs:
     runs-on: macOS-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
-      - uses: actions/setup-java@v1
+        uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
         with:
+          distribution: 'zulu'
           java-version: '11'
       - name: test
         uses: reactivecircus/android-emulator-runner@v2
         with:
           api-level: 28
           script: ./gradlew clean test --stacktrace
       - name: upload coverage
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
           files: ./plugin/build/reports/jacoco/test/jacocoTestReport.xml

.gitignore

@@ -15,3 +15,4 @@ repo/
 build/
 */out/*/classes/*
 plugin/src/test/test-fixtures/multi-module/gradle.properties
+plugin/src/test/test-fixtures/multi-module/build.gradle

README.md

@@ -123,6 +123,15 @@ rootCoverage {
    // Since 1.4: Sets jacoco.includeNoLocationClasses, so you don't have to. Helpful when using Robolectric
    // which usually requires this attribute to be true
    includeNoLocationClasses false
+
+   // Upcoming in 1.7: If set to true instrumented tests will be attempt to run on
+   // Gradle Managed Devices before trying devices connected through other means (ADB).
+   runOnGradleManagedDevices false
+   
+   // Upcoming in 1.7: The name of the Gradle Managed device to run instrumented tests on.
+   // This is only used if `runOnGradleManagedDevices` is set to true. If not given tests will be
+   // run on all available Gradle Managed Devices
+   gradleManagedDeviceName "nexusoneapi30"
 }
 ```
 

plugin/src/main/kotlin/org/neotech/plugin/rootcoverage/JaCoCoConfiguration.kt

@@ -67,6 +67,8 @@ internal fun Project.getExecutionDataFileTree(includeUnitTestResults: Boolean, i
 
         // Android Build Tools Plugin 7.1+
         buildFolderPatterns.add("outputs/code_coverage/*/connected/*/coverage.ec")
+        // Gradle Managed Devices
+        buildFolderPatterns.add("outputs/managed_device_code_coverage/*/coverage.ec")
     }
     return if(buildFolderPatterns.isEmpty()) {
         null

plugin/src/main/kotlin/org/neotech/plugin/rootcoverage/RootCoveragePlugin.kt

@@ -5,6 +5,7 @@ import com.android.build.api.artifact.MultipleArtifact
 import com.android.build.api.dsl.BuildType
 import com.android.build.api.variant.AndroidComponentsExtension
 import com.android.build.api.variant.Variant
+import com.android.build.gradle.BaseExtension
 import org.gradle.api.GradleException
 import org.gradle.api.NamedDomainObjectContainer
 import org.gradle.api.Plugin
@@ -152,7 +153,24 @@ class RootCoveragePlugin : Plugin<Project> {
             dependsOn("$path:test${name}UnitTest")
         }
         if (rootProjectExtension.shouldExecuteAndroidTests() && (buildType.enableAndroidTestCoverage || buildType.isTestCoverageEnabled)) {
-            dependsOn("$path:connected${name}AndroidTest")
+
+            // Attempt to run on instrumented tests, giving priority to the following devices in this order:
+            // - A user provided Gradle Managed Device.
+            // - All Gradle Managed Devices if any is available.
+            // - All through ADB connected devices.
+            val gradleManagedDevices = subProject.extensions.getByType(BaseExtension::class.java).testOptions.managedDevices.devices
+            if(rootProjectExtension.runOnGradleManagedDevices && !rootProjectExtension.gradleManagedDeviceName.isNullOrEmpty()) {
+                dependsOn("$path:${rootProjectExtension.gradleManagedDeviceName}${name}AndroidTest")
+            } else if (rootProjectExtension.runOnGradleManagedDevices && gradleManagedDevices.isNotEmpty()) {
+                dependsOn("$path:allDevices${name}AndroidTest")
+            } else {
+                dependsOn("$path:connected${name}AndroidTest")
+            }
+        } else {
+            // If this plugin should not run instrumented tests on it's own, at least make sure it runs after those tasks (if they are
+            // selected to run as well).
+            mustRunAfter("$path:allDevices${name}AndroidTest")
+            mustRunAfter("$path:connected${name}AndroidTest")
         }
 
         sourceDirectories.from(variant.sources.java?.all)
@@ -162,6 +180,7 @@ class RootCoveragePlugin : Plugin<Project> {
                 subProject.fileTree(directory.asFile, excludes = rootProjectExtension.getFileFilterPatterns())
             }
         })
+
         executionData.from(
             subProject.getExecutionDataFileTree(
                 includeUnitTestResults = rootProjectExtension.includeUnitTestResults && (buildType.enableUnitTestCoverage || buildType.isTestCoverageEnabled),

plugin/src/main/kotlin/org/neotech/plugin/rootcoverage/RootCoveragePluginExtension.kt

@@ -73,6 +73,18 @@ open class RootCoveragePluginExtension {
      */
     var includeUnitTestResults = true
 
+    /**
+     * The name of the Gradle Managed device to run instrumented tests on, if null tests will be attempted to run on any
+     * connected emulator.
+     */
+    var gradleManagedDeviceName: String? = null
+
+    /**
+     * If set to true (default is false) this plugin will attempt to run on Gradle Managed Devices before trying devices
+     * connected through other means (Non Gradle Managed Devices).
+     */
+    var runOnGradleManagedDevices: Boolean = false
+    
     internal fun shouldExecuteAndroidTests() = executeTests && executeAndroidTests && includeAndroidTestResults
 
     internal fun shouldExecuteUnitTests() = executeTests && executeUnitTests && includeUnitTestResults

plugin/src/test/kotlin/org/neotech/plugin/rootcoverage/IntegrationTest.kt

@@ -1,16 +1,22 @@
 package org.neotech.plugin.rootcoverage
 
 import com.google.common.truth.Truth.assertThat
+import groovy.text.SimpleTemplateEngine
 import org.gradle.testkit.runner.BuildResult
 import org.gradle.testkit.runner.GradleRunner
 import org.gradle.testkit.runner.TaskOutcome
+import org.junit.Assume
+import org.junit.Before
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.Parameterized
+import org.neotech.plugin.rootcoverage.util.SimpleTemplate
 import org.neotech.plugin.rootcoverage.util.SystemOutputWriter
 import org.neotech.plugin.rootcoverage.util.createGradlePropertiesFile
 import org.neotech.plugin.rootcoverage.util.createLocalPropertiesFile
+import org.neotech.plugin.rootcoverage.util.getProperties
 import org.neotech.plugin.rootcoverage.util.put
+import org.neotech.plugin.rootcoverage.util.toGroovyString
 import java.io.File
 import java.util.Properties
 
@@ -19,11 +25,29 @@ class IntegrationTest(
     // Used by Junit as the test name, see @Parameters
     @Suppress("unused") private val name: String,
     private val projectRoot: File,
-    private val gradleVersion: String
+    configurationFile: File,
+    private val gradleVersion: String,
 ) {
 
+    private val configuration = configurationFile.getProperties()
+
+    @Before
+    fun before(){
+        // Ignore tests that require Gradle Managed Devices on CI (because GitHub Actions does not seem to support these well).
+        val isGradleManagedDeviceTest = configuration.getProperty("runOnGradleManagedDevices", "false").toBoolean()
+        Assume.assumeFalse(System.getenv("GITHUB_ACTIONS") != null && isGradleManagedDeviceTest)
+    }
+
     @Test
     fun execute() {
+        val template = SimpleTemplate().apply {
+            putValue("configuration", configuration.toGroovyString())
+        }
+
+        File(projectRoot, "build.gradle.tmp").inputStream().use {
+            File(projectRoot, "build.gradle").writeText(template.process(it, Charsets.UTF_8))
+        }
+
         createLocalPropertiesFile(projectRoot)
         createGradlePropertiesFile(projectRoot, properties = Properties().apply {
             put("android.useAndroidX", "true")
@@ -76,6 +100,17 @@ class IntegrationTest(
     private fun BuildResult.assertAppCoverageReport() {
         assertThat(task(":app:coverageReport")!!.outcome).isEqualTo(TaskOutcome.SUCCESS)
 
+        if (configuration.getProperty("runOnGradleManagedDevices", "false").toBoolean()) {
+            // Assert that the tests have been run on Gradle Managed Devices
+            val device = configuration.getProperty("gradleManagedDeviceName", "allDevices")
+            assertThat(task(":app:${device}DebugAndroidTest")!!.outcome).isEqualTo(TaskOutcome.SUCCESS)
+            assertThat(task(":library_android:${device}DebugAndroidTest")!!.outcome).isEqualTo(TaskOutcome.SUCCESS)
+        } else {
+            // Assert that the tests have been run on connected devices
+            assertThat(task(":app:connectedDebugAndroidTest")!!.outcome).isEqualTo(TaskOutcome.SUCCESS)
+            assertThat(task(":library_android:connectedDebugAndroidTest")!!.outcome).isEqualTo(TaskOutcome.SUCCESS)
+        }
+
         val report = CoverageReport.from(File(projectRoot, "app/build/reports/jacoco.csv"))
 
         report.assertNotInReport("org.neotech.app", "MustBeExcluded")
@@ -124,12 +159,20 @@ class IntegrationTest(
         @JvmStatic
         fun parameters(): List<Array<Any>> {
 
-            val testFixtures = File("src/test/test-fixtures").listFiles()?.filter { it.isDirectory }
-                ?: error("Could not list test fixture directories")
+            val testFixtures =
+                File("src/test/test-fixtures").listFiles()?.filter { it.isDirectory } ?: error("Could not list test fixture directories")
             val gradleVersions = arrayOf("7.4", "7.4.2", "7.5.1")
-            return testFixtures.flatMap { file ->
-                gradleVersions.map { gradleVersion ->
-                    arrayOf("${file.name}-$gradleVersion", file, gradleVersion)
+            return testFixtures.flatMap { fixture ->
+                val configurations = File(fixture, "configurations").listFiles() ?: error("Configurations folder not found in $fixture")
+                configurations.flatMap { configuration ->
+                    gradleVersions.map { gradleVersion ->
+                        arrayOf(
+                            "${fixture.name}-${configuration.nameWithoutExtension}-$gradleVersion",
+                            fixture,
+                            configuration,
+                            gradleVersion
+                        )
+                    }
                 }
             }
         }

plugin/src/test/kotlin/org/neotech/plugin/rootcoverage/util/ProjectGeneration.kt

@@ -27,10 +27,19 @@ internal fun Properties.put(properties: Properties) {
 }
 
 internal fun File.getProperties(): Properties = inputStream().use {
-        Properties().apply {
-            load(it)
-        }
+    Properties().apply {
+        load(it)
+    }
+}
+
+internal fun Properties.toGroovyString(): String = map {
+    val stringValue = it.value as String
+    if (stringValue.toBooleanStrictOrNull() != null) {
+        "${it.key} ${it.value}"
+    } else {
+        "${it.key} \"${it.value}\""
     }
+}.joinToString(separator = System.lineSeparator())
 
 /**
  * Tries to resolve the Android SDK directory. This function first tries the ANDROID_HOME system

plugin/src/test/kotlin/org/neotech/plugin/rootcoverage/util/SimpleTemplate.kt

@@ -0,0 +1,31 @@
+package org.neotech.plugin.rootcoverage.util
+
+import java.io.Closeable
+import java.io.InputStream
+import java.nio.charset.Charset
+
+
+/**
+ * Super quick and dirty "template engine" loosely based on mustache style templating.
+ */
+internal class SimpleTemplate {
+
+    private val map = mutableMapOf<String, String>()
+
+    fun putValue(key: String, value: String) {
+        map[key] = value
+    }
+
+    fun process(inputStream: InputStream, charset: Charset): String {
+        val content = inputStream.bufferedReader(charset).readLines()
+        val result = mutableListOf<String>()
+        content.forEach { line ->
+            var adjustedLine = line
+            map.forEach {
+                adjustedLine = line.replace("{{${it.key}}}", it.value)
+            }
+            result.add(adjustedLine)
+        }
+        return result.joinToString(System.lineSeparator())
+    }
+}

plugin/src/test/test-fixtures/multi-module/app/build.gradle

@@ -37,8 +37,17 @@ android {
         unitTests {
             includeAndroidResources = true
         }
+        managedDevices {
+            devices {
+                nexusoneapi30 (com.android.build.api.dsl.ManagedVirtualDevice) {
+                    device = "Nexus One"
+                    apiLevel = 30
+                    systemImageSource = "aosp-atd"
+                }
+            }
+        }
     }
-    
+
     kotlinOptions {
         jvmTarget = "1.8"
     }