Add test scheduler to support AndroidTest json files (#57)

* Add ability to use test run configs

This CL adds a new argument to use the json files inside the
androidTest to run tests. It is less efficient than then androidx.dev
implementation because I've not ported the remote zip file. We might
do that as a followup

* add support for test suite tags, add tests

* add more logs

Author: yigit

AndroidXCI/cli/src/main/kotlin/dev/androidx/ci/cli/Main.kt

@@ -128,6 +128,26 @@ private class Cli : CliktCommand() {
         envvar = "ANDROIDX_BUCKET_PATH"
     ).required()
 
+    val useTestConfigFiles by option(
+        help = """
+            Internal for AndroidX.
+            If set, the action will look for json files matching *AndroidTest.json inside the Github archives.
+            See TestScheduler.TestRunConfig for the json file structure.
+            Defaults to false.
+        """.trimIndent(),
+        envvar = "ANDROIDX_USE_TEST_CONFIG_FILES"
+    )
+
+    val testSuiteTags by option(
+        help = """
+            Internal for AndroidX.
+            Comma separated list of testSuiteTags that should be run.
+            Only used if `useTestConfigFiles` is set to true.
+            Defaults to empty list, which runs all testSuiteTags.
+        """.trimIndent(),
+        envvar = "ANDROIDX_TEST_SUITE_TAGS"
+    )
+
     override fun run() {
         logFile?.let(::configureLogger)
         val repoParts = githubRepository.split("/")
@@ -155,7 +175,9 @@ private class Cli : CliktCommand() {
                     ?.let(::createDevicePicker),
                 artifactNameFilter = artifactNameFilter,
                 bucketName = gcpBucketName,
-                bucketPath = gcpBucketPath
+                bucketPath = gcpBucketPath,
+                useTestConfigFiles = useTestConfigFiles?.toBoolean() ?: false,
+                testSuiteTags = testSuiteTags?.split(',')?.map { it.trim() } ?: emptyList()
             )
             testRunner.runTests()
         }

AndroidXCI/lib/src/main/kotlin/dev/androidx/ci/testRunner/TestRunner.kt

@@ -25,18 +25,11 @@ import dev.androidx.ci.gcloud.GoogleCloudApi
 import dev.androidx.ci.generated.ftl.TestMatrix
 import dev.androidx.ci.github.GithubApi
 import dev.androidx.ci.github.dto.ArtifactsResponse
-import dev.androidx.ci.github.zipArchiveStream
 import dev.androidx.ci.testRunner.vo.TestResult
-import dev.androidx.ci.testRunner.vo.UploadedApk
 import kotlinx.coroutines.CoroutineDispatcher
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.async
-import kotlinx.coroutines.awaitAll
-import kotlinx.coroutines.coroutineScope
 import org.apache.logging.log4j.kotlin.logger
 import java.io.File
 import java.util.concurrent.TimeUnit
-import java.util.zip.ZipEntry
 
 /**
  * Main class that is responsible to run tests, download reports etc.
@@ -75,7 +68,11 @@ class TestRunner internal constructor(
     /**
      * Device picker
      */
-    private val devicePicker: DevicePicker? = null
+    private val devicePicker: DevicePicker? = null,
+    /**
+     * The actual class that will decide which tests to run out of the artifacts
+     */
+    testSchedulerFactory: TestScheduler.Factory,
 ) {
     private val logger = logger()
     private val testMatrixStore = TestMatrixStore(
@@ -96,6 +93,12 @@ class TestRunner internal constructor(
         hostRunId = hostRunId,
         targetRunId = targetRunId
     )
+    private val testScheduler = testSchedulerFactory.create(
+        githubApi = githubApi,
+        firebaseTestLabController = testLabController,
+        apkStore = apkStore,
+        devicePicker = devicePicker
+    )
 
     /**
      * Runs all the test. This never throws, instead, returns an error result if something goes
@@ -109,16 +112,9 @@ class TestRunner internal constructor(
             val allTestMatrices = artifactsResponse.artifacts
                 .filter(githubArtifactFilter)
                 .flatMap { artifact ->
-                    logger.info { "will upload apks for $artifact" }
-                    val uploadedApks = uploadApksToGoogleCloud(artifact)
-                    logger.info { "will start tests for these apks: $uploadedApks" }
-                    testLabController.pairAndStartTests(
-                        apks = uploadedApks,
-                        placeholderApk = apkStore.getPlaceholderApk(),
-                        devicePicker = devicePicker
-                    ).also { testMatrices ->
-                        logger.info { "started all tests for $testMatrices" }
-                    }
+                    testScheduler.enqueueTests(artifact)
+                }.also { testMatrices ->
+                    logger.info { "started all tests for $testMatrices" }
                 }
             logger.info("will wait for test results")
             testLabController.collectTestResults(
@@ -148,30 +144,6 @@ class TestRunner internal constructor(
         return result
     }
 
-    private suspend fun uploadApksToGoogleCloud(artifact: ArtifactsResponse.Artifact): List<UploadedApk> {
-        return coroutineScope {
-            val uploads = githubApi.zipArchiveStream(
-                path = artifact.archiveDownloadUrl,
-                unwrapNestedZipEntries = true
-            ).filter {
-                it.entry.name.endsWith(".apk")
-            }.map {
-                uploadApkToGcsAsync(it.entry, it.bytes)
-            }.toList()
-            uploads.awaitAll()
-        }
-    }
-
-    private fun CoroutineScope.uploadApkToGcsAsync(
-        zipEntry: ZipEntry,
-        bytes: ByteArray,
-    ) = async {
-        apkStore.uploadApk(
-            name = zipEntry.name,
-            bytes = bytes
-        )
-    }
-
     companion object {
         internal const val RESULT_JSON_FILE_NAME = "result.json"
         fun create(
@@ -187,6 +159,8 @@ class TestRunner internal constructor(
             bucketPath: String,
             devicePicker: DevicePicker? = null,
             artifactNameFilter: (String) -> Boolean = { true },
+            useTestConfigFiles: Boolean,
+            testSuiteTags: List<String>
         ): TestRunner {
             val credentials = ServiceAccountCredentials.fromStream(
                 googleCloudCredentials.byteInputStream(Charsets.UTF_8)
@@ -235,7 +209,8 @@ class TestRunner internal constructor(
                 outputFolder = outputFolder,
                 targetRunId = targetRunId,
                 hostRunId = hostRunId,
-                devicePicker = devicePicker
+                devicePicker = devicePicker,
+                testSchedulerFactory = TestScheduler.createFactory(useTestConfigFiles, testSuiteTags)
             )
         }