don't fail for skipped tests (#62)

this CL changes the default behavior for the ci-action to not fail for
skipped tests. I've also updated default locale to en_US to match androidx
ifnra.

Test: TestRunnerTest

Author: yigit

AndroidXCI/cli/src/main/kotlin/dev/androidx/ci/cli/Main.kt

@@ -263,7 +263,7 @@ private class Cli : CliktCommand() {
                 AndroidDevice(
                     orientation = "portrait",
                     androidModelId = model.id,
-                    locale = "en",
+                    locale = "en_US",
                     androidVersionId = spec.sdk
                 )
             }

AndroidXCI/lib/src/main/kotlin/dev/androidx/ci/testRunner/FirebaseTestLabController.kt

@@ -63,7 +63,7 @@ internal class FirebaseTestLabController(
             ?: error("Cannot find supported version for $defaultModel in test device catalog: $catalog")
         listOf(
             AndroidDevice(
-                locale = "en",
+                locale = "en_US",
                 androidModelId = defaultModel.id,
                 androidVersionId = defaultModelVersion.toString(),
                 orientation = "portrait"

AndroidXCI/lib/src/main/kotlin/dev/androidx/ci/testRunner/StatusReporter.kt

@@ -51,8 +51,8 @@ internal class StatusReporter(
     suspend fun reportEnd(testResult: TestResult) {
         val newState = when {
             testResult is TestResult.IncompleteRun -> CommitInfo.State.ERROR
-            testResult.allTestsPassed -> CommitInfo.State.SUCCESS
-            else -> CommitInfo.State.FAILURE
+            testResult.hasFailedTest -> CommitInfo.State.FAILURE
+            else -> CommitInfo.State.SUCCESS
         }
         updateState(state = newState)
     }

AndroidXCI/lib/src/main/kotlin/dev/androidx/ci/testRunner/vo/TestResult.kt

@@ -19,6 +19,7 @@ package dev.androidx.ci.testRunner.vo
 import com.squareup.moshi.Moshi
 import com.squareup.moshi.adapters.PolymorphicJsonAdapterFactory
 import dev.androidx.ci.generated.ftl.TestMatrix
+import dev.androidx.ci.generated.ftl.TestMatrix.OutcomeSummary.FAILURE
 import dev.androidx.ci.generated.ftl.TestMatrix.OutcomeSummary.SUCCESS
 import dev.zacsweers.moshix.reflect.MetadataKotlinJsonAdapterFactory
 
@@ -27,6 +28,8 @@ sealed class TestResult(
 ) {
     abstract val allTestsPassed: Boolean
 
+    abstract val hasFailedTest: Boolean
+
     abstract val failureLog: String
 
     data class CompleteRun(
@@ -37,6 +40,11 @@ sealed class TestResult(
                 it.outcomeSummary != SUCCESS
             }
         }
+        override val hasFailedTest: Boolean by lazy {
+            matrices.any {
+                it.outcomeSummary == FAILURE
+            }
+        }
         override val failureLog: String
             get() = buildString {
                 val failed = matrices.filter {
@@ -54,6 +62,8 @@ sealed class TestResult(
     ) : TestResult(Type.INCOMPLETE_RUN) {
         override val allTestsPassed: Boolean
             get() = false
+        override val hasFailedTest: Boolean
+            get() = true
         override val failureLog: String
             get() = buildString {
                 appendLine("FTL failed with an exception:")

AndroidXCI/lib/src/test/kotlin/dev/androidx/ci/testRunner/StateReporterTest.kt

@@ -87,7 +87,11 @@ internal class StateReporterTest {
             projectId = "myProject",
             requestId = "requestId",
             testMatrix = matrix
-        )
+        ).let {
+            it.copy(
+                outcomeSummary = TestMatrix.OutcomeSummary.FAILURE
+            )
+        }
         reporter.reportEnd(
             TestResult.CompleteRun(matrices = listOf(failedTestMatrix))
         )

AndroidXCI/lib/src/test/kotlin/dev/androidx/ci/testRunner/TestRunnerTest.kt

@@ -20,6 +20,7 @@ import com.google.common.truth.Truth.assertThat
 import dev.androidx.ci.fake.FakeBackend
 import dev.androidx.ci.generated.ftl.TestMatrix
 import dev.androidx.ci.generated.ftl.TestMatrix.OutcomeSummary.FAILURE
+import dev.androidx.ci.generated.ftl.TestMatrix.OutcomeSummary.SKIPPED
 import dev.androidx.ci.generated.ftl.TestMatrix.OutcomeSummary.SUCCESS
 import dev.androidx.ci.github.dto.ArtifactsResponse
 import dev.androidx.ci.github.dto.CommitInfo
@@ -200,6 +201,52 @@ internal class TestRunnerTest(
         assertOutputFolderContents(result)
     }
 
+    @Test
+    fun skippedTest() = testScope.runTest {
+        val artifact1 = fakeBackend.createArchive(
+            testPairs = listOf(
+                FakeBackend.TestPair(
+                    testFilePrefix = "bio",
+                    testApk = "biometric-integration-tests-testapp_testapp-debug-androidTest.apk",
+                    appApk = "biometric-integration-tests-testapp_testapp-debug.apk",
+                )
+            ),
+            contentNames = listOf(
+                "biometric-integration-tests-testapp_testapp-debug-androidTest.apk",
+                "biometric-integration-tests-testapp_testapp-debug.apk",
+                "biometric-integration-tests-testapp_testapp-release.apk"
+            )
+        )
+        createRuns(
+            listOf(
+                artifact1
+            )
+        )
+        val runTests = async {
+            testRunner.runTests()
+        }
+        runCurrent()
+        assertThat(runTests.isActive).isTrue()
+        val testMatrices = fakeBackend.fakeFirebaseTestLabApi.getTestMatrices()
+        assertThat(testMatrices).hasSize(1)
+        fakeBackend.finishAllTests(SKIPPED)
+        advanceUntilIdle()
+        val result = runTests.await()
+        assertThat(result.allTestsPassed).isFalse()
+        assertThat(result.hasFailedTest).isFalse()
+        check(result is TestResult.CompleteRun)
+        assertThat(result.matrices).hasSize(1)
+        result.matrices.first().let {
+            // make sure it returns updated test matrices
+            assertThat(it.state).isEqualTo(TestMatrix.State.FINISHED)
+            assertThat(it.outcomeSummary).isEqualTo(TestMatrix.OutcomeSummary.SKIPPED)
+        }
+        assertThat(getRunState(TARGET_RUN_ID)).isEqualTo(
+            CommitInfo.State.SUCCESS
+        )
+        assertOutputFolderContents(result)
+    }
+
     @Test
     fun multipleTestsOnMultipleArtifacts_oneFailure() = testScope.runTest {
         val artifact1 = fakeBackend.createArchive(