Skip to content

Commit c0b0888

Browse files
rnakadeyigit
rnakade
and
yigit
authored
Retrieve results successfully when merged_results.xml is missing (#71)
* Retrieve results successfully when merged_results.xml is missing * Update AndroidXCI/lib/src/main/kotlin/dev/androidx/ci/testRunner/TestRunnerServiceImpl.kt Co-authored-by: Yigit Boyar <[email protected]> * addressing feedback --------- Co-authored-by: Yigit Boyar <[email protected]>
1 parent a1d0e94 commit c0b0888

File tree

4 files changed

+316
-5
lines changed

4 files changed

+316
-5
lines changed

‎AndroidXCI/lib/src/main/kotlin/dev/androidx/ci/testRunner/TestRunnerService.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ interface TestRunnerService {
283283
/**
284284
* The xml file which includes all results from all test runs (including re-runs)
285285
*/
286-
val mergedResults: ResultFileResource,
286+
val mergedResults: ResultFileResource?,
287287
/**
288288
* Each individual test run. This list usually contains 1 item but might contain more than one if
289289
* the [DevicePicker] choose multiple devices OR some tests were re-run due to failures.

‎AndroidXCI/lib/src/main/kotlin/dev/androidx/ci/testRunner/TestRunnerServiceImpl.kt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,16 @@ internal class TestRunnerServiceImpl internal constructor(
213213
}
214214
}
215215
}
216+
// remove this if block when b/299975596 is fixed
217+
if (mergedXmlBlobs.isEmpty()) {
218+
return byFullDeviceId.values.map {
219+
TestRunnerService.TestRunResult(
220+
deviceId = it.deviceRun.deviceId,
221+
mergedResults = null,
222+
testRuns = listOf(it)
223+
)
224+
}
225+
}
216226
return mergedXmlBlobs.map { mergedXmlEntry ->
217227
val relatedRuns = byFullDeviceId.entries.filter {
218228
it.key.startsWith(mergedXmlEntry.key)

‎AndroidXCI/lib/src/test/kotlin/dev/androidx/ci/testRunner/TestRunnerServiceImplTest.kt‎

Lines changed: 303 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -521,7 +521,7 @@ class TestRunnerServiceImplTest {
521521
).isEqualTo("redfin-30-en-portrait")
522522

523523
assertThat(
524-
result.mergedResults.readFully()
524+
result.mergedResults?.readFully()
525525
).isEqualTo(
526526
"merged-results".toByteArray(Charsets.UTF_8)
527527
)
@@ -889,7 +889,7 @@ class TestRunnerServiceImplTest {
889889
).isEqualTo("redfin-30-en-portrait")
890890

891891
assertThat(
892-
result.mergedResults.readFully()
892+
result.mergedResults?.readFully()
893893
).isEqualTo(
894894
"merged-results".toByteArray(Charsets.UTF_8)
895895
)
@@ -1047,6 +1047,307 @@ class TestRunnerServiceImplTest {
10471047
)
10481048
}
10491049

1050+
@Test
1051+
fun emptyResults() = runBlocking {
1052+
val resultRelativePath = "my-test-matrix-results"
1053+
val resultPath = "${fakeBackend.fakeGoogleCloudApi.rootGcsPath}/$resultRelativePath"
1054+
val testMatrix = fakeBackend.fakeFirebaseTestLabApi.createTestMatrix(
1055+
projectId = fakeBackend.firebaseProjectId,
1056+
requestId = "requestId",
1057+
testMatrix = TestMatrix(
1058+
resultStorage = ResultStorage(
1059+
googleCloudStorage = GoogleCloudStorage(resultPath),
1060+
toolResultsExecution = ToolResultsExecution(
1061+
executionId = "test_executionId",
1062+
historyId = "test_historyId"
1063+
)
1064+
),
1065+
projectId = fakeBackend.firebaseProjectId,
1066+
environmentMatrix = EnvironmentMatrix(),
1067+
testSpecification = TestSpecification()
1068+
)
1069+
)
1070+
val testMatrixId = testMatrix.testMatrixId!!
1071+
1072+
assertThat(
1073+
subject.getTestMatrix(testMatrixId)
1074+
).isEqualTo(
1075+
testMatrix
1076+
)
1077+
// incomplete, no results
1078+
assertThat(
1079+
subject.getTestMatrixResults(testMatrix)
1080+
).isNull()
1081+
fakeBackend.finishTest(
1082+
testMatrixId = testMatrixId,
1083+
outcome = TestMatrix.OutcomeSummary.SUCCESS
1084+
)
1085+
assertThat(
1086+
subject.getTestMatrixResults(testMatrix)
1087+
).isNull()
1088+
1089+
val results = subject.getTestMatrixResults(testMatrixId)
1090+
assertThat(results).isEmpty()
1091+
// check screenshots is null when list of testIdentifiers is empty
1092+
val screenshots = subject.getTestMatrixArtifacts(
1093+
testMatrixId,
1094+
emptyList()
1095+
)
1096+
assertThat(
1097+
screenshots
1098+
).isNull()
1099+
}
1100+
1101+
@Test
1102+
fun mergedResultsMissing() = runBlocking {
1103+
val resultRelativePath = "my-test-matrix-results"
1104+
val resultPath = "${fakeBackend.fakeGoogleCloudApi.rootGcsPath}/$resultRelativePath"
1105+
val testMatrix = fakeBackend.fakeFirebaseTestLabApi.createTestMatrix(
1106+
projectId = fakeBackend.firebaseProjectId,
1107+
requestId = "requestId",
1108+
testMatrix = TestMatrix(
1109+
resultStorage = ResultStorage(
1110+
googleCloudStorage = GoogleCloudStorage(resultPath),
1111+
toolResultsExecution = ToolResultsExecution(
1112+
executionId = "test_executionId",
1113+
historyId = "test_historyId"
1114+
)
1115+
),
1116+
projectId = fakeBackend.firebaseProjectId,
1117+
environmentMatrix = EnvironmentMatrix(),
1118+
testSpecification = TestSpecification()
1119+
)
1120+
)
1121+
val testMatrixId = testMatrix.testMatrixId!!
1122+
1123+
assertThat(
1124+
subject.getTestMatrix(testMatrixId)
1125+
).isEqualTo(
1126+
testMatrix
1127+
)
1128+
// incomplete, no results
1129+
assertThat(
1130+
subject.getTestMatrixResults(testMatrix)
1131+
).isNull()
1132+
fakeBackend.finishTest(
1133+
testMatrixId = testMatrixId,
1134+
outcome = TestMatrix.OutcomeSummary.SUCCESS
1135+
)
1136+
assertThat(
1137+
subject.getTestMatrixResults(testMatrix)
1138+
).isNull()
1139+
1140+
fakeBackend.fakeGoogleCloudApi.upload(
1141+
"$resultRelativePath/redfin-30-en-portrait/logcat",
1142+
"logcat".toByteArray(Charsets.UTF_8)
1143+
)
1144+
fakeBackend.fakeGoogleCloudApi.upload(
1145+
"$resultRelativePath/redfin-30-en-portrait/test_result_1.xml",
1146+
"test_result_1 content xml".toByteArray(Charsets.UTF_8)
1147+
)
1148+
fakeBackend.fakeGoogleCloudApi.upload(
1149+
"$resultRelativePath/redfin-30-en-portrait/test_cases/0000_logcat",
1150+
"test1 logcat".toByteArray(Charsets.UTF_8)
1151+
)
1152+
fakeBackend.fakeGoogleCloudApi.upload(
1153+
"$resultRelativePath/redfin-30-en-portrait/test_cases/0001_logcat",
1154+
"test2 logcat".toByteArray(Charsets.UTF_8)
1155+
)
1156+
fakeBackend.fakeGoogleCloudApi.upload(
1157+
"$resultRelativePath/redfin-30-en-portrait/artifacts/sdcard/Android/data/test/cache/androidx_screenshots/class1_name1_emulator_actual.png",
1158+
"class1 name1 emulator actual".toByteArray(Charsets.UTF_8)
1159+
)
1160+
fakeBackend.fakeGoogleCloudApi.upload(
1161+
"$resultRelativePath/redfin-30-en-portrait/artifacts/sdcard/Android/data/test/cache/androidx_screenshots/class1_name1_emulator_diff.png",
1162+
"class1 name1 emulator diff".toByteArray(Charsets.UTF_8)
1163+
)
1164+
fakeBackend.fakeGoogleCloudApi.upload(
1165+
"$resultRelativePath/redfin-30-en-portrait/artifacts/sdcard/Android/data/test/cache/androidx_screenshots/class1_name1_emulator_expected.png",
1166+
"class1 name1 emulator expected".toByteArray(Charsets.UTF_8)
1167+
)
1168+
fakeBackend.fakeGoogleCloudApi.upload(
1169+
"$resultRelativePath/redfin-30-en-portrait/artifacts/sdcard/Android/data/test/cache/androidx_screenshots/class1_name1_emulator_goldResult.textproto",
1170+
"class1 name1 emulator textproto".toByteArray(Charsets.UTF_8)
1171+
)
1172+
// No test is associated with this artifact. findArtifacts should not throw errors, even when unexpected files are encountered
1173+
fakeBackend.fakeGoogleCloudApi.upload(
1174+
"$resultRelativePath/redfin-30-en-portrait/artifacts/sdcard/Android/data/test/cache/androidx_screenshots/class5_name5_emulator_goldResult.textproto",
1175+
"class5 name5 emulator textproto".toByteArray(Charsets.UTF_8)
1176+
)
1177+
1178+
fakeToolsResultApi.addStep(
1179+
projectId = fakeBackend.firebaseProjectId,
1180+
executionId = "test_executionId",
1181+
historyId = "test_historyId",
1182+
step = Step(
1183+
stepId = UUID.randomUUID().toString(),
1184+
testExecutionStep = TestExecutionStep(
1185+
toolExecution = ToolExecution(
1186+
toolOutputs = listOf(
1187+
ToolOutputReference(
1188+
testCase = TestCaseReference(
1189+
className = "class1",
1190+
name = "name1"
1191+
),
1192+
output = FileReference(
1193+
fileUri = "$resultPath/redfin-30-en-portrait/test_cases/0000_logcat"
1194+
)
1195+
),
1196+
ToolOutputReference(
1197+
testCase = TestCaseReference(
1198+
className = "class3",
1199+
name = "name3"
1200+
),
1201+
output = FileReference(
1202+
fileUri = "$resultPath/redfin-30-en-portrait/test_cases/0002_logcat"
1203+
)
1204+
)
1205+
)
1206+
)
1207+
)
1208+
)
1209+
)
1210+
val results = subject.getTestMatrixResults(testMatrixId)
1211+
assertThat(results).hasSize(1)
1212+
val result = results!!.single()
1213+
assertThat(
1214+
result.deviceId
1215+
).isEqualTo("redfin-30-en-portrait")
1216+
1217+
assertThat(
1218+
result.mergedResults
1219+
).isNull()
1220+
1221+
assertThat(result.testRuns).hasSize(1)
1222+
result.testRuns.first().let { testRun ->
1223+
val testIdentifier1 = TestRunnerService.TestIdentifier(
1224+
className = "class1",
1225+
name = "name1",
1226+
runNumber = testRun.deviceRun.runNumber
1227+
)
1228+
val screenshots1 = subject.getTestMatrixArtifacts(
1229+
testMatrixId,
1230+
listOf(testIdentifier1)
1231+
)
1232+
assertThat(
1233+
testRun.deviceRun.deviceId
1234+
).isEqualTo(
1235+
"redfin-30-en-portrait"
1236+
)
1237+
1238+
assertThat(
1239+
testRun.logcat?.readFully()
1240+
).isEqualTo(
1241+
"logcat".toByteArray(Charsets.UTF_8)
1242+
)
1243+
1244+
assertThat(
1245+
testRun.xmlResults.map { it.readFully().toString(Charsets.UTF_8) }
1246+
).containsExactly(
1247+
"test_result_1 content xml"
1248+
)
1249+
1250+
assertThat(
1251+
testRun.testCaseArtifacts.size
1252+
).isEqualTo(
1253+
1
1254+
)
1255+
assertThat(
1256+
testRun.testCaseArtifacts[
1257+
testIdentifier1
1258+
]?.size
1259+
).isEqualTo(
1260+
1
1261+
)
1262+
// step and logcat both have valid values for test1
1263+
assertThat(
1264+
testRun.testCaseArtifacts[
1265+
testIdentifier1
1266+
]?.first {
1267+
it.resourceType == TestRunnerService.TestCaseArtifact.ResourceType.LOGCAT
1268+
}?.resultFileResource?.gcsPath.toString()
1269+
).isEqualTo(
1270+
"$resultPath/redfin-30-en-portrait/test_cases/0000_logcat"
1271+
)
1272+
assertThat(
1273+
testRun.testCaseArtifacts[
1274+
testIdentifier1
1275+
]?.first {
1276+
it.resourceType == TestRunnerService.TestCaseArtifact.ResourceType.LOGCAT
1277+
}?.resultFileResource?.readFully()?.toString(Charsets.UTF_8)
1278+
).isEqualTo(
1279+
"test1 logcat"
1280+
)
1281+
assertThat(
1282+
screenshots1?.get(testIdentifier1)?.count {
1283+
it.resourceType == TestRunnerService.TestCaseArtifact.ResourceType.PNG
1284+
}
1285+
).isEqualTo(
1286+
3
1287+
)
1288+
assertThat(
1289+
screenshots1?.get(testIdentifier1)?.count {
1290+
it.resourceType == TestRunnerService.TestCaseArtifact.ResourceType.TEXTPROTO
1291+
}
1292+
).isEqualTo(
1293+
1
1294+
)
1295+
assertThat(
1296+
screenshots1?.get(testIdentifier1)?.first {
1297+
it.resourceType == TestRunnerService.TestCaseArtifact.ResourceType.TEXTPROTO
1298+
}?.resultFileResource?.gcsPath.toString()
1299+
).isEqualTo(
1300+
"$resultPath/redfin-30-en-portrait/artifacts/sdcard/Android/data/test/cache/androidx_screenshots/class1_name1_emulator_goldResult.textproto"
1301+
)
1302+
1303+
// step for test2 is missing
1304+
assertThat(
1305+
testRun.testCaseArtifacts[
1306+
TestRunnerService.TestIdentifier(
1307+
className = "class2",
1308+
name = "name2",
1309+
runNumber = testRun.deviceRun.runNumber
1310+
)
1311+
]
1312+
).isNull()
1313+
// No screenshots for test2
1314+
val testIdentifier2 = TestRunnerService.TestIdentifier(
1315+
className = "class2",
1316+
name = "name2",
1317+
runNumber = testRun.deviceRun.runNumber
1318+
)
1319+
val screenshots2 = subject.getTestMatrixArtifacts(
1320+
testMatrixId,
1321+
listOf(testIdentifier2)
1322+
)
1323+
assertThat(
1324+
screenshots2
1325+
).isEmpty()
1326+
// logcat for test3 is missing from gcloud folder
1327+
assertThat(
1328+
testRun.testCaseArtifacts[
1329+
TestRunnerService.TestIdentifier(
1330+
className = "class3",
1331+
name = "name3",
1332+
runNumber = testRun.deviceRun.runNumber
1333+
)
1334+
]
1335+
).isNull()
1336+
// No screenshots for test3 either
1337+
val testIdentifier3 = TestRunnerService.TestIdentifier(
1338+
className = "class2",
1339+
name = "name2",
1340+
runNumber = testRun.deviceRun.runNumber
1341+
)
1342+
val screenshots3 = subject.getTestMatrixArtifacts(
1343+
testMatrixId,
1344+
listOf(testIdentifier3)
1345+
)
1346+
assertThat(
1347+
screenshots3
1348+
).isEmpty()
1349+
}
1350+
}
10501351
@Test
10511352
fun parseDeviceId() {
10521353
assertThat(

‎AndroidXCI/lib/src/test/kotlin/dev/androidx/ci/testRunner/TestRunnerServicePlayground.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ internal class TestRunnerServicePlayground {
2828
println(testMatrix)
2929
subject.getTestMatrixResults(testMatrixId)?.forEach { resultFiles ->
3030
println(resultFiles)
31-
resultFiles.mergedResults.openInputStream().use {
32-
println(it.readAllBytes().toString(Charsets.UTF_8))
31+
resultFiles.mergedResults?.openInputStream().use {
32+
println(it?.readAllBytes()?.toString(Charsets.UTF_8))
3333
}
3434
}
3535
}

0 commit comments

Comments
 (0)