Skip to content

Commit 730f73e

Browse files
sunmou99sunmou99
authored
Test result report missing logs
1 parent 6d60f1d commit 730f73e

File tree

2 files changed

+87
-11
lines changed

2 files changed

+87
-11
lines changed

‎.github/workflows/integration_tests.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,14 @@ jobs:
314314
--noadd_timestamp \
315315
--short_output_paths \
316316
${additional_flags[*]}
317+
- name: Prepare results summary artifact
318+
if: ${{ !cancelled() }}
319+
shell: bash
320+
run: |
321+
if [ ! -f build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json ]; then
322+
# No summary was created, make a placeholder one.
323+
echo "__SUMMARY_MISSING__" > build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json
324+
fi
317325
- name: Upload Desktop integration tests artifact
318326
uses: actions/[email protected]
319327
if: ${{ !cancelled() }}
@@ -442,6 +450,13 @@ jobs:
442450
--noadd_timestamp \
443451
--short_output_paths \
444452
${additional_flags[*]}
453+
- name: Prepare results summary artifact
454+
if: ${{ !cancelled() }}
455+
shell: bash
456+
run: |
457+
if [ ! -f build-results-android-${{ matrix.os }}.log.json ]; then
458+
echo "__SUMMARY_MISSING__" > build-results-android-${{ matrix.os }}.log.json
459+
fi
445460
- name: Upload Android integration tests artifact
446461
uses: actions/[email protected]
447462
if: ${{ !cancelled() }}
@@ -540,6 +555,13 @@ jobs:
540555
--noadd_timestamp \
541556
--short_output_paths \
542557
${additional_flags[*]}
558+
- name: Prepare results summary artifact
559+
if: ${{ !cancelled() }}
560+
shell: bash
561+
run: |
562+
if [ ! -f build-results-ios-macos-latest.log.json ]; then
563+
echo "__SUMMARY_MISSING__" > build-results-ios-macos-latest.log.json
564+
fi
543565
- name: Upload iOS integration tests artifact
544566
uses: actions/[email protected]
545567
if: ${{ !cancelled() }}
@@ -637,6 +659,13 @@ jobs:
637659
--noadd_timestamp \
638660
--short_output_paths \
639661
${additional_flags[*]}
662+
- name: Prepare results summary artifact
663+
if: ${{ !cancelled() }}
664+
shell: bash
665+
run: |
666+
if [ ! -f build-results-tvos-macos-latest.log.json ]; then
667+
echo "__SUMMARY_MISSING__" > build-results-tvos-macos-latest.log.json
668+
fi
640669
- name: Upload tvOS integration tests artifact
641670
uses: actions/[email protected]
642671
if: ${{ !cancelled() }}
@@ -704,6 +733,13 @@ jobs:
704733
python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}" --artifact testapps
705734
- name: Run Desktop integration tests
706735
run: python scripts/gha/desktop_tester.py --testapp_dir testapps --logfile_name "desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}"
736+
- name: Prepare results summary artifact
737+
if: ${{ !cancelled() }}
738+
shell: bash
739+
run: |
740+
if [ ! -f testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json ]; then
741+
echo "__SUMMARY_MISSING__" > testapps/test-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}.log.json
742+
fi
707743
- name: Upload Desktop test results artifact
708744
if: ${{ !cancelled() }}
709745
uses: actions/[email protected]
@@ -782,6 +818,13 @@ jobs:
782818
--logfile_name "android-${{ matrix.build_os }}-${{ matrix.android_device }}" \
783819
--code_platform cpp \
784820
--key_file scripts/gha-encrypted/gcs_key_file.json
821+
- name: Prepare results summary artifact
822+
if: ${{ !cancelled() }}
823+
shell: bash
824+
run: |
825+
if [ ! -f "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}.log.json" ]; then
826+
echo "__SUMMARY_MISSING__" > "testapps/test-results-android-${{ matrix.build_os }}-${{ matrix.android_device }}.log.json"
827+
fi
785828
- name: Upload Android test results artifact
786829
if: ${{ !cancelled() }}
787830
uses: actions/[email protected]
@@ -859,6 +902,13 @@ jobs:
859902
--logfile_name "ios-macos-latest-${{ matrix.ios_device }}" \
860903
--code_platform cpp \
861904
--key_file scripts/gha-encrypted/gcs_key_file.json
905+
- name: Prepare results summary artifact
906+
if: ${{ !cancelled() }}
907+
shell: bash
908+
run: |
909+
if [ ! -f "testapps/test-results-ios-macos-latest-${{ matrix.ios_device }}.log.json" ]; then
910+
echo "__SUMMARY_MISSING__" > "testapps/test-results-ios-macos-latest-${{ matrix.ios_device }}.log.json"
911+
fi
862912
- name: Upload iOS test results artifact
863913
if: ${{ !cancelled() }}
864914
uses: actions/[email protected]
@@ -920,6 +970,13 @@ jobs:
920970
--tvos_device "${{ matrix.tvos_device }}" \
921971
--logfile_name "tvos-macos-latest-${{ matrix.tvos_device }}" \
922972
--ci
973+
- name: Prepare results summary artifact
974+
if: ${{ !cancelled() }}
975+
shell: bash
976+
run: |
977+
if [ ! -f "testapps/test-results-tvos-macos-latest-${{ matrix.tvos_device }}.log.json" ]; then
978+
echo "__SUMMARY_MISSING__" > "testapps/test-results-tvos-macos-latest-${{ matrix.tvos_device }}.log.json"
979+
fi
923980
- name: Upload tvOS test results artifact
924981
if: ${{ !cancelled() }}
925982
uses: actions/[email protected]

‎scripts/gha/summarize_test_results.py

Lines changed: 30 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
2323
| Failures | Configs |
2424
|------------|----------------------------------------------------|
25-
| messaging |[BUILD] [ERROR] [Windows] [boringssl] |
25+
| missing_log|[BUILD] [ERROR] [Windows] [boringssl] |
26+
| messaging |[BUILD] [ERROR] [Windows] [openssl] |
2627
| |[TEST] [ERROR] [Android] [All os] [emulator_min] |
2728
| |[TEST] [FAILURE] [Android] [macos] [emulator_target]|
2829
| |▼(1 failed tests) |
@@ -86,6 +87,7 @@
8687

8788
TESTAPPS_HEADER = "Failures"
8889
CONFIGS_HEADER = "Configs"
90+
MISSING_LOG = "missing_log"
8991

9092
LOG_HEADER = "INTEGRATION TEST FAILURES"
9193

@@ -110,6 +112,13 @@ def print_table(log_results,
110112
output_lines.append(("|" + "-%s-|" * len(headers)) %
111113
tuple([ re.sub("[^|]","-", header) for header in headers ]))
112114

115+
if MISSING_LOG in log_results.keys():
116+
columns = [
117+
re.sub(r'\b \b', space_char, MISSING_LOG.ljust(testapps_width)),
118+
format_result(log_results.pop(MISSING_LOG), space_char=space_char, list_separator=list_separator, justify=configs_width)
119+
]
120+
output_lines.append(("|" + " %s |" * len(headers)) % tuple(columns))
121+
113122
# Iterate through platforms and print out table lines.
114123
for testapp in sorted(log_results.keys()):
115124
columns = [
@@ -195,27 +204,37 @@ def summarize_logs(dir, markdown=False, github_log=False):
195204
log_data = {}
196205
# log_data format:
197206
# { testapps: {"build": [configs]},
198-
# "test": {"errors": [configs]},
207+
# {"test": {"errors": [configs]},
199208
# {"failures": {failed_test: [configs]}}}}
200209
for build_log_file in build_log_files:
201210
configs = get_configs_from_file_name(build_log_file, build_log_name_re)
202211
with open(build_log_file, "r") as log_reader:
203-
log_reader_data = json.loads(log_reader.read())
204-
for (testapp, error) in log_reader_data["errors"].items():
212+
log_text = log_reader.read()
213+
if "__SUMMARY_MISSING__" in log_text:
205214
any_failures = True
206-
log_data.setdefault(testapp, {}).setdefault("build", []).append(configs)
215+
log_data.setdefault(MISSING_LOG, {}).setdefault("build", []).append(configs)
216+
else:
217+
log_reader_data = json.loads(log_text)
218+
for (testapp, error) in log_reader_data["errors"].items():
219+
any_failures = True
220+
log_data.setdefault(testapp, {}).setdefault("build", []).append(configs)
207221

208222
for test_log_file in test_log_files:
209223
configs = get_configs_from_file_name(test_log_file, test_log_name_re)
210224
with open(test_log_file, "r") as log_reader:
211-
log_reader_data = json.loads(log_reader.read())
212-
for (testapp, error) in log_reader_data["errors"].items():
225+
log_text = log_reader.read()
226+
if "__SUMMARY_MISSING__" in log_text:
213227
any_failures = True
214-
log_data.setdefault(testapp, {}).setdefault("test", {}).setdefault("errors", []).append(configs)
215-
for (testapp, failures) in log_reader_data["failures"].items():
216-
for (test, failure) in failures["failed_tests"].items():
228+
log_data.setdefault(MISSING_LOG, {}).setdefault("test", {}).setdefault("errors", []).append(configs)
229+
else:
230+
log_reader_data = json.loads(log_text)
231+
for (testapp, error) in log_reader_data["errors"].items():
217232
any_failures = True
218-
log_data.setdefault(testapp, {}).setdefault("test", {}).setdefault("failures", {}).setdefault(test, []).append(configs)
233+
log_data.setdefault(testapp, {}).setdefault("test", {}).setdefault("errors", []).append(configs)
234+
for (testapp, failures) in log_reader_data["failures"].items():
235+
for (test, failure) in failures["failed_tests"].items():
236+
any_failures = True
237+
log_data.setdefault(testapp, {}).setdefault("test", {}).setdefault("failures", {}).setdefault(test, []).append(configs)
219238

220239
# log_results format:
221240
# { testapps: {configs: [failed tests]} }

0 commit comments

Comments
 (0)