Skip to content

Commit 6a40907

Browse files
sunmou99sunmou99
authored
Generate Integration Test Summary Report (#147)
1 parent c9b33af commit 6a40907

File tree

6 files changed

+554
-64
lines changed

6 files changed

+554
-64
lines changed

‎.github/workflows/integration_tests.yml

Lines changed: 103 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,14 @@ on:
99

1010
workflow_dispatch:
1111
inputs:
12-
build_os:
13-
description: 'Build OS (windows-latest,macos-latest)'
14-
default: 'macos-latest'
15-
required: true
1612
unity_versions:
1713
description: 'Unity version'
1814
default: '2019'
1915
required: true
16+
build_os:
17+
description: 'Build OS (windows-latest,macos-latest)'
18+
default: 'macos-latest'
19+
required: true
2020
platforms:
2121
description: 'CSV of Android,iOS,Windows,macOS,Linux'
2222
default: 'Android,iOS,Windows,macOS,Linux'
@@ -182,16 +182,26 @@ jobs:
182182
echo "::set-output name=mobile_test_on::${mobile_test_on}"
183183
echo "::set-output name=android_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k android_device -t ${mobile_test_on} )"
184184
echo "::set-output name=ios_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${TEST_MATRIX_PARAM} -k ios_device -t ${mobile_test_on} )"
185+
- name: Update PR label and comment
186+
if: steps.set_outputs.outputs.pr_number
187+
run: |
188+
#Add the in-progress label and remove any previous labels.
189+
python scripts/gha/it_workflow.py --stage start \
190+
--token ${{github.token}} \
191+
--issue_number ${{steps.set_outputs.outputs.pr_number}} \
192+
--actor ${{github.actor}} \
193+
--commit ${{steps.set_outputs.outputs.github_ref}} \
194+
--run_id ${{github.run_id}}
185195
186196
build:
187-
name: build-${{matrix.os}}-${{ matrix.unity_version }}-${{ needs.check_and_prepare.outputs.platform }}
197+
name: build-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}
188198
runs-on: ${{matrix.os}}
189199
needs: check_and_prepare
190200
strategy:
191201
fail-fast: false
192202
matrix:
193-
os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_build_os) }}
194203
unity_version: ${{ fromJson(needs.check_and_prepare.outputs.matrix_unity_versions) }}
204+
os: ${{ fromJson(needs.check_and_prepare.outputs.matrix_build_os) }}
195205
env:
196206
# LC_ALL, LANG and U3D_PASSWORD are needed for U3D.
197207
LC_ALL: en_US.UTF-8
@@ -267,7 +277,7 @@ jobs:
267277
--ios_sdk ${{ needs.check_and_prepare.outputs.mobile_test_on }} \
268278
--plugin_dir ~/Downloads/firebase_unity_sdk \
269279
--output_directory "${{ github.workspace }}" \
270-
--artifact_name "${{matrix.os}}-${{ matrix.unity_version }}-all" \
280+
--artifact_name "${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}" \
271281
--force_latest_runtime \
272282
--ci
273283
- name: Return Unity license
@@ -288,51 +298,51 @@ jobs:
288298
if: ${{ !cancelled() }}
289299
shell: bash
290300
run: |
291-
if [ ! -f build-results-${{matrix.os}}-${{ matrix.unity_version }}-all.log.json ]; then
301+
if [ ! -f build-results-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}.log.json ]; then
292302
# No summary was created, make a placeholder one.
293-
echo "__SUMMARY_MISSING__" > build-results-${{matrix.os}}-${{ matrix.unity_version }}-all.log.json
303+
echo "__SUMMARY_MISSING__" > build-results-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}.log.json
294304
fi
295305
- name: Upload build results artifact
296306
uses: actions/[email protected]
297307
if: ${{ !cancelled() }}
298308
with:
299309
name: log-artifact
300-
path: build-results-${{matrix.os}}-${{ matrix.unity_version }}-all*
310+
path: build-results-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}*
301311
retention-days: ${{ env.artifactRetentionDays }}
302312
- name: Upload Android integration tests artifact
303313
uses: actions/[email protected]
304314
if: contains(needs.check_and_prepare.outputs.platform, 'Android') && ${{ !cancelled() }}
305315
with:
306-
name: testapps-${{matrix.os}}-${{ matrix.unity_version }}-Android
307-
path: testapps-${{matrix.os}}-${{ matrix.unity_version }}-all/Android
316+
name: testapps-${{ matrix.unity_version }}-${{matrix.os}}-Android
317+
path: testapps-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}/Android
308318
retention-days: ${{ env.artifactRetentionDays }}
309319
- name: Upload iOS integration tests artifact
310320
uses: actions/[email protected]
311321
if: contains(needs.check_and_prepare.outputs.platform, 'iOS') && ${{ !cancelled() }}
312322
with:
313-
name: testapps-${{matrix.os}}-${{ matrix.unity_version }}-iOS
314-
path: testapps-${{matrix.os}}-${{ matrix.unity_version }}-all/iOS
323+
name: testapps-${{ matrix.unity_version }}-${{matrix.os}}-iOS
324+
path: testapps-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}/iOS
315325
retention-days: ${{ env.artifactRetentionDays }}
316326
- name: Upload Linux integration tests artifact
317327
uses: actions/[email protected]
318328
if: contains(needs.check_and_prepare.outputs.platform, 'Linux') && ${{ !cancelled() }}
319329
with:
320-
name: testapps-${{matrix.os}}-${{ matrix.unity_version }}-ubuntu-latest
321-
path: testapps-${{matrix.os}}-${{ matrix.unity_version }}-all/Linux
330+
name: testapps-${{ matrix.unity_version }}-${{matrix.os}}-ubuntu-latest
331+
path: testapps-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}/Linux
322332
retention-days: ${{ env.artifactRetentionDays }}
323333
- name: Upload macOS integration tests artifact
324334
uses: actions/[email protected]
325335
if: contains(needs.check_and_prepare.outputs.platform, 'macOS') && ${{ !cancelled() }}
326336
with:
327-
name: testapps-${{matrix.os}}-${{ matrix.unity_version }}-macos-latest
328-
path: testapps-${{matrix.os}}-${{ matrix.unity_version }}-all/macOS
337+
name: testapps-${{ matrix.unity_version }}-${{matrix.os}}-macos-latest
338+
path: testapps-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}/macOS
329339
retention-days: ${{ env.artifactRetentionDays }}
330340
- name: Upload Windows integration tests artifact
331341
uses: actions/[email protected]
332342
if: contains(needs.check_and_prepare.outputs.platform, 'Windows') && ${{ !cancelled() }}
333343
with:
334-
name: testapps-${{matrix.os}}-${{ matrix.unity_version }}-windows-latest
335-
path: testapps-${{matrix.os}}-${{ matrix.unity_version }}-all/Windows
344+
name: testapps-${{ matrix.unity_version }}-${{matrix.os}}-windows-latest
345+
path: testapps-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}/Windows
336346
retention-days: ${{ env.artifactRetentionDays }}
337347
- name: Download log artifacts
338348
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
@@ -342,6 +352,7 @@ jobs:
342352
name: log-artifact
343353
- name: Update PR label and comment
344354
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
355+
shell: bash
345356
run: |
346357
python scripts/gha/it_workflow.py --stage progress \
347358
--token ${{github.token}} \
@@ -353,14 +364,14 @@ jobs:
353364
if: ${{ !cancelled() }}
354365
shell: bash
355366
run: |
356-
cat build-results-${{matrix.os}}-${{ matrix.unity_version }}-all.log
367+
cat build-results-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}.log
357368
if [[ "${{ job.status }}" != "success" ]]; then
358369
exit 1
359370
fi
360371
361372
362373
test_desktop:
363-
name: test-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}-desktop
374+
name: test-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop
364375
runs-on: ${{ matrix.os }}
365376
if: (contains(needs.check_and_prepare.outputs.platform, 'Windows') || contains(needs.check_and_prepare.outputs.platform, 'macOS') || contains(needs.check_and_prepare.outputs.platform, 'Linux')) && !cancelled()
366377
needs: [check_and_prepare, build]
@@ -378,7 +389,7 @@ jobs:
378389
uses: actions/[email protected]
379390
with:
380391
path: testapps
381-
name: testapps-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}
392+
name: testapps-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}
382393
- name: Setup python
383394
uses: actions/setup-python@v2
384395
with:
@@ -393,20 +404,20 @@ jobs:
393404
shell: bash
394405
run: |
395406
python scripts/gha/desktop_tester.py --testapp_dir testapps \
396-
--logfile_name "${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}-desktop"
407+
--logfile_name "${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop"
397408
- name: Prepare results summary artifact
398409
if: ${{ !cancelled() }}
399410
shell: bash
400411
run: |
401-
if [ ! -f testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}-desktop.log.json ]; then
402-
mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}-desktop.log.json
412+
if [ ! -f testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop.log.json ]; then
413+
mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop.log.json
403414
fi
404415
- name: Upload Desktop test results artifact
405416
if: ${{ !cancelled() }}
406417
uses: actions/[email protected]
407418
with:
408419
name: log-artifact
409-
path: testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}-desktop*
420+
path: testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop*
410421
retention-days: ${{ env.artifactRetentionDays }}
411422
- name: Download log artifacts
412423
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
@@ -429,13 +440,13 @@ jobs:
429440
if: ${{ !cancelled() }}
430441
shell: bash
431442
run: |
432-
cat testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}-desktop.log
443+
cat testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop.log
433444
if [[ "${{ job.status }}" != "success" ]]; then
434445
exit 1
435446
fi
436447
437448
test_mobile:
438-
name: test-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}-${{ matrix.mobile_test_on }}
449+
name: test-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.platform }}-${{ matrix.mobile_test_on }}
439450
runs-on: macos-11
440451
if: (contains(needs.check_and_prepare.outputs.platform, 'Android') || contains(needs.check_and_prepare.outputs.platform, 'iOS')) && !cancelled()
441452
needs: [check_and_prepare, build]
@@ -459,7 +470,7 @@ jobs:
459470
uses: actions/[email protected]
460471
with:
461472
path: testapps
462-
name: testapps-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}
473+
name: testapps-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.platform }}
463474
- name: Setup python
464475
uses: actions/setup-python@v2
465476
with:
@@ -479,29 +490,29 @@ jobs:
479490
run: |
480491
python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
481492
python scripts/gha/test_lab.py --testapp_dir testapps \
482-
--logfile_name "${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}" \
493+
--logfile_name "${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.platform }}" \
483494
--code_platform unity \
484495
--key_file scripts/gha-encrypted/gcs_key_file.json
485496
- name: Run Mobile integration tests on virtual device locally
486497
timeout-minutes: 60
487498
if: matrix.mobile_test_on == 'virtual'
488499
run: |
489500
python scripts/gha/test_simulator.py --testapp_dir testapps \
490-
--logfile_name "${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}" \
501+
--logfile_name "${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.platform }}" \
491502
--ci
492503
- name: Prepare results summary artifact
493504
if: ${{ !cancelled() }}
494505
shell: bash
495506
run: |
496-
if [ ! -f testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}.log.json ]; then
497-
mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}.log.json
507+
if [ ! -f testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.platform }}.log.json ]; then
508+
mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.platform }}.log.json
498509
fi
499510
- name: Upload Desktop test results artifact
500511
if: ${{ !cancelled() }}
501512
uses: actions/[email protected]
502513
with:
503514
name: log-artifact
504-
path: testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}*
515+
path: testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.platform }}*
505516
retention-days: ${{ env.artifactRetentionDays }}
506517
- name: Download log artifacts
507518
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
@@ -522,7 +533,63 @@ jobs:
522533
if: ${{ !cancelled() }}
523534
shell: bash
524535
run: |
525-
cat testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}.log
536+
cat testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.platform }}.log
526537
if [[ "${{ job.status }}" != "success" ]]; then
527538
exit 1
528539
fi
540+
541+
542+
summarize_results:
543+
name: "summarize-results"
544+
needs: [check_and_prepare, build, test_desktop, test_mobile]
545+
runs-on: ubuntu-latest
546+
if: ${{ !cancelled() }}
547+
steps:
548+
- uses: actions/checkout@v2
549+
with:
550+
ref: ${{needs.check_and_prepare.outputs.github_ref}}
551+
- name: Setup python
552+
uses: actions/setup-python@v2
553+
with:
554+
python-version: ${{ env.pythonVersion }}
555+
- name: Install python deps
556+
run: pip install -r scripts/gha/requirements.txt
557+
- name: Download log artifacts
558+
uses: actions/[email protected]
559+
with:
560+
path: test_results
561+
name: log-artifact
562+
# Use a different token to remove the "in-progress" label,
563+
# to allow the removal to trigger the "Check Labels" workflow.
564+
- name: Generate token for GitHub API
565+
uses: tibdex/github-app-token@v1
566+
id: generate-token
567+
with:
568+
app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
569+
private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
570+
- name: Update PR label and comment
571+
if: ${{ needs.check_and_prepare.outputs.pr_number }}
572+
run: |
573+
python scripts/gha/it_workflow.py --stage end \
574+
--token ${{github.token}} \
575+
--issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
576+
--actor ${{github.actor}} \
577+
--commit ${{needs.check_and_prepare.outputs.github_ref}} \
578+
--run_id ${{github.run_id}} \
579+
--new_token ${{steps.generate-token.outputs.token}}
580+
- name: Update Daily Report
581+
if: needs.check_and_prepare.outputs.trigger == 'scheduled_trigger'
582+
run: |
583+
if [[ "${{ github.event.inputs.test_pull_request }}" == "nightly-packaging" ]]; then
584+
additional_flags=(--build_against sdk)
585+
else
586+
additional_flags=(--build_against repo)
587+
fi
588+
python scripts/gha/it_workflow.py --stage report \
589+
--token ${{github.token}} \
590+
--actor ${{github.actor}} \
591+
--commit ${{needs.check_and_prepare.outputs.github_ref}} \
592+
--run_id ${{github.run_id}} \
593+
${additional_flags[*]}
594+
- name: Summarize results into GitHub log
595+
run: python scripts/gha/summarize_test_results.py --dir test_results --github_log

‎scripts/gha/build_testapps.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -975,11 +975,11 @@ def _run(args, timeout=3000, capture_output=False, text=None, check=True):
975975
class Failure(object):
976976
"""Holds context for the failure of a testapp to build/run."""
977977
testapp = attr.ib()
978-
platform = attr.ib()
978+
description = attr.ib()
979979
error_message = attr.ib()
980980

981981
def describe(self):
982-
return "%s, %s: %s" % (self.testapp, self.platform, self.error_message)
982+
return "%s, %s: %s" % (self.testapp, self.description, self.error_message)
983983

984984

985985
@attr.s(frozen=True, eq=False)

‎scripts/gha/integration_testing/test_validation.py

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@
3838

3939
import attr
4040

41+
from print_matrix_configuration import PARAMETERS
42+
4143
UNITY = "unity"
4244
CPP = "cpp"
4345

@@ -263,14 +265,10 @@ def summarize_test_results(tests, platform, summary_dir, file_name="summary.log"
263265
summary_json["failures"][testapp]["ftl_links"].append(test.ftl_link)
264266
if hasattr(test, "raw_result_link") and test.raw_result_link:
265267
summary_json["failures"][testapp]["raw_result_links"].append(test.raw_result_link)
266-
failed_tests = re.findall(r"\[ FAILED \] (.+)[.](.+)", results.summary)
268+
failed_tests = re.findall(r"Test (.+) failed!", results.summary)
267269
for failed_test in failed_tests:
268-
failed_test = failed_test[0] + "." + failed_test[1]
269-
pattern = fr'\[ RUN \] {failed_test}(.*?)\[ FAILED \] {failed_test}'
270-
failure_log = re.search(pattern, test.logs, re.MULTILINE | re.DOTALL)
271-
if failure_log:
272-
summary_json["failures"][testapp]["failed_tests"][failed_test] = failure_log.group()
273-
summary.append("\n%s FAILED:\n%s\n" % (failed_test, failure_log.group()))
270+
summary_json["failures"][testapp]["failed_tests"][failed_test] = "See workflow log"
271+
summary.append("\n%s FAILED\n" % (failed_test))
274272
summary_json["flakiness"] = {get_name(test.testapp_path):{"logs": [], "ftl_links": [], "raw_result_links": [], "flaky_tests": dict()} for (test, _) in flaky_testapps}
275273
for (test, results) in flaky_testapps:
276274
testapp = get_name(test.testapp_path)
@@ -279,14 +277,10 @@ def summarize_test_results(tests, platform, summary_dir, file_name="summary.log"
279277
summary_json["flakiness"][testapp]["ftl_links"].append(test.ftl_link)
280278
if hasattr(test, "raw_result_link") and test.raw_result_link:
281279
summary_json["flakiness"][testapp]["raw_result_links"].append(test.raw_result_link)
282-
flaky_tests = re.findall(r"\[ FAILED \] (.+)[.](.+)", results.summary)
280+
flaky_tests = re.findall(r"Test (.+) failed!", results.summary)
283281
for flaky_test in flaky_tests:
284-
flaky_test = flaky_test[0] + "." + flaky_test[1]
285-
pattern = fr'\[ RUN \] {flaky_test}(.*?)\[ FAILED \] {flaky_test}'
286-
failure_log = re.search(pattern, test.logs, re.MULTILINE | re.DOTALL)
287-
if failure_log:
288-
summary_json["flakiness"][testapp]["flaky_tests"][flaky_test] = failure_log.group()
289-
summary.append("\n%s FAILED:\n%s\n" % (flaky_test, failure_log.group()))
282+
summary_json["flakiness"][testapp]["flaky_tests"][flaky_test] = "See workflow log"
283+
summary.append("\n%s FAILED\n" % (flaky_test))
290284

291285
with open(os.path.join(summary_dir, file_name+".json"), "a") as f:
292286
f.write(json.dumps(summary_json, indent=2))
@@ -325,7 +319,11 @@ def write_summary(testapp_dir, summary, file_name="summary.log"):
325319

326320
def get_name(testapp_path):
327321
"""Returns testapp api."""
328-
return testapp_path.split(os.sep)[-2]
322+
testapps = PARAMETERS["integration_tests"]["config"]["apis"].split(",")
323+
for testapp in testapps:
324+
if testapp in testapp_path:
325+
return testapp
326+
return testapp_path
329327

330328

331329
def _tail(text, n):

0 commit comments

Comments
 (0)