Skip to content

Commit 1e23d8e

Browse files
sunmou99sunmou99
authored
don't report test if build failed & disable failed testapps (#196)
1 parent db59d10 commit 1e23d8e

File tree

4 files changed

+27
-29
lines changed

4 files changed

+27
-29
lines changed

‎.github/workflows/integration_tests.yml

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -302,14 +302,14 @@ jobs:
302302
echo "__SUMMARY_MISSING__" > build-results-${{ matrix.unity_version }}-${{matrix.os}}-${{ matrix.platform }}.log.json
303303
fi
304304
- name: Upload build results artifact
305-
uses: actions/upload-artifact@v2.2.2
305+
uses: actions/upload-artifact@v3
306306
if: ${{ !cancelled() }}
307307
with:
308308
name: log-artifact
309309
path: build-results-${{ matrix.unity_version }}-${{matrix.os}}-${{ matrix.platform }}*
310310
retention-days: ${{ env.artifactRetentionDays }}
311311
- name: Upload Android integration tests artifact
312-
uses: actions/upload-artifact@v2.2.2
312+
uses: actions/upload-artifact@v3
313313
if: contains(matrix.platform, 'Android') && ${{ !cancelled() }}
314314
with:
315315
name: testapps-${{ matrix.unity_version }}-${{matrix.os}}-Android
@@ -320,7 +320,7 @@ jobs:
320320
shell: bash
321321
run: rm -rf testapps-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}/Android || true
322322
- name: Upload iOS integration tests artifact
323-
uses: actions/upload-artifact@v2.2.2
323+
uses: actions/upload-artifact@v3
324324
if: contains(matrix.platform, 'iOS') && ${{ !cancelled() }}
325325
with:
326326
name: testapps-${{ matrix.unity_version }}-${{matrix.os}}-iOS
@@ -331,7 +331,7 @@ jobs:
331331
shell: bash
332332
run: rm -rf testapps-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}/iOS || true
333333
- name: Upload Linux integration tests artifact
334-
uses: actions/upload-artifact@v2.2.2
334+
uses: actions/upload-artifact@v3
335335
if: contains(matrix.platform, 'Linux') && ${{ !cancelled() }}
336336
with:
337337
name: testapps-${{ matrix.unity_version }}-${{matrix.os}}-ubuntu-latest
@@ -342,7 +342,7 @@ jobs:
342342
shell: bash
343343
run: rm -rf testapps-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}/Linux || true
344344
- name: Upload macOS integration tests artifact
345-
uses: actions/upload-artifact@v2.2.2
345+
uses: actions/upload-artifact@v3
346346
if: contains(matrix.platform, 'macOS') && ${{ !cancelled() }}
347347
with:
348348
name: testapps-${{ matrix.unity_version }}-${{matrix.os}}-macos-latest
@@ -353,7 +353,7 @@ jobs:
353353
shell: bash
354354
run: rm -rf testapps-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}/macOS || true
355355
- name: Upload Windows integration tests artifact
356-
uses: actions/upload-artifact@v2.2.2
356+
uses: actions/upload-artifact@v3
357357
if: contains(matrix.platform, 'Windows') && ${{ !cancelled() }}
358358
with:
359359
name: testapps-${{ matrix.unity_version }}-${{matrix.os}}-windows-latest
@@ -365,7 +365,7 @@ jobs:
365365
run: rm -rf testapps-${{ matrix.unity_version }}-${{matrix.os}}-${{ needs.check_and_prepare.outputs.platform }}/Windows || true
366366
- name: Download log artifacts
367367
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
368-
uses: actions/download-artifact@v2.0.8
368+
uses: actions/download-artifact@v3
369369
with:
370370
path: test_results
371371
name: log-artifact
@@ -405,7 +405,7 @@ jobs:
405405
with:
406406
ref: ${{needs.check_and_prepare.outputs.github_ref}}
407407
- name: Download Desktop integration tests artifact
408-
uses: actions/download-artifact@v2.0.8
408+
uses: actions/download-artifact@v3
409409
with:
410410
path: testapps
411411
name: testapps-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}
@@ -428,19 +428,20 @@ jobs:
428428
if: ${{ !cancelled() }}
429429
shell: bash
430430
run: |
431-
if [ ! -f testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop.log.json ]; then
431+
# If testapps do not exist, then it's a build error not test error.
432+
if [ -d "testapps/testapps-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}" && ! -f testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop.log.json ]; then
432433
mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop.log.json
433434
fi
434435
- name: Upload Desktop test results artifact
435436
if: ${{ !cancelled() }}
436-
uses: actions/upload-artifact@v2.2.2
437+
uses: actions/upload-artifact@v3
437438
with:
438439
name: log-artifact
439440
path: testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.os }}-desktop*
440441
retention-days: ${{ env.artifactRetentionDays }}
441442
- name: Download log artifacts
442443
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
443-
uses: actions/download-artifact@v2.0.8
444+
uses: actions/download-artifact@v3
444445
with:
445446
path: test_results
446447
name: log-artifact
@@ -492,7 +493,7 @@ jobs:
492493
echo "::set-output name=device_type::$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.mobile_device }} -get_device_type)"
493494
echo "::set-output name=device_platform::$( python scripts/gha/print_matrix_configuration.py -k ${{ matrix.mobile_device }} -get_device_platform)"
494495
- name: Download Desktop integration tests artifact
495-
uses: actions/download-artifact@v2.0.8
496+
uses: actions/download-artifact@v3
496497
with:
497498
path: testapps
498499
name: testapps-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ steps.device-info.outputs.device_platform }}
@@ -524,19 +525,20 @@ jobs:
524525
if: ${{ !cancelled() }}
525526
shell: bash
526527
run: |
527-
if [ ! -f testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.mobile_device }}-mobile.log.json ]; then
528+
# If testapps do not exist, then it's a build error not test error.
529+
if [ -d "testapps/testapps-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ steps.device-info.outputs.device_platform }}" && ! -f testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.mobile_device }}-mobile.log.json ]; then
528530
mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.mobile_device }}-mobile.log.json
529531
fi
530532
- name: Upload Mobile test results artifact
531533
if: ${{ !cancelled() }}
532-
uses: actions/upload-artifact@v2.2.2
534+
uses: actions/upload-artifact@v3
533535
with:
534536
name: log-artifact
535537
path: testapps/test-results-${{ matrix.unity_version }}-${{ matrix.build_os }}-${{ matrix.mobile_device }}-mobile*
536538
retention-days: ${{ env.artifactRetentionDays }}
537539
- name: Download log artifacts
538540
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
539-
uses: actions/download-artifact@v2.0.8
541+
uses: actions/download-artifact@v3
540542
with:
541543
path: test_results
542544
name: log-artifact
@@ -575,7 +577,7 @@ jobs:
575577
- name: Install python deps
576578
run: pip install -r scripts/gha/requirements.txt
577579
- name: Download log artifacts
578-
uses: actions/download-artifact@v2.0.8
580+
uses: actions/download-artifact@v3
579581
with:
580582
path: test_results
581583
name: log-artifact
@@ -600,16 +602,10 @@ jobs:
600602
- name: Update Daily Report
601603
if: needs.check_and_prepare.outputs.trigger == 'scheduled_trigger'
602604
run: |
603-
if [[ "${{ github.event.inputs.test_pull_request }}" == "nightly-packaging" ]]; then
604-
additional_flags=(--build_against sdk)
605-
else
606-
additional_flags=(--build_against repo)
607-
fi
608605
python scripts/gha/it_workflow.py --stage report \
609606
--token ${{github.token}} \
610607
--actor ${{github.actor}} \
611608
--commit ${{needs.check_and_prepare.outputs.github_ref}} \
612-
--run_id ${{github.run_id}} \
613-
${additional_flags[*]}
609+
--run_id ${{github.run_id}}
614610
- name: Summarize results into GitHub log
615611
run: python scripts/gha/summarize_test_results.py --dir test_results --github_log

‎.github/workflows/package.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ jobs:
209209
trigger_integration_tests:
210210
# Trigger the integration_tests workflow.
211211
needs: [package_sdks]
212-
if: (github.event.inputs.skipIntegrationTests == 0 || github.event.inputs.skipIntegrationTests == '') && !cancelled()
212+
if: (github.event.inputs.skipIntegrationTests == 0 || github.event.inputs.skipIntegrationTests == '') && !cancelled() && !failure()
213213
runs-on: ubuntu-latest
214214
steps:
215215
- name: Checkout repo

‎scripts/gha/print_matrix_configuration.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,10 @@
6767
}
6868
},
6969
"config": {
70-
"platform": "Windows,macOS,Linux,Android,iOS",
71-
"apis": "analytics,auth,crashlytics,database,dynamic_links,functions,installations,messaging,remote_config,storage",
72-
"apis": "analytics,auth,crashlytics,database,dynamic_links,firestore,functions,installations,messaging,remote_config,storage",
70+
"platform": "Windows,macOS,Linux,iOS",
71+
# "platform": "Windows,macOS,Linux,Android,iOS",
72+
"apis": "analytics,auth,crashlytics,database,functions,installations,remote_config",
73+
# "apis": "analytics,auth,crashlytics,database,dynamic_links,firestore,functions,installations,messaging,remote_config,storage",
7374
"mobile_test_on": "real,virtual"
7475
}
7576
},

‎scripts/gha/summarize_test_results.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ def summarize_logs(dir, markdown=False, github_log=False):
238238
success_or_only_flakiness = False
239239
log_data.setdefault(testapp, {}).setdefault("test", {}).setdefault("errors", []).append(configs)
240240
for (testapp, failures) in log_reader_data["failures"].items():
241+
success_or_only_flakiness = False
241242
log_data.setdefault(testapp, {}).setdefault("test", {}).setdefault("failures", []).append(configs)
242243
# for (test, _) in failures["failed_tests"].items():
243244
# success_or_only_flakiness = False
@@ -360,15 +361,15 @@ def reorganize_configs(configs):
360361
# ['ubuntu', 'windows']
361362
# -> ['2/3 os: ubuntu,windows': ]
362363
def combine_configs(error_configs, all_configs):
363-
logging.info("error_configs: %s; all_configs: %s", error_configs, all_configs)
364+
# logging.info("error_configs: %s; all_configs: %s", error_configs, all_configs)
364365
for i in range(len(error_configs)):
365366
error_configs[i] = combine_config(error_configs[i], all_configs[i], i)
366367
return error_configs
367368

368369

369370
def combine_config(config, config_value, k):
370371
config_before_combination = config.copy()
371-
logging.info("config: %s; config_value: %s", config, config_value)
372+
# logging.info("config: %s; config_value: %s", config, config_value)
372373
config_name = BUILD_CONFIGS[k]
373374
# if certain config failed for all values, add message "All *"
374375
if len(config_value) > 1 and len(config) == len(config_value):

0 commit comments

Comments
 (0)