Skip to content

Commit c9b33af

Browse files
sunmou99sunmou99
authored
Preparation for Integration Test Summary Report (#145)
1 parent a23b60b commit c9b33af

File tree

5 files changed

+553
-23
lines changed

5 files changed

+553
-23
lines changed

‎.github/workflows/integration_tests.yml

Lines changed: 162 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ on:
44
schedule:
55
- cron: "0 9 * * *" # 9am UTC = 1am PST / 2am PDT
66

7+
pull_request:
8+
types: [ labeled, closed ]
9+
710
workflow_dispatch:
811
inputs:
912
build_os:
@@ -41,6 +44,9 @@ on:
4144
description: 'Optional: Pull request # to build and test? (With optional commit hash, separated by a colon. Specify the FULL hash.)'
4245

4346
env:
47+
triggerLabelPrefix: "tests-requested: "
48+
triggerLabelFull: "tests-requested: full"
49+
triggerLabelQuick: "tests-requested: quick"
4450
pythonVersion: '3.7'
4551
artifactRetentionDays: 2
4652

@@ -69,32 +75,69 @@ jobs:
6975
env:
7076
GITHUB_TOKEN: ${{ github.token }}
7177
### It sets "github_ref,trigger,pr_number,requested_tests" outputs to control the following jobs and steps
72-
### trigger value: manual_trigger, scheduled_trigger;
73-
### TODO: label_trigger, postsubmit_trigger, presubmit_trigger
78+
### trigger value: manual_trigger, scheduled_trigger, label_trigger, postsubmit_trigger
7479
- id: set_outputs
7580
run: |
7681
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
77-
echo "::set-output name=trigger::manual_trigger"
78-
if [[ "${{ github.event.inputs.use_expanded_matrix }}" == "1" ]]; then
79-
echo "::set-output name=requested_tests::expanded"
80-
fi
81-
if [[ -z "${{github.event.inputs.test_pull_request}}" ]]; then
82-
# test_pull_request not specified
82+
if [[ "${{ github.event.inputs.test_pull_request }}" != "nightly-packaging" ]]; then
83+
# Triggered manually
84+
echo "::set-output name=trigger::manual_trigger"
85+
if [[ "${{ github.event.inputs.use_expanded_matrix }}" == "1" ]]; then
86+
echo "::set-output name=requested_tests::expanded"
87+
fi
88+
if [[ -z "${{github.event.inputs.test_pull_request}}" ]]; then
89+
# test_pull_request not specified
90+
echo "::set-output name=github_ref::$GITHUB_SHA"
91+
elif [[ "${{github.event.inputs.test_pull_request}}" == *:* ]]; then
92+
# If specified as pr:commit_hash, split them.
93+
echo "::set-output name=github_ref::$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f2)"
94+
echo "::set-output name=pr_number::$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f1)"
95+
else
96+
# Just the PR specified, use refs/pull/<number>/merge as the ref.
97+
echo "::set-output name=github_ref::refs/pull/${{github.event.inputs.test_pull_request}}/merge"
98+
echo "::set-output name=pr_number::${{ github.event.inputs.test_pull_request }}"
99+
fi
100+
elif [[ "${{ github.event.inputs.test_pull_request }}" == "nightly-packaging" ]]; then
101+
# Triggered by scheduled packaging SDK workflow.
102+
echo "::set-output name=trigger::scheduled_trigger"
83103
echo "::set-output name=github_ref::$GITHUB_SHA"
84-
elif [[ "${{github.event.inputs.test_pull_request}}" == *:* ]]; then
85-
# If specified as pr:commit_hash, split them.
86-
echo "::set-output name=github_ref::$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f2)"
87-
echo "::set-output name=pr_number::$(echo ${{ github.event.inputs.test_pull_request }} | cut -d: -f1)"
88-
else
89-
# Just the PR specified, use refs/pull/<number>/merge as the ref.
90-
echo "::set-output name=github_ref::refs/pull/${{github.event.inputs.test_pull_request}}/merge"
91-
echo "::set-output name=pr_number::${{ github.event.inputs.test_pull_request }}"
104+
echo "::set-output name=requested_tests::expanded"
92105
fi
93106
elif [[ "${{ github.event_name }}" == "schedule" ]]; then
94107
echo "::set-output name=trigger::scheduled_trigger"
95108
echo "::set-output name=github_ref::$GITHUB_SHA"
96109
echo "::set-output name=requested_tests::expanded"
110+
elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
111+
echo "::set-output name=github_ref::$GITHUB_SHA"
112+
if [[ "${{ github.event.action }}" == "labeled" && "${{ github.event.label.name }}" == "${{ env.triggerLabelPrefix }}"* ]]; then
113+
echo "::set-output name=trigger::label_trigger"
114+
echo "::set-output name=pr_number::${{ github.event.pull_request.number }}"
115+
if [[ "${{ github.event.label.name }}" == "${{ env.triggerLabelQuick }}" ]]; then
116+
echo "::set-output name=requested_tests::auto"
117+
else
118+
echo "::set-output name=requested_tests::expanded"
119+
fi
120+
elif [[ "${{ github.event.action }}" == "closed" && ${{ github.event.pull_request.merged == true }} ]]; then
121+
echo "::set-output name=trigger::postsubmit_trigger"
122+
echo "::set-output name=pr_number::${{ github.event.pull_request.number }}"
123+
echo "::set-output name=requested_tests::auto"
124+
fi
97125
fi
126+
### If it's not a defined trigger, cancel workflow
127+
### e.g. Triggered by non-"test-request" label; triggered by not merged PR close event.
128+
- name: Cancel workflow
129+
if: ${{ !steps.set_outputs.outputs.trigger }}
130+
uses: andymckay/[email protected]
131+
- name: Wait for workflow cancellation
132+
if: ${{ !steps.set_outputs.outputs.trigger }}
133+
run: |
134+
sleep 300
135+
exit 1 # fail out if the cancellation above somehow failed.
136+
- name: Cancel previous runs on the same PR
137+
if: steps.set_outputs.outputs.trigger == 'label_trigger'
138+
uses: styfle/[email protected]
139+
with:
140+
access_token: ${{ github.token }}
98141
- uses: actions/checkout@v2
99142
with:
100143
ref: ${{steps.set_outputs.outputs.github_ref}}
@@ -111,6 +154,19 @@ jobs:
111154
if [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "expanded" ]]; then
112155
TEST_MATRIX_PARAM=-e=1
113156
echo "::warning ::Running on the expanded matrix"
157+
elif [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "minimal" ]]; then
158+
TEST_MATRIX_PARAM=-m=1
159+
echo "::warning ::Running on the minimal matrix"
160+
elif [[ "${{ steps.set_outputs.outputs.requested_tests }}" == "auto" ]]; then
161+
# auto-diff only apply when running in a PR.
162+
# diff against the PR's base. "git merge-base main branch_name" will give the common ancestor of both branches.
163+
MERGE_BASE=$(git merge-base origin/${{github.event.pull_request.head.ref}} origin/${{github.event.pull_request.base.ref}} || true)
164+
# If origin/<branch> is no longer valid, then just run all tests.
165+
if [[ -n "${MERGE_BASE}" ]]; then
166+
echo "::warning ::Auto-diff origin/${{github.event.pull_request.head.ref}}..${MERGE_BASE}"
167+
git diff --name-only origin/${{github.event.pull_request.head.ref}}..${MERGE_BASE}
168+
TEST_MATRIX_PARAM="--auto_diff origin/${{github.event.pull_request.head.ref}}..${MERGE_BASE}"
169+
fi
114170
fi
115171
116172
# To feed input into the job matrix, we first need to convert to a JSON
@@ -228,6 +284,21 @@ jobs:
228284
--version ${{ matrix.unity_version }} \
229285
--logfile "testapps/release_license.log"
230286
cat testapps/release_license.log
287+
- name: Prepare results summary artifact
288+
if: ${{ !cancelled() }}
289+
shell: bash
290+
run: |
291+
if [ ! -f build-results-${{matrix.os}}-${{ matrix.unity_version }}-all.log.json ]; then
292+
# No summary was created, make a placeholder one.
293+
echo "__SUMMARY_MISSING__" > build-results-${{matrix.os}}-${{ matrix.unity_version }}-all.log.json
294+
fi
295+
- name: Upload build results artifact
296+
uses: actions/[email protected]
297+
if: ${{ !cancelled() }}
298+
with:
299+
name: log-artifact
300+
path: build-results-${{matrix.os}}-${{ matrix.unity_version }}-all*
301+
retention-days: ${{ env.artifactRetentionDays }}
231302
- name: Upload Android integration tests artifact
232303
uses: actions/[email protected]
233304
if: contains(needs.check_and_prepare.outputs.platform, 'Android') && ${{ !cancelled() }}
@@ -263,6 +334,21 @@ jobs:
263334
name: testapps-${{matrix.os}}-${{ matrix.unity_version }}-windows-latest
264335
path: testapps-${{matrix.os}}-${{ matrix.unity_version }}-all/Windows
265336
retention-days: ${{ env.artifactRetentionDays }}
337+
- name: Download log artifacts
338+
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
339+
uses: actions/[email protected]
340+
with:
341+
path: test_results
342+
name: log-artifact
343+
- name: Update PR label and comment
344+
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
345+
run: |
346+
python scripts/gha/it_workflow.py --stage progress \
347+
--token ${{github.token}} \
348+
--issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
349+
--actor ${{github.actor}} \
350+
--commit ${{needs.check_and_prepare.outputs.github_ref}} \
351+
--run_id ${{github.run_id}}
266352
- name: Summarize build results
267353
if: ${{ !cancelled() }}
268354
shell: bash
@@ -308,6 +394,37 @@ jobs:
308394
run: |
309395
python scripts/gha/desktop_tester.py --testapp_dir testapps \
310396
--logfile_name "${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}-desktop"
397+
- name: Prepare results summary artifact
398+
if: ${{ !cancelled() }}
399+
shell: bash
400+
run: |
401+
if [ ! -f testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}-desktop.log.json ]; then
402+
mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}-desktop.log.json
403+
fi
404+
- name: Upload Desktop test results artifact
405+
if: ${{ !cancelled() }}
406+
uses: actions/[email protected]
407+
with:
408+
name: log-artifact
409+
path: testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.os }}-desktop*
410+
retention-days: ${{ env.artifactRetentionDays }}
411+
- name: Download log artifacts
412+
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
413+
uses: actions/[email protected]
414+
with:
415+
path: test_results
416+
name: log-artifact
417+
- name: Update PR label and comment
418+
shell: bash
419+
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
420+
run: |
421+
pushd ${{env.GCS_UPLOAD_DIR}}
422+
python scripts/gha/it_workflow.py --stage progress \
423+
--token ${{github.token}} \
424+
--issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
425+
--actor ${{github.actor}} \
426+
--commit ${{needs.check_and_prepare.outputs.github_ref}} \
427+
--run_id ${{github.run_id}}
311428
- name: Summarize test results
312429
if: ${{ !cancelled() }}
313430
shell: bash
@@ -372,6 +489,35 @@ jobs:
372489
python scripts/gha/test_simulator.py --testapp_dir testapps \
373490
--logfile_name "${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}" \
374491
--ci
492+
- name: Prepare results summary artifact
493+
if: ${{ !cancelled() }}
494+
shell: bash
495+
run: |
496+
if [ ! -f testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}.log.json ]; then
497+
mkdir -p testapps && echo "__SUMMARY_MISSING__" > testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}.log.json
498+
fi
499+
- name: Upload Desktop test results artifact
500+
if: ${{ !cancelled() }}
501+
uses: actions/[email protected]
502+
with:
503+
name: log-artifact
504+
path: testapps/test-results-${{ matrix.build_os }}-${{ matrix.unity_version }}-${{ matrix.platform }}*
505+
retention-days: ${{ env.artifactRetentionDays }}
506+
- name: Download log artifacts
507+
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
508+
uses: actions/[email protected]
509+
with:
510+
path: test_results
511+
name: log-artifact
512+
- name: Update PR label and comment
513+
if: ${{ needs.check_and_prepare.outputs.pr_number && failure() && !cancelled() }}
514+
run: |
515+
python scripts/gha/it_workflow.py --stage progress \
516+
--token ${{github.token}} \
517+
--issue_number ${{needs.check_and_prepare.outputs.pr_number}}\
518+
--actor ${{github.actor}} \
519+
--commit ${{needs.check_and_prepare.outputs.github_ref}} \
520+
--run_id ${{github.run_id}}
375521
- name: Summarize test results
376522
if: ${{ !cancelled() }}
377523
shell: bash

‎scripts/gha/build_testapps.py

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@
9999
import time
100100
import requests
101101
import zipfile
102+
import json
102103

103104
from absl import app
104105
from absl import flags
@@ -320,7 +321,7 @@ def main(argv):
320321
try:
321322
setup_unity_project(dir_helper, setup_options)
322323
except (subprocess.SubprocessError, RuntimeError) as e:
323-
failures.append(Failure(description=build_desc, error_message=str(e)))
324+
failures.append(Failure(testapp=testapp, description=build_desc, error_message=str(e)))
324325
logging.info(str(e))
325326
continue # If setup failed, don't try to build. Move to next testapp.
326327
for p in platforms:
@@ -338,6 +339,7 @@ def main(argv):
338339
except (subprocess.SubprocessError, RuntimeError) as e:
339340
failures.append(
340341
Failure(
342+
testapp=testapp,
341343
description=build_desc + " " + p,
342344
error_message=str(e)))
343345
logging.info(str(e))
@@ -664,6 +666,13 @@ def _summarize_results(testapps, platforms, versions, failures, output_dir, arti
664666

665667
logging.info(summary)
666668
test_validation.write_summary(output_dir, summary, file_name)
669+
670+
summary_json = {}
671+
summary_json["type"] = "build"
672+
summary_json["testapps"] = testapps
673+
summary_json["errors"] = {failure.testapp:failure.error_message for failure in failures}
674+
with open(os.path.join(output_dir, file_name+".json"), "a") as f:
675+
f.write(json.dumps(summary_json, indent=2))
667676
return 1 if failures else 0
668677

669678

@@ -965,11 +974,12 @@ def _run(args, timeout=3000, capture_output=False, text=None, check=True):
965974
@attr.s(frozen=True, eq=False)
966975
class Failure(object):
967976
"""Holds context for the failure of a testapp to build/run."""
968-
description = attr.ib()
977+
testapp = attr.ib()
978+
platform = attr.ib()
969979
error_message = attr.ib()
970980

971981
def describe(self):
972-
return "%s: %s" % (self.description, self.error_message)
982+
return "%s, %s: %s" % (self.testapp, self.platform, self.error_message)
973983

974984

975985
@attr.s(frozen=True, eq=False)

0 commit comments

Comments
 (0)