Compare Test Time Elapsed #194
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compare Test Time Elapsed | |
| on: | |
| workflow_run: | |
| # This runs on the default branch when a CI workflow on any branch is complete | |
| workflows: ["CI"] | |
| types: | |
| - completed | |
| env: | |
| HEAD_DIFF_TOL: 0.05 # Fail if test time increases by more than this ratio, against default branch head | |
| REL_DIFF_TOL: 0.10 # For comparison against latest release | |
| DIFF_THRESHOLD_MS: 3 # Only check tests which take more than this time | |
| jobs: | |
| check-elapsed-test-times: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [windows-2019, ubuntu-latest, macos-14-large, macos-latest] | |
| fail-fast: false | |
| steps: | |
| - name: Get Filename | |
| if: ${{ matrix.os == 'windows-2019' }} | |
| shell: bash | |
| run: echo "CI_WF=SSC Windows Test Time Elapsed" >> $GITHUB_ENV | |
| - name: Get Filename | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: echo "CI_WF=SSC Linux Test Time Elapsed" >> $GITHUB_ENV | |
| - name: Get Filename | |
| if: ${{ matrix.os == 'macos-14-large' }} | |
| run: echo "CI_WF=SSC Mac Intel Test Time Elapsed" >> $GITHUB_ENV | |
| - name: Get Filename | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| run: echo "CI_WF=SSC Mac Arm Test Time Elapsed" >> $GITHUB_ENV | |
| - name: Check filename | |
| shell: bash | |
| run: | | |
| echo "Downloading" ${{env.CI_WF}} "from" ${{github.workflow_ref}} $GITHUB_REF | |
| - name: Get Calling Workflow | |
| uses: actions/github-script@v7 | |
| id: workflow_id | |
| with: | |
| script: | | |
| return context.payload.workflow_run.id | |
| result-encoding: string | |
| - name: Export Context Vars to ENV | |
| shell: bash | |
| run: | | |
| echo "WORKFLOW_ID=${{ steps.workflow_id.outputs.result }}" >> $GITHUB_ENV | |
| - name: Checkout SSC | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ssc | |
| - name: Check Test-Running Workflows were Successfully Completed | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| shell: bash | |
| run: | | |
| pip install pandas requests | |
| python '${{ github.workspace }}/ssc/test/compare_elapsed_time.py' check_workflow ${{env.WORKFLOW_ID}} | |
| - name: Get Artifact from Completed CI (on Feature Branch) | |
| uses: actions/github-script@v7 | |
| id: get-artifact | |
| with: | |
| script: | | |
| let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| run_id: context.payload.workflow_run.id | |
| }); | |
| console.log("Downloaded Elapsed Test Times from Workflow ID:"); | |
| console.log(context.payload.workflow_run.id); | |
| let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { | |
| return (artifact.name == "${{env.CI_WF}}") | |
| })[0]; | |
| let download = await github.rest.actions.downloadArtifact({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| artifact_id: matchArtifact.id, | |
| archive_format: 'zip', | |
| }); | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const temp = '${{ github.workspace }}'; | |
| if (!fs.existsSync(temp)){ | |
| fs.mkdirSync(temp); | |
| } | |
| fs.writeFileSync(path.join(temp, 'test_times.zip'), Buffer.from(download.data)); | |
| return path.join(temp, 'test_times.zip') | |
| result-encoding: string | |
| - name: Unzip Test Times | |
| if: ${{ matrix.os == 'windows-2019' }} | |
| run: | | |
| 7z x ${{steps.get-artifact.outputs.result}} | |
| - name: Unzip Test Times | |
| if: ${{ matrix.os != 'windows-2019' }} | |
| run: unzip ${{steps.get-artifact.outputs.result}} | |
| - name: Compare Test Times | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| shell: bash | |
| run: | | |
| # compare the downloaded times from the feature branch to that of the default branch | |
| echo ${{ github.ref_name }} | |
| outputString=$(python '${{ github.workspace }}/ssc/test/compare_elapsed_time.py' compare '${{ github.workspace }}/gtest_elapsed_times.csv' ${{ github.ref_name }}) | |
| echo $outputString | |
| echo "RESULT=$outputString" >> $GITHUB_ENV | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ env.RESULT != 'Pass' }} | |
| with: | |
| name: Test Time Elapsed ${{ matrix.os }} | |
| path: | | |
| ${{ github.workspace }}/ssc/test/failing_test_times.csv | |
| - name: Enforce Test Times Limits | |
| shell: bash | |
| run: | | |
| if [ "$RESULT" != "Pass" ] | |
| then | |
| exit 1 | |
| fi |