Analyses Snapshot Test #5990
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Analyses Snapshot Test | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| OPEN_PR_ON_FAILURE: | |
| description: 'If the test fails, open a PR to update the snapshots' | |
| type: boolean | |
| required: true | |
| default: false | |
| schedule: | |
| - cron: '26 7 * * *' # 7:26 AM UTC | |
| pull_request: | |
| paths: | |
| - 'api/**' | |
| - '!api/tests/**' | |
| - '!api/docs/**' | |
| - '!api/release-notes-internal.md' | |
| - '!api/release-notes.md' | |
| - 'shared-data/**' | |
| - '!shared-data/js/**' | |
| - '.github/workflows/analyses-snapshot-test.yaml' | |
| - 'analyses-snapshot-testing/**' | |
| types: | |
| - opened #default | |
| - synchronize #default | |
| - reopened #default | |
| - labeled | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| collect-chunks: | |
| timeout-minutes: 2 | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix_json: ${{ steps.set-matrix.outputs.json }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Setup UV | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| python-version: '3.10' | |
| enable-cache: true | |
| cache-dependency-glob: './analyses-snapshot-testing/uv.lock' | |
| - name: Sync the project | |
| working-directory: ./analyses-snapshot-testing | |
| run: make ci-setup | |
| - name: Generate Chunks | |
| working-directory: ./analyses-snapshot-testing | |
| run: make gen-chunks | |
| - name: Save Chunks | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: chunks | |
| path: analyses-snapshot-testing/chunks/ | |
| - name: Set Matrix | |
| id: set-matrix | |
| run: echo "json=$(jq -c . analyses-snapshot-testing/chunks/matrix.json)" >> $GITHUB_OUTPUT | |
| diff-from-target-branch: | |
| timeout-minutes: 2 | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Are the analyses snapshots in my PR branch in sync with the target branch? | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| git fetch origin ${{ github.event.pull_request.base.ref }} | |
| DIFF_OUTPUT=$(git diff HEAD origin/${{ github.event.pull_request.base.ref }} -- analyses-snapshot-testing/tests/__snapshots__/analyses_snapshot_test) | |
| if [ -n "$DIFF_OUTPUT" ]; then | |
| echo "## Analyses Snapshot Diff Check" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- :x: Analyses snapshots do **NOT** match \`${{ github.event.pull_request.base.ref }}\` snapshots." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo " - Is this because you have not pulled and merged \`${{ github.event.pull_request.base.ref }}\`?" >> $GITHUB_STEP_SUMMARY | |
| echo " - Or is this because you have already updated your snapshots and are all good 😊?" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "## Analyses Snapshot Diff Check" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo ":white_check_mark: **Analyses snapshots match \`${{ github.event.pull_request.base.ref }}\` snapshots.**" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| process-chunk: | |
| timeout-minutes: 5 | |
| needs: collect-chunks | |
| strategy: | |
| matrix: | |
| chunk: ${{ fromJson(needs.collect-chunks.outputs.matrix_json) }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Setup UV | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| python-version: '3.10' | |
| enable-cache: true | |
| cache-dependency-glob: './analyses-snapshot-testing/uv.lock' | |
| - name: Sync the project | |
| working-directory: ./analyses-snapshot-testing | |
| run: make ci-setup | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: chunks | |
| path: analyses-snapshot-testing/chunks/ | |
| - name: Generate Analyses from Chunk | |
| working-directory: ./analyses-snapshot-testing | |
| run: make analyze-chunk CHUNK=${{ matrix.chunk.chunk_id }} | |
| - name: Upload Analysis Results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| # Each chunk uploads its files as one artifact named after the chunk | |
| name: analysis-results-${{ matrix.chunk.chunk_id }}.zip | |
| path: analyses-snapshot-testing/analysis_results/* | |
| if-no-files-found: error | |
| execute-tests: | |
| timeout-minutes: 15 | |
| needs: process-chunk | |
| runs-on: ubuntu-latest | |
| env: | |
| # If we're running because of workflow_dispatch, use the user input to decide | |
| # whether to open a PR on failure. Otherwise, there is no user input, | |
| # so we only open a PR if the PR has the label 'gen-analyses-snapshot-pr' | |
| OPEN_PR_ON_FAILURE: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.OPEN_PR_ON_FAILURE) || ((github.event_name != 'workflow_dispatch') && (contains(github.event.pull_request.labels.*.name, 'gen-analyses-snapshot-pr'))) }} | |
| PR_TARGET_BRANCH: ${{ github.event.pull_request.base.ref || 'edge'}} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Setup UV | |
| uses: astral-sh/setup-uv@v6 | |
| with: | |
| python-version: '3.10' | |
| enable-cache: true | |
| cache-dependency-glob: './analyses-snapshot-testing/uv.lock' | |
| - name: Sync the project | |
| working-directory: ./analyses-snapshot-testing | |
| run: make ci-setup | |
| - name: Download All Analysis Results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: analysis-results-* | |
| path: all_analysis_results | |
| - name: Consolidate Analysis Results | |
| run: | | |
| find all_analysis_results -type f -exec cp {} analyses-snapshot-testing/analysis_results/ \; | |
| - name: List All Gathered Files (for debug) | |
| run: ls -lh analyses-snapshot-testing/analysis_results/ | |
| - name: Run Test | |
| id: run_test | |
| working-directory: analyses-snapshot-testing | |
| run: make snapshot-test | |
| - name: Run Audit | |
| id: run_audit | |
| working-directory: analyses-snapshot-testing | |
| run: make snapshot-audit-test | |
| - name: Upload Report | |
| if: '!cancelled()' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-report | |
| path: analyses-snapshot-testing/results/ | |
| - name: Handle Test Failure | |
| id: handle_failure | |
| if: always() && steps.run_test.outcome == 'failure' && (env.OPEN_PR_ON_FAILURE == 'true' || github.event_name == 'schedule') | |
| working-directory: analyses-snapshot-testing | |
| run: make snapshot-test-update | |
| - name: Create Snapshot update Request | |
| id: create_pull_request | |
| if: always() && steps.handle_failure.outcome == 'success' && env.OPEN_PR_ON_FAILURE == 'true' && github.event_name == 'pull_request' | |
| uses: peter-evans/create-pull-request@v7 | |
| with: | |
| commit-message: 'fix(analyses-snapshot-testing): heal analyses snapshots' | |
| title: 'fix(analyses-snapshot-testing): heal ${{ github.event.pull_request.head.ref }} snapshots' | |
| body: 'This PR was requested on the PR https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}' | |
| branch: 'analyses-snapshot-testing/heal-${{ github.event.pull_request.head.ref }}' | |
| base: ${{ github.event.pull_request.head.ref }} | |
| add-paths: | | |
| analyses-snapshot-testing/tests/__snapshots__/analyses_snapshot_test/ | |
| - name: Comment on feature PR | |
| if: always() && steps.create_pull_request.outcome == 'success' && github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const message = 'A PR has been opened to address analyses snapshot changes. Please review the changes here: https://github.com/${{ github.repository }}/pull/${{ steps.create_pull_request.outputs.pull-request-number }}'; | |
| github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: message | |
| }); | |
| - name: Create Snapshot update Request on edge overnight failure | |
| if: always() && steps.handle_failure.outcome == 'success' && github.event_name == 'schedule' | |
| uses: peter-evans/create-pull-request@v7 | |
| with: | |
| commit-message: 'fix(analyses-snapshot-testing): heal edge snapshots' | |
| title: 'fix(analyses-snapshot-testing): heal edge snapshots' | |
| body: 'The edge overnight analyses snapshot test is failing. This PR was opened to alert us to the failure.' | |
| branch: 'analyses-snapshot-testing/heal-edge' | |
| base: edge | |
| add-paths: | | |
| analyses-snapshot-testing/tests/__snapshots__/analyses_snapshot_test/ |