Flaky Test Detector #38
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Flaky Test Detector | |
| # Only run this workflow manually from the Actions tab | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| ruby: | |
| description: "Ruby version(s) (space separated)" | |
| required: true | |
| default: "3.4" | |
| crdb: | |
| description: "CockroachDB version(s) (space separated)" | |
| required: true | |
| default: "v25.2" | |
| max: | |
| description: "Maximum number of tests" | |
| required: true | |
| default: "256" # Maximum number of jobs on GitHub Actions | |
| # This allows a subsequently queued workflow run to interrupt previous runs. | |
| concurrency: | |
| group: "${{ github.workflow }} @ ${{ github.ref }}" | |
| cancel-in-progress: true | |
| jobs: | |
| prepare-matrix: | |
| runs-on: ubuntu-latest | |
| name: Prepare Matrix | |
| steps: | |
| - id: generate-matrix | |
| run: | | |
| crdb=$(jq --raw-input --compact-output 'split(" ")' <<<"${{ github.event.inputs.crdb }}") | |
| ruby=$(jq --raw-input --compact-output 'split(" ")' <<<"${{ github.event.inputs.ruby }}") | |
| crdb_len=$(wc -w <<<"${{ github.event.inputs.crdb }}") | |
| ruby_len=$(wc -w <<<"${{ github.event.inputs.ruby }}") | |
| (( seeds_count = ${{github.event.inputs.max}} / ( crdb_len * ruby_len ) )) | |
| seeds=$(shuf --input-range=1-65535 --head-count=$seeds_count | jq --slurp --compact-output) | |
| echo $seeds | |
| echo "crdb=$crdb" >> $GITHUB_OUTPUT | |
| echo "ruby=$ruby" >> $GITHUB_OUTPUT | |
| echo "seeds=$seeds" >> $GITHUB_OUTPUT | |
| outputs: | |
| crdb: ${{ steps.generate-matrix.outputs.crdb }} | |
| ruby: ${{ steps.generate-matrix.outputs.ruby }} | |
| seeds: ${{ steps.generate-matrix.outputs.seeds }} | |
| test: | |
| runs-on: ubuntu-latest | |
| needs: prepare-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| crdb: ${{ fromJSON(needs.prepare-matrix.outputs.crdb) }} | |
| ruby: ${{ fromJSON(needs.prepare-matrix.outputs.ruby) }} | |
| seed: ${{ fromJSON(needs.prepare-matrix.outputs.seeds) }} | |
| name: Test (crdb=${{ matrix.crdb }} ruby=${{ matrix.ruby }} seed=${{ matrix.seed }}) | |
| steps: | |
| - name: Set Up Actions | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/actions/test-runner | |
| id: test | |
| with: | |
| crdb: ${{ matrix.crdb }} | |
| ruby: ${{ matrix.ruby }} | |
| TESTOPTS: --fail-fast | |
| env: | |
| JSON_REPORTER: "report.json" | |
| SEED: ${{ matrix.seed }} | |
| - name: Upload Report | |
| if: ${{ failure() && steps.test.conclusion == 'failure' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: report-${{ matrix.crdb }}-${{ matrix.ruby }}-${{ matrix.seed }} | |
| path: report.json | |
| collect: | |
| if: failure() || cancelled() | |
| needs: test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download Reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: reports | |
| - name: Aggregate Reports | |
| run: | | |
| cat <<EOF >> $GITHUB_STEP_SUMMARY | |
| # Failing Tests | |
| <table> | |
| <thead> | |
| <tr> | |
| <th>Seed</th> | |
| <th>Time</th> | |
| <th>Failure</th> | |
| </tr> | |
| </thead> | |
| <tbody> | |
| EOF | |
| jq --slurp --raw-output ' | |
| sort_by(.total_time)[0:100][] | |
| | {seed, time: .total_time | strftime("%H:%M:%S"), failure: .failed_tests[0].NAME } | |
| | "<tr><td>\(.seed)</td><td>\(.time)</td><td>\(.failure)</td></tr>" | |
| ' reports/*/report.json >> $GITHUB_STEP_SUMMARY | |
| cat <<EOF >> $GITHUB_STEP_SUMMARY | |
| </tbody> | |
| </table> | |
| EOF | |
| # Do not print json if too large. | |
| [[ "$(du -s reports | cut -f1)" -gt 124 ]] && exit 0 | |
| echo >> $GITHUB_STEP_SUMMARY | |
| echo '```json' >> $GITHUB_STEP_SUMMARY | |
| jq --slurp --compact-output '.' reports/*/report.json >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY |