Backend Flaky Tests Analysis #153
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backend Flaky Tests Analysis | |
| on: | |
| schedule: | |
| - cron: "0 1 * * *" # Runs every day at 1:00 AM UTC | |
| workflow_dispatch: | |
| inputs: | |
| num_runs: | |
| description: "Number of times to run tests" | |
| required: false | |
| default: "50" | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| env: | |
| MIX_ENV: test | |
| NUM_RUNS: ${{ github.event.inputs.num_runs || 50 }} | |
| jobs: | |
| setup-matrix: | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| ELIXIR_BC: ${{ steps.compute-matrix.outputs.ELIXIR_BC }} | |
| ERLANG_BC: ${{ steps.compute-matrix.outputs.ERLANG_BC }} | |
| ELIXIR_DEV: ${{ steps.compute-matrix.outputs.ELIXIR_DEV }} | |
| ERLANG_DEV: ${{ steps.compute-matrix.outputs.ERLANG_DEV }} | |
| run_range: ${{ steps.compute-matrix.outputs.run_range }} | |
| total-runs: ${{ steps.compute-matrix.outputs.total-runs }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: gather versions | |
| uses: endorama/asdf-parse-tool-versions@v1 | |
| - name: Compute matrix | |
| id: compute-matrix | |
| run: | | |
| echo "ELIXIR_BC=1.15.7-otp-26" >> $GITHUB_OUTPUT | |
| echo "ERLANG_BC=26.2.1" >> $GITHUB_OUTPUT | |
| echo "ELIXIR_DEV=${{ env.ELIXIR_VERSION }}" >> $GITHUB_OUTPUT | |
| echo "ERLANG_DEV=${{ env.ERLANG_VERSION }}" >> $GITHUB_OUTPUT | |
| echo "run_range=[$(seq -s, 1 ${{ env.NUM_RUNS }})]" >> $GITHUB_OUTPUT | |
| echo "total-runs=${{ env.NUM_RUNS }}" >> $GITHUB_OUTPUT | |
| elixir-deps: | |
| name: Elixir test dependencies (Elixir ${{ matrix.elixir }}) | |
| needs: [setup-matrix] | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| matrix: | |
| include: | |
| - elixir: ${{ needs.setup-matrix.outputs.ELIXIR_BC }} | |
| otp: ${{ needs.setup-matrix.outputs.ERLANG_BC }} | |
| - elixir: ${{ needs.setup-matrix.outputs.ELIXIR_DEV }} | |
| otp: ${{ needs.setup-matrix.outputs.ERLANG_DEV }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup | |
| id: setup-elixir | |
| uses: erlef/setup-beam@v1 | |
| with: | |
| otp-version: ${{ matrix.otp }} | |
| elixir-version: ${{ matrix.elixir }} | |
| - name: Retrieve Elixir Cached Dependencies | |
| uses: actions/cache@v5 | |
| id: mix-cache | |
| with: | |
| path: | | |
| deps | |
| _build/${{ env.MIX_ENV }} | |
| priv/plts | |
| key: erlang-${{ steps.setup-elixir.outputs.otp-version }}-elixir-${{ steps.setup-elixir.outputs.elixir-version }}-${{ hashFiles('mix.lock') }}-${{ env.MIX_ENV }} | |
| - name: Install Dependencies | |
| if: steps.mix-cache.outputs.cache-hit != 'true' | |
| run: | | |
| mkdir -p priv/plts | |
| mix local.rebar --force | |
| mix local.hex --force | |
| mix deps.get | |
| mix deps.compile --warnings-as-errors | |
| mix dialyzer --plt | |
| test-backend: | |
| name: Backend Tests (Elixir ${{ matrix.elixir }}, Run ${{ matrix.run }}) | |
| needs: [setup-matrix, elixir-deps] | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| run: ${{ fromJson(needs.setup-matrix.outputs.run_range) }} | |
| elixir: | |
| - ${{ needs.setup-matrix.outputs.ELIXIR_BC }} | |
| - ${{ needs.setup-matrix.outputs.ELIXIR_DEV }} | |
| include: | |
| - elixir: ${{ needs.setup-matrix.outputs.ELIXIR_BC }} | |
| otp: ${{ needs.setup-matrix.outputs.ERLANG_BC }} | |
| - elixir: ${{ needs.setup-matrix.outputs.ELIXIR_DEV }} | |
| otp: ${{ needs.setup-matrix.outputs.ERLANG_DEV }} | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup | |
| id: setup-elixir | |
| uses: erlef/setup-beam@v1 | |
| with: | |
| otp-version: ${{ matrix.otp }} | |
| elixir-version: ${{ matrix.elixir }} | |
| - name: Retrieve Elixir Cached Dependencies | |
| uses: actions/cache@v5 | |
| id: mix-cache | |
| with: | |
| path: | | |
| deps | |
| _build/${{ env.MIX_ENV }} | |
| priv/plts | |
| key: erlang-${{ steps.setup-elixir.outputs.otp-version }}-elixir-${{ steps.setup-elixir.outputs.elixir-version }}-${{ hashFiles('mix.lock') }}-${{ env.MIX_ENV }} | |
| - name: Compile | |
| run: mix compile --warnings-as-errors | |
| - name: "Docker compose dependencies" | |
| uses: isbang/compose-action@v2.5.0 | |
| with: | |
| compose-file: "./docker-compose.yaml" | |
| down-flags: "--volumes" | |
| - name: Run backend tests | |
| env: | |
| WRITE_JUNIT: 1 | |
| run: mix test --warnings-as-errors --max-cases 1 | |
| - name: Copy and sanitize JUnit reports | |
| if: always() | |
| run: | | |
| mkdir -p ./junit-reports | |
| for file in /tmp/*.xml; do | |
| if [ -f "$file" ]; then | |
| # Sanitize filename: GitHub Actions artifact names cannot contain colons | |
| base=$(basename "$file" | tr ':' '-' | tr ' ' '_') | |
| awk '{gsub(/:/, "-"); print}' "$file" > "./junit-reports/$base" | |
| fi | |
| done | |
| - name: Upload backend test junit reports | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: backend-junit-reports-elixir${{ matrix.elixir }}-run${{ matrix.run }} | |
| path: junit-reports/*.xml | |
| retention-days: 1 | |
| flaky-tests-analysis: | |
| needs: [setup-matrix, test-backend] | |
| if: ${{ !cancelled() }} | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| include: | |
| - suite-name: "Backend — Elixir ${{ needs.setup-matrix.outputs.ELIXIR_DEV }}, OTP ${{ needs.setup-matrix.outputs.ERLANG_DEV }}" | |
| junit-artifact-pattern: backend-junit-reports-elixir${{ needs.setup-matrix.outputs.ELIXIR_DEV }}-* | |
| - suite-name: "Backend — Elixir ${{ needs.setup-matrix.outputs.ELIXIR_BC }}, OTP ${{ needs.setup-matrix.outputs.ERLANG_BC }}" | |
| junit-artifact-pattern: backend-junit-reports-elixir${{ needs.setup-matrix.outputs.ELIXIR_BC }}-* | |
| uses: ./.github/workflows/flaky-tests-analysis.yaml | |
| with: | |
| suite-name: ${{ matrix.suite-name }} | |
| junit-artifact-pattern: ${{ matrix.junit-artifact-pattern }} | |
| total-runs: ${{ needs.setup-matrix.outputs.total-runs }} |