web/.github/workflows/backend-flaky-tests-detection.yaml at main · trento-project/web · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
name: Backend Flaky Tests Analysis

on:
  schedule:
    - cron: "0 1 * * *" # Runs every day at 1:00 AM UTC
  workflow_dispatch:
    inputs:
      num_runs:
        description: "Number of times to run tests"
        required: false
        default: "50"

concurrency:
  group: ${{ github.workflow }}

env:
  MIX_ENV: test
  NUM_RUNS: ${{ github.event.inputs.num_runs || 50 }}

jobs:
  setup-matrix:
    runs-on: ubuntu-24.04
    outputs:
      ELIXIR_BC: ${{ steps.compute-matrix.outputs.ELIXIR_BC }}
      ERLANG_BC: ${{ steps.compute-matrix.outputs.ERLANG_BC }}
      ELIXIR_DEV: ${{ steps.compute-matrix.outputs.ELIXIR_DEV }}
      ERLANG_DEV: ${{ steps.compute-matrix.outputs.ERLANG_DEV }}
      run_range: ${{ steps.compute-matrix.outputs.run_range }}
      total-runs: ${{ steps.compute-matrix.outputs.total-runs }}
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
      - name: gather versions
        uses: endorama/asdf-parse-tool-versions@23a7edd74b0f97626c857c48e32ebbe4ba596a23 # v1.5.1
      - name: Compute matrix
        id: compute-matrix
        run: |
          echo "ELIXIR_BC=1.15.7-otp-26" >> $GITHUB_OUTPUT
          echo "ERLANG_BC=26.2.1" >> $GITHUB_OUTPUT
          echo "ELIXIR_DEV=${{ env.ELIXIR_VERSION }}" >> $GITHUB_OUTPUT
          echo "ERLANG_DEV=${{ env.ERLANG_VERSION }}" >> $GITHUB_OUTPUT
          echo "run_range=[$(seq -s, 1 ${{ env.NUM_RUNS }})]" >> $GITHUB_OUTPUT
          echo "total-runs=${{ env.NUM_RUNS }}" >> $GITHUB_OUTPUT

  elixir-deps:
    name: Elixir test dependencies (Elixir ${{ matrix.elixir }})
    needs: [setup-matrix]
    runs-on: ubuntu-24.04
    strategy:
      matrix:
        include:
          - elixir: ${{ needs.setup-matrix.outputs.ELIXIR_BC }}
            otp: ${{ needs.setup-matrix.outputs.ERLANG_BC }}
          - elixir: ${{ needs.setup-matrix.outputs.ELIXIR_DEV }}
            otp: ${{ needs.setup-matrix.outputs.ERLANG_DEV }}
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          fetch-depth: 0
      - name: Setup
        id: setup-elixir
        uses: erlef/setup-beam@fc68ffb90438ef2936bbb3251622353b3dcb2f93 # v1
        with:
          otp-version: ${{ matrix.otp }}
          elixir-version: ${{ matrix.elixir }}
      - name: Retrieve Elixir Cached Dependencies
        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
        id: mix-cache
        with:
          path: |
            deps
            _build/${{ env.MIX_ENV }}
            priv/plts
          key: erlang-${{ steps.setup-elixir.outputs.otp-version }}-elixir-${{ steps.setup-elixir.outputs.elixir-version }}-${{ hashFiles('mix.lock') }}-${{ env.MIX_ENV }}
      - name: Install Dependencies
        if: steps.mix-cache.outputs.cache-hit != 'true'
        run: |
          mkdir -p priv/plts
          mix local.rebar --force
          mix local.hex --force
          mix deps.get
          mix deps.compile --warnings-as-errors
          mix dialyzer --plt

  test-backend:
    name: Backend Tests (Elixir ${{ matrix.elixir }}, Run ${{ matrix.run }})
    needs: [setup-matrix, elixir-deps]
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        run: ${{ fromJson(needs.setup-matrix.outputs.run_range) }}
        elixir:
          - ${{ needs.setup-matrix.outputs.ELIXIR_BC }}
          - ${{ needs.setup-matrix.outputs.ELIXIR_DEV }}
        include:
          - elixir: ${{ needs.setup-matrix.outputs.ELIXIR_BC }}
            otp: ${{ needs.setup-matrix.outputs.ERLANG_BC }}
          - elixir: ${{ needs.setup-matrix.outputs.ELIXIR_DEV }}
            otp: ${{ needs.setup-matrix.outputs.ERLANG_DEV }}
    timeout-minutes: 10
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          fetch-depth: 0
      - name: Setup
        id: setup-elixir
        uses: erlef/setup-beam@fc68ffb90438ef2936bbb3251622353b3dcb2f93 # v1
        with:
          otp-version: ${{ matrix.otp }}
          elixir-version: ${{ matrix.elixir }}
      - name: Retrieve Elixir Cached Dependencies
        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
        id: mix-cache
        with:
          path: |
            deps
            _build/${{ env.MIX_ENV }}
            priv/plts
          key: erlang-${{ steps.setup-elixir.outputs.otp-version }}-elixir-${{ steps.setup-elixir.outputs.elixir-version }}-${{ hashFiles('mix.lock') }}-${{ env.MIX_ENV }}
      - name: Compile
        run: mix compile --warnings-as-errors
      - name: "Docker compose dependencies"
        uses: isbang/compose-action@4894d2492015c1774ee5a13a95b1072093087ec3 # v2.5.0
        with:
          compose-file: "./docker-compose.yaml"
          down-flags: "--volumes"
      - name: Run backend tests
        env:
          WRITE_JUNIT: 1
        run: mix test --warnings-as-errors --max-cases 1
      - name: Copy and sanitize JUnit reports
        if: always()
        run: |
          mkdir -p ./junit-reports
          for file in /tmp/*.xml; do
            if [ -f "$file" ]; then
              # Sanitize filename: GitHub Actions artifact names cannot contain colons
              base=$(basename "$file" | tr ':' '-' | tr ' ' '_')
              awk '{gsub(/:/, "-"); print}' "$file" > "./junit-reports/$base"
            fi
          done
      - name: Upload backend test junit reports
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: backend-junit-reports-elixir${{ matrix.elixir }}-run${{ matrix.run }}
          path: junit-reports/*.xml
          retention-days: 1

  flaky-tests-analysis:
    needs: [setup-matrix, test-backend]
    if: ${{ !cancelled() }}
    strategy:
      max-parallel: 1
      matrix:
        include:
          - suite-name: "Backend — Elixir ${{ needs.setup-matrix.outputs.ELIXIR_DEV }}, OTP ${{ needs.setup-matrix.outputs.ERLANG_DEV }}"
            junit-artifact-pattern: backend-junit-reports-elixir${{ needs.setup-matrix.outputs.ELIXIR_DEV }}-*
          - suite-name: "Backend — Elixir ${{ needs.setup-matrix.outputs.ELIXIR_BC }}, OTP ${{ needs.setup-matrix.outputs.ERLANG_BC }}"
            junit-artifact-pattern: backend-junit-reports-elixir${{ needs.setup-matrix.outputs.ELIXIR_BC }}-*
    uses: ./.github/workflows/flaky-tests-analysis.yaml
    with:
      suite-name: ${{ matrix.suite-name }}
      junit-artifact-pattern: ${{ matrix.junit-artifact-pattern }}
      total-runs: ${{ needs.setup-matrix.outputs.total-runs }}