clc-stackage/.github/workflows/ci.yaml at master · haskell/clc-stackage · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
name: ci
on:
  push:
    branches:
      - master

  pull_request:
    branches:
      - master

  schedule:
    # Every day at 00:00 UTC.
    #
    # https://crontab.guru
    - cron: '0 0 * * *'

  workflow_dispatch:
jobs:
  cabal:
    strategy:
      fail-fast: false
      matrix:
        os:
          - "macos-latest"
          - "ubuntu-latest"
          - "windows-latest"
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v6
      - uses: haskell-actions/setup@v2
        with:
          # Should be the current stackage nightly, though this will likely go
          # out-of-date eventually, until a problem is reported.
          ghc-version: "9.12"
      - name: Configure
        run: |
          cabal configure --enable-tests --ghc-options -Werror

      - name: Build executable
        run: cabal build exe:clc-stackage

      - name: Unit Tests
        id: unit
        run: cabal test unit

      - name: Functional Tests
        id: functional
        # We want to run these tests even if the unit tests fail, because
        # it is useful to know if e.g. the unit tests fail due to one
        # stackage endpoint failing, but the functional tests pass due to
        # a backup working.
        if: always()
        shell: bash
        run: NO_CLEANUP=1 cabal test functional

      - name: Print functional failures
        if: ${{ failure() && steps.functional.conclusion == 'failure' }}
        shell: bash
        run: .github/scripts/print_logs.sh
  dry-run:
    strategy:
      fail-fast: false
      matrix:
        os:
        - "ubuntu-latest"
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v6

      - name: Setup nix
        uses: cachix/install-nix-action@v31
        with:
          github_access_token: ${{ secrets.GITHUB_TOKEN }}
          nix_path: nixpkgs=channel:nixos-unstable

      - name: Dry run
        run: nix develop .#ci -Lv -c bash -c '.github/scripts/dry_run.sh'

      # Upload installed binary so that build-batch does not need to re-install
      # it.
      - name: Upload clc-stackage binary
        uses: actions/upload-artifact@v7
        with:
          name: clc-stackage-binary
          path: ./bin/clc-stackage
          retention-days: 1

  # Uses jq's 'range(m; n)' operator to create list of indexes from [m, n)
  # for the build-batch job. Slightly nicer than manually listing all of them.
  build-batch-indexes:
    runs-on: "ubuntu-latest"
    outputs:
      indexes: ${{ steps.set-batch-indexes.outputs.indexes }}
    steps:
      - id: set-batch-indexes
        run: echo "indexes=$(jq -cn '[range(1; 19)]')" >> $GITHUB_OUTPUT

  # Ideally CI would run a job that actually builds all packages, but this
  # can take a very long time, potentially longer than github's free CI limits
  # (last time checked: 5.5 hrs).
  #
  # What we can do instead, is perform the usual batch process of dividing the
  # package set into groups, then have a different job build each group.
  # This does /not/ run up against github's free CI limits.
  #
  # To do this, we have the script batch_index.sh divide the package set into
  # groups, per --batch. Then, using github's matrix strategy, have each
  # job build only a specific group by passing its index as --batch-index.
  #
  # In other words, each job runs
  #
  #     clc-stackage --batch N --batch-index k
  #
  # where k is matrix.index, hence each building a different group.
  # The only other consideration we have, then, is to make sure we have enough
  # indices to cover the whole package set.
  #
  # Currently, we choose --batch to be 200, and the total package set is
  # around 3400, which is filtered to about 3100 packages to build. We thus
  # need at least ceiling(3100 / 200) = 16 indexes to cover this.
  #
  # There is no harm in going overboard e.g. if we have an index that is out of
  # range, that job will simply end with a warning message. We should
  # therefore err on the side of adding too many indices, rather than too few.
  build-batch:
    needs: [build-batch-indexes, dry-run]
    strategy:
      fail-fast: false
      matrix:
        index: ${{ fromJSON(needs.build-batch-indexes.outputs.indexes) }}
    name: Batch group ${{ matrix.index }}
    runs-on: "ubuntu-latest"
    steps:
      - uses: actions/checkout@v6

      - name: Setup nix
        uses: cachix/install-nix-action@v31
        with:
          github_access_token: ${{ secrets.GITHUB_TOKEN }}
          nix_path: nixpkgs=channel:nixos-unstable

      # Download clc-stackage binary from dry-run job.
      - name: Download binary
        uses: actions/download-artifact@v7
        with:
          name: clc-stackage-binary
          path: ./bin

      - name: Build
        run: nix develop .#ci -Lv -c bash -c '.github/scripts/batch_index.sh ${{ matrix.index }}'