Add real build to CI

tbidne · tbidne · commit 519a184459cb · 2026-04-09T11:46:17.000+10:00
Adds the build-batch job to CI which actually builds the entire
snapshot. It does this by dividing the snapshot into batch groups
per --batch, then having a different job build each group via
github action's matrix strategy and new clc-stackage --batch-index
param.

This allows us to build the entire snapshot such that overall CI time
is reasonable i.e. ~ 1 hour.
diff --git a/.github/scripts/batch_index.sh b/.github/scripts/batch_index.sh
@@ -0,0 +1,62 @@
+set -e
+
+# Similar to dry_run.sh, except actually builds exactly one batch group.
+# This way CI can spread the full build across multiple jobs, keeping the
+# total time reasonable.
+batch_index=$1
+
+# -f not -x since downloaded exe may not have executable permissions.
+if [[ -f ./bin/clc-stackage ]]; then
+  echo "*** ./bin/clc-stackage exists, not re-installing ***"
+
+  # May need to add permissions, if this exe was downloaded
+  chmod a+x ./bin/clc-stackage
+else
+  echo "*** Updating cabal ***"
+  cabal update
+
+  echo "*** Installing clc-stackage ***"
+  cabal install exe:clc-stackage --installdir=./bin --overwrite-policy=always
+fi
+
+if [[ -d output ]]; then
+  rm -r output
+fi
+
+echo "*** Building with --batch-index $batch_index ***"
+
+set +e
+
+./bin/clc-stackage \
+  --batch 200 \
+  --batch-index $batch_index \
+  --cabal-options="--semaphore" \
+  --cleanup off
+
+ec=$?
+
+if [[ $ec != 0 ]]; then
+  echo "*** clc-stackage failed ***"
+else
+  echo "*** clc-stackage succeeded ***"
+fi
+
+# Print out the logs + the packages we built, in case it is useful e.g.
+# what did CI actually do.
+if [[ -f generated/generated.cabal ]]; then
+  echo "*** Printing generated cabal file ***"
+  cat generated/generated.cabal
+else
+  echo "*** No generated/generated.cabal ***"
+fi
+
+if [[ -f generated/cabal.project.local ]]; then
+  echo "*** Printing generated cabal.project.local file ***"
+  cat generated/cabal.project.local
+else
+  echo "*** No generated/cabal.project.local ***"
+fi
+
+.github/scripts/print_logs.sh
+
+exit $ec
diff --git a/.github/scripts/dry_run.sh b/.github/scripts/dry_run.sh
@@ -1,15 +1,19 @@
 set -e
 
-echo "*** Updating cabal ***"
-
-cabal update
+if [[ -f ./bin/clc-stackage ]]; then
+  echo "*** ./bin/clc-stackage exists, not re-installing ***"
+  chmod a+x ./bin/clc-stackage
+else
+  echo "*** Updating cabal ***"
+  cabal update
 
-echo "*** Installing clc-stackage ***"
+  # --overwrite-policy=always and deleting output/ are unnecessary for CI since
+  # this script will only be run one time, but it's helpful when we are
+  # testing the script locally.
 
-# --overwrite-policy=always and deleting output/ are unnecessary for CI since
-# this script will only be run one time, but it's helpful when we are
-# testing the script locally.
-cabal install exe:clc-stackage --installdir=./bin --overwrite-policy=always
+  echo "*** Installing clc-stackage ***"
+  cabal install exe:clc-stackage --installdir=./bin --overwrite-policy=always
+fi
 
 if [[ -d output ]]; then
   rm -r output
@@ -18,7 +22,7 @@ fi
 echo "*** Building all with --dry-run ***"
 
 set +e
-./bin/clc-stackage --batch 100 --cabal-options="--dry-run"
+./bin/clc-stackage --batch 200 --cabal-options="--dry-run"
 
 ec=$?
 
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -26,7 +26,7 @@ jobs:
           - "windows-latest"
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - uses: haskell-actions/setup@v2
         with:
           # Should be the current stackage nightly, though this will likely go
@@ -57,21 +57,94 @@ jobs:
         if: ${{ failure() && steps.functional.conclusion == 'failure' }}
         shell: bash
         run: .github/scripts/print_logs.sh
-  nix:
+  dry-run:
     strategy:
       fail-fast: false
       matrix:
         os:
         - "ubuntu-latest"
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Setup nix
-        uses: cachix/install-nix-action@v30
+        uses: cachix/install-nix-action@v31
         with:
           github_access_token: ${{ secrets.GITHUB_TOKEN }}
           nix_path: nixpkgs=channel:nixos-unstable
 
       - name: Dry run
         run: nix develop .#ci -Lv -c bash -c '.github/scripts/dry_run.sh'
+
+      # Upload installed binary so that build-batch does not need to re-install
+      # it.
+      - name: Upload clc-stackage binary
+        uses: actions/upload-artifact@v7
+        with:
+          name: clc-stackage-binary
+          path: ./bin/clc-stackage
+          retention-days: 1
+
+  # Uses jq's 'range(m; n)' operator to create list of indexes from [m, n)
+  # for the build-batch job. Slightly nicer than manually listing all of them.
+  build-batch-indexes:
+    runs-on: "ubuntu-latest"
+    outputs:
+      indexes: ${{ steps.set-batch-indexes.outputs.indexes }}
+    steps:
+      - id: set-batch-indexes
+        run: echo "indexes=$(jq -cn '[range(1; 19)]')" >> $GITHUB_OUTPUT
+
+  # Ideally CI would run a job that actually builds all packages, but this
+  # can take a very long time, potentially longer than github's free CI limits
+  # (last time checked: 5.5 hrs).
+  #
+  # What we can do instead, is perform the usual batch process of dividing the
+  # package set into groups, then have a different job build each group.
+  # This does /not/ run up against github's free CI limits.
+  #
+  # To do this, we have the script batch_index.sh divide the package set into
+  # groups, per --batch. Then, using github's matrix strategy, have each
+  # job build only a specific group by passing its index as --batch-index.
+  #
+  # In other words, each job runs
+  #
+  #     clc-stackage --batch N --batch-index k
+  #
+  # where k is matrix.index, hence each building a different group.
+  # The only other consideration we have, then, is to make sure we have enough
+  # indices to cover the whole package set.
+  #
+  # Currently, we choose --batch to be 200, and the total package set is
+  # around 3400, which is filtered to about 3100 packages to build. We thus
+  # need at least ceiling(3100 / 200) = 16 indexes to cover this.
+  #
+  # There is no harm in going overboard e.g. if we have an index that is out of
+  # range, that job will simply end with a warning message. We should
+  # therefore err on the side of adding too many indices, rather than too few.
+  build-batch:
+    needs: [build-batch-indexes, dry-run]
+    strategy:
+      fail-fast: false
+      matrix:
+        index: ${{ fromJSON(needs.build-batch-indexes.outputs.indexes) }}
+    name: Batch group ${{ matrix.index }}
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup nix
+        uses: cachix/install-nix-action@v31
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      # Download clc-stackage binary from dry-run job.
+      - name: Download binary
+        uses: actions/download-artifact@v7
+        with:
+          name: clc-stackage-binary
+          path: ./bin
+
+      - name: Build
+        run: nix develop .#ci -Lv -c bash -c '.github/scripts/batch_index.sh ${{ matrix.index }}'
diff --git a/README.md b/README.md
@@ -119,7 +119,7 @@ In general, user mitigations for solver / build problems include:
     compiler = pkgs.haskell.packages.ghc<vers>;
     ```
 
-    can be a useful guide as to which GHC was last tested, as CI uses this ghc to build everything with `--dry-run`, which should report solver errors (e.g. bounds) at the very least.
+    can be a useful guide as to which GHC was last tested, as CI uses this ghc to build everything.
 
 - If you encounter an error that you think indicates a problem with the configuration here (e.g. new package needs to be excluded, new constraint added), please open an issue. While that is being resolved, the mitigations from the [previous section](#troubleshooting) may be useful.
 
diff --git a/src/CLC/Stackage/Builder/Env.hs b/src/CLC/Stackage/Builder/Env.hs
@@ -34,6 +34,10 @@ data BuildEnv = MkBuildEnv
   { -- | If we have @Just n@, 'packagesToBuild' will be split into groups of at most
     -- size @n@. If @Nothing@, the entire set will be built in one go.
     batch :: Maybe Int,
+    -- | 1-based index for building the Nth package group only, according to
+    -- --batch. Intended for CI use, where building all groups takes too much
+    -- time.
+    batchIndex :: Maybe Int,
     -- | Build arguments for cabal.
     buildArgs :: [String],
     -- | Optional path to cabal executable.
diff --git a/src/CLC/Stackage/Runner.hs b/src/CLC/Stackage/Runner.hs
@@ -8,7 +8,7 @@ where
 
 import CLC.Stackage.Builder qualified as Builder
 import CLC.Stackage.Builder.Env
-  ( BuildEnv (hLogger, progress),
+  ( BuildEnv (batchIndex, hLogger, progress),
     Progress (failuresRef),
   )
 import CLC.Stackage.Builder.Writer qualified as Writer
@@ -20,6 +20,10 @@ import Control.Exception (bracket, throwIO)
 import Control.Monad (when)
 import Data.Foldable (for_)
 import Data.IORef (readIORef)
+import Data.List.NonEmpty (NonEmpty)
+import Data.List.NonEmpty qualified as NE
+import Data.Text (Text)
+import Data.Text qualified as T
 import System.Exit (ExitCode (ExitFailure))
 import System.IO qualified as IO
 
@@ -52,7 +56,24 @@ runModifyPackages hLogger modifyPackages = withHiddenInput $ do
 
     Logging.putTimeInfoStr buildEnv.hLogger "Starting build(s)"
 
-    for_ pkgGroupsIdx $ \(pkgGroup, idx) -> Builder.buildProject buildEnv idx pkgGroup
+    case buildEnv.batchIndex of
+      -- No batch index: normal, build all groups sequentially.
+      Nothing -> for_ pkgGroupsIdx $ \(pkgGroup, idx) ->
+        Builder.buildProject buildEnv idx pkgGroup
+      Just batchIndex ->
+        -- Some batch index: if it is in range, build that group.
+        case index batchIndex pkgGroupsIdx of
+          Just (pkgGroup, idx) -> Builder.buildProject buildEnv idx pkgGroup
+          Nothing -> do
+            let msg =
+                  mconcat
+                    [ "Nothing to build. Index '",
+                      showt batchIndex,
+                      "' is out of range for ",
+                      showt $ NE.length pkgGroupsIdx,
+                      " group(s)."
+                    ]
+            Logging.putTimeWarnStr buildEnv.hLogger msg
 
     numErrors <- length <$> readIORef buildEnv.progress.failuresRef
     when (numErrors > 0) $ throwIO $ ExitFailure 1
@@ -75,3 +96,16 @@ withHiddenInput m = bracket hideInput unhideInput (const m)
     unhideInput (buffMode, echoMode) = do
       IO.hSetBuffering IO.stdin buffMode
       IO.hSetEcho IO.stdin echoMode
+
+index :: Int -> NonEmpty a -> Maybe a
+index idx = go idx' . NE.toList
+  where
+    go _ [] = Nothing
+    go 0 (x : _) = Just x
+    go !n (_ : xs) = go (n - 1) xs
+
+    -- Subtract one since the index is one-based, not zero.
+    idx' = idx - 1
+
+showt :: (Show a) => a -> Text
+showt = T.pack . show
diff --git a/src/CLC/Stackage/Runner/Args.hs b/src/CLC/Stackage/Runner/Args.hs
@@ -58,6 +58,10 @@ data Args = MkArgs
   { -- | If given, batches packages together so we build more than one.
     -- Defaults to batching everything together in the same group.
     batch :: Maybe Int,
+    -- | 1-based index for building the Nth package group only, according to
+    -- --batch. Intended for CI use, where building all groups takes too much
+    -- time.
+    batchIndex :: Maybe Int,
     -- | Global options to pass to cabal e.g. --store-dir.
     cabalGlobalOpts :: [String],
     -- | Options to pass to cabal e.g. --semaphore.
@@ -191,12 +195,13 @@ parseCliArgs =
       ~(cabalGlobalOpts, cabalOpts, cabalPath, cabalUpdate) <- parseCabalGroup
       ~(cache, retryFailures) <- parseCacheGroup
       ~(groupFailFast, packageFailFast) <- parseFailuresGroup
-      ~(batch, printPackageSet, snapshotPath) <- parseMiscGroup
+      ~(batch, batchIndex, printPackageSet, snapshotPath) <- parseMiscGroup
       ~(cleanup, colorLogs, writeLogs) <- parseOutputGroup
 
       pure $
         MkArgs
           { batch,
+            batchIndex,
             cabalGlobalOpts,
             cabalOpts,
             cabalPath,
@@ -236,8 +241,9 @@ parseCliArgs =
 
     parseMiscGroup =
       OA.parserOptionGroup "Misc options:" $
-        (,,)
+        (,,,)
           <$> parseBatch
+          <*> parseBatchIndex
           <*> parsePrintPackageSet
           <*> parseSnapshotPath
 
@@ -267,6 +273,22 @@ parseBatch =
           ]
       )
 
+-- Determines which --batch group to build. Normally we want to build all
+-- groups, so this arg is intended only for CI, where a single CI job cannot
+-- build everything, or it will time out. Hence this is marked 'internal'
+-- to hide it from the --help page, as its presence would only confuse.
+parseBatchIndex :: Parser (Maybe Int)
+parseBatchIndex =
+  OA.optional $
+    OA.option
+      OA.auto
+      ( mconcat
+          [ OA.long "batch-index",
+            OA.internal,
+            OA.metavar "NAT"
+          ]
+      )
+
 parseCabalGlobalOpts :: Parser [String]
 parseCabalGlobalOpts =
   OA.option
diff --git a/src/CLC/Stackage/Runner/Env.hs b/src/CLC/Stackage/Runner/Env.hs
@@ -181,6 +181,7 @@ setup hLoggerRaw modifyPackages = do
       buildEnv =
         MkBuildEnv
           { batch = cliArgs.batch,
+            batchIndex = cliArgs.batchIndex,
             buildArgs,
             cabalPath,
             groupFailFast = cliArgs.groupFailFast,
@@ -217,8 +218,10 @@ teardown :: RunnerEnv -> IO ()
 teardown env = do
   endTime <- env.buildEnv.hLogger.getLocalTime
   when env.cleanup $ do
-    Dir.removeFile Paths.generatedCabalPath
-    Dir.removeFile Paths.generatedCabalProjectLocalPath
+    -- removePathForcibly as these might not exist e.g. if the we did not
+    -- build anything.
+    Dir.removePathForcibly Paths.generatedCabalPath
+    Dir.removePathForcibly Paths.generatedCabalProjectLocalPath
 
   results <- getResults env.buildEnv
   let report =
diff --git a/test/unit/Unit/Prelude.hs b/test/unit/Unit/Prelude.hs
@@ -9,6 +9,7 @@ import CLC.Stackage.Builder.Env
   ( BuildEnv
       ( MkBuildEnv,
         batch,
+        batchIndex,
         buildArgs,
         cabalPath,
         groupFailFast,
@@ -66,6 +67,7 @@ mkBuildEnv = do
   pure $
     MkBuildEnv
       { batch = Nothing,
+        batchIndex = Nothing,
         buildArgs = [],
         cabalPath = "cabal",
         groupFailFast = False,
