Merge pull request #1936 from lightninglabs/chore/ci-itest-split

ci: parallelize integration tests using matrix strategy

Author: Roasbeef

.github/workflows/main.yaml

@@ -274,9 +274,9 @@ jobs:
           format: 'golang'
           parallel: true
 
-  # Integration tests on SQLite backend.
-  integration-test:
-    name: run itests
+  # Build integration test binaries once, shared by all itest jobs.
+  build-itest:
+    name: build itest binaries
     runs-on: ubuntu-latest
     steps:
       - name: git checkout
@@ -290,69 +290,120 @@ jobs:
         with:
           go-version: '${{ env.GO_VERSION }}'
 
+      - name: Build itest binaries
+        run: make build-itest build-itest-binary
+
+      - name: Upload itest binaries
+        uses: actions/upload-artifact@v4
+        with:
+          name: itest-binaries
+          path: |
+            itest/itest.test
+            itest/btcd-itest
+            itest/lnd-itest
+            itest/chantools/chantools
+
+  # Integration tests on SQLite backend.
+  integration-test:
+    name: run itests
+    runs-on: ubuntu-latest
+    needs: build-itest
+    strategy:
+      fail-fast: false
+      matrix:
+        tranche: [0, 1, 2, 3]
+    steps:
+      - name: git checkout
+        uses: actions/checkout@v5
+
+      - name: Clean up runner space
+        uses: ./.github/actions/cleanup-space
+
+      - name: Download itest binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: itest-binaries
+          path: itest
+
+      - name: Initialize environment
+        run: |
+          chmod +x itest/itest.test itest/btcd-itest itest/lnd-itest itest/chantools/chantools
+          mkdir -p ~/.aperture
+
       - name: run itest
-        run: make itest-parallel
+        run: scripts/itest_part.sh ${{ matrix.tranche }} 4 0 --verbose
 
       - name: Zip log files on failure
         if: ${{ failure() }}
-        run: 7z a logs-itest.zip itest/**/*.log
+        run: 7z a logs-itest-${{ matrix.tranche }}.zip itest/**/*.log
 
       - name: Upload log files on failure
         uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
-          name: logs-itest
-          path: logs-itest.zip
+          name: logs-itest-${{ matrix.tranche }}
+          path: logs-itest-${{ matrix.tranche }}.zip
           retention-days: 5
 
       - name: Send coverage
         uses: coverallsapp/github-action@v2
         if: ${{ success() }}
         continue-on-error: true
         with:
-          file: itest/coverage.txt
-          flag-name: 'itest'
+          file: itest/regtest/cover/coverage-tranche${{ matrix.tranche }}.txt
+          flag-name: 'itest-${{ matrix.tranche }}'
           format: 'golang'
           parallel: true
 
   # Integration tests on Postgres backend.
   integration-test-postgres:
     name: run itests postgres
     runs-on: ubuntu-latest
+    needs: build-itest
+    strategy:
+      fail-fast: false
+      matrix:
+        tranche: [0, 1, 2, 3]
     steps:
       - name: git checkout
         uses: actions/checkout@v5
 
       - name: Clean up runner space
         uses: ./.github/actions/cleanup-space
 
-      - name: Setup go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
+      - name: Download itest binaries
+        uses: actions/download-artifact@v4
         with:
-          go-version: '${{ env.GO_VERSION }}'
+          name: itest-binaries
+          path: itest
+
+      - name: Initialize environment
+        run: |
+          chmod +x itest/itest.test itest/btcd-itest itest/lnd-itest itest/chantools/chantools
+          mkdir -p ~/.aperture
 
       - name: run itest
-        run: make itest-parallel dbbackend=postgres tranches=4
+        run: scripts/itest_part.sh ${{ matrix.tranche }} 4 0 -dbbackend=postgres --verbose
 
       - name: Zip log files on failure
         if: ${{ failure() }}
-        run: 7z a logs-itest-postgres.zip itest/**/*.log
+        run: 7z a logs-itest-postgres-${{ matrix.tranche }}.zip itest/**/*.log
 
       - name: Upload log files on failure
         uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
-          name: logs-itest-postgres
-          path: logs-itest-postgres.zip
+          name: logs-itest-postgres-${{ matrix.tranche }}
+          path: logs-itest-postgres-${{ matrix.tranche }}.zip
           retention-days: 5
 
       - name: Send coverage
         uses: coverallsapp/github-action@v2
         if: ${{ success() }}
         continue-on-error: true
         with:
-          file: itest/coverage.txt
-          flag-name: 'itest'
+          file: itest/regtest/cover/coverage-tranche${{ matrix.tranche }}.txt
+          flag-name: 'itest-postgres-${{ matrix.tranche }}'
           format: 'golang'
           parallel: true
 

scripts/itest_part.sh

@@ -11,6 +11,17 @@ SHUFFLE_SEED_PARAM=$3
 # the $@ special variable.
 shift 3
 
+# Check for --verbose flag in the remaining arguments.
+VERBOSE=0
+TEST_FLAGS=()
+for arg in "$@"; do
+  if [ "$arg" == "--verbose" ]; then
+    VERBOSE=1
+  else
+    TEST_FLAGS+=("$arg")
+  fi
+done
+
 # Windows insists on having the .exe suffix for an executable, we need to add
 # that here if necessary.
 EXEC="$WORKDIR"/itest.test
@@ -20,15 +31,21 @@ export GOCOVERDIR="$WORKDIR/regtest/cover"
 COVERFILE="$GOCOVERDIR/coverage-tranche$TRANCHE.txt"
 mkdir -p "$GOCOVERDIR"
 mkdir -p "$LOG_DIR"
-echo $EXEC "$@" -test.coverprofile=$COVERFILE -test.gocoverdir=$GOCOVERDIR -logoutput -logdir=$LOG_DIR -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE -shuffleseed=$SHUFFLE_SEED_PARAM
+echo $EXEC "${TEST_FLAGS[@]}" -test.coverprofile=$COVERFILE -test.gocoverdir=$GOCOVERDIR -logoutput -logdir=$LOG_DIR -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE -shuffleseed=$SHUFFLE_SEED_PARAM
 
 # Exit code 255 causes the parallel jobs to abort, so if one part fails the
 # other is aborted too.
 cd "$WORKDIR" || exit 255
-$EXEC "$@" -test.coverprofile=$COVERFILE -test.gocoverdir=$GOCOVERDIR -logoutput -logdir=$LOG_DIR -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE -shuffleseed=$SHUFFLE_SEED_PARAM > $LOG_DIR/output.log 2>&1
 
-# Capture the exit code of the test run.
-exit_code=$?
+if [ $VERBOSE -eq 1 ]; then
+    $EXEC "${TEST_FLAGS[@]}" -test.coverprofile=$COVERFILE -test.gocoverdir=$GOCOVERDIR -logoutput -logdir=$LOG_DIR -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE -shuffleseed=$SHUFFLE_SEED_PARAM 2>&1 | tee "$LOG_DIR/output.log"
+    # Capture the exit code of the test run (first command in pipe).
+    exit_code=${PIPESTATUS[0]}
+else
+    $EXEC "${TEST_FLAGS[@]}" -test.coverprofile=$COVERFILE -test.gocoverdir=$GOCOVERDIR -logoutput -logdir=$LOG_DIR -btcdexec=$BTCD_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE -shuffleseed=$SHUFFLE_SEED_PARAM > "$LOG_DIR/output.log" 2>&1
+    # Capture the exit code of the test run.
+    exit_code=$?
+fi
 if [ $exit_code -ne 0 ]; then
     echo "Tranche $TRANCHE failed with exit code $exit_code"
     tail -n 100 "$LOG_DIR/output.log"