Refactor reproducibility check

Author: biabbas

.github/workflows/repro_check.yml

@@ -1,79 +1,103 @@
-name: Build and Diff Projects
+# Workflow that runs after a merge to master, builds toolchain in 2 different
+# directories to check for reproducible builds. This check helps track which
+# commits cause reproducibility issues. This check should pass before stable
+# release are made.
+
+name: Reproducibility check
 
 on:
   push:
-    branches:
-      - master
-  pull_request:
     branches:
       - master
       - reproducible
 
 jobs:
   build_and_compare:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
+
+    strategy:
+      matrix:
+        build_dir: [buildA, buildA_extended]
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.x'  # Adjust to the version you need
+        uses: actions/checkout@v4
 
-      - name: Build and store buildA binaries
+# Build the two toolchains for comparision
+      - name: Build and store ${{ matrix.build_dir }} binaries
         run: |
-          mkdir ../buildA
-          cp -r "$(pwd)" ../buildA
-          pushd ../buildA
-          SOURCE_DIR=$(dirname $(find . -maxdepth 2  -name x.py))
-          $SOURCE_DIR/configure --set rust.channel=nightly
-          $SOURCE_DIR/x.py build --stage 2 -j$(($(nproc)*2/3))
-          rm -rf $SOURCE_DIR
-          STAGE2_DIR=`find build -name stage2`
-          cp -r "$STAGE2_DIR" .
-          echo "Contents stage 2 dir : `ls stage2`"
-          rm -rf build
-          popd
+          mkdir ../${{ matrix.build_dir }}
+          cp -r "$(pwd)" ../${{ matrix.build_dir }}
+          pushd ../${{ matrix.build_dir }}
 
-      - name: Build and store buildA_extended binaries
-        run: |
-          mkdir ../buildA_extended
-          cp -r "$(pwd)" ../buildA_extended
-          pushd ../buildA_extended
-          SOURCE_DIR=$(dirname $(find . -maxdepth 2  -name x.py))
+          # Find source directory.
+          SOURCE_DIR=$(dirname $(find . -maxdepth 2 -name x.py))
+
+          # FIXME: Setting channel to nightly because only nightly builds succeed on ci.
           $SOURCE_DIR/configure --set rust.channel=nightly
+
+          # Build rust till stage 2
           $SOURCE_DIR/x.py build --stage 2 -j$(($(nproc)*2/3))
+
+          # Remove copy of source directory to save space.
           rm -rf $SOURCE_DIR
-          STAGE2_DIR=`find build -name stage2`
+
+          # Find stage 2 artifacts directory
+          STAGE2_DIR=$(find build -name stage2)
+
+          # Save stage 2 directory contents before we delete build
           cp -r "$STAGE2_DIR" .
-          echo "Contents stage 2 dir : `ls stage2`"
+          echo "Contents stage 2 dir : $(ls stage2)"
+
+          # Remove build directory to save space
           rm -rf build
           popd
 
-      - name: Compare builds and archive artifacts
+# Compare the two builds
+      - name: Compare builds
+        id: compare
         run: |
           # Ensure the directories exist
           if [[ ! -d "../buildA" || ! -d "../buildA_extended" ]]; then
             echo "Error: Build directories not found!"
             exit 1
           fi
 
-          # Perform a diff between the two builds
-          diff -r ../buildA/stage2 ../buildA_extended/stage2 || echo "Differences found!"
-
-          tar -czf buildA.tar.gz ../buildA
-          tar -czf buildA_extended.tar.gz ../buildA_extended
+          # Perform a recursive diff between the stage2 directories of both builds
+          # If there are differences, record the result so we can upload artifacts and then fail later
+          # The binaries should be identical, so the cause of difference should be analysed and fixed
+          # appropriately.
+          if diff -r ../buildA/stage2 ../buildA_extended/stage2; then
+            echo "No differences found."
+            echo "has_diff=false" >> $GITHUB_OUTPUT
+          else
+            echo "Differences found!"
+            echo "has_diff=true" >> $GITHUB_OUTPUT
+          fi
 
+# Upload buildA directory as an artifact (for debugging purposes)
+# This artifact contains stage2 folder from buildA. This artifact would
+# be helpful to debug reproducibility issue when this test fails.
       - name: Upload buildA artifact
+        if: steps.compare.outputs.has_diff == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: buildA
-          path: buildA.tar.gz
+          path: ../buildA
 
+# Upload buildA_extended directory as an artifact (for debugging purposes)
+# This artifact contains stage2 folder from buildA_extended. This artifact would
+# be helpful to debug reproducibility issue when this test fails.
       - name: Upload buildA_extended artifact
+        if: steps.compare.outputs.has_diff == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: buildA_extended
-          path: buildA_extended.tar.gz
+          path: ../buildA_extended
+
+# Fail the job if differences were found between the builds
+      - name: Fail the job if there are differences
+        if: steps.compare.outputs.has_diff == 'true'
+        run: |
+          echo "Differences found between buildA and buildA_extended, Reproducibility check failed"
+          exit 1