Merge remote-tracking branch 'origin/master' into paulliu/ckbtc-utxo-cache

Author: ninegua

.github/CODEOWNERS

@@ -11,10 +11,7 @@
 /licenses/                @dfinity/idx
 /bin/ict                  @dfinity/idx
 /bin/                     @dfinity/idx
-/bin/afl_test.sh          @dfinity/product-security
-/bin/afl_wrapper.sh       @dfinity/product-security
-/bin/build-all-fuzzers.sh @dfinity/product-security
-/bin/fuzzing_coverage.sh  @dfinity/product-security
+/bin/fuzzing/             @dfinity/product-security
 
 # [Bazel]
 .bazelrc                           @dfinity/idx

.github/actions/bazel-test-all/action.yaml

@@ -7,6 +7,9 @@ inputs:
   release-build:
     required: false
     default: true
+  execlogs-artifact-name:
+    required: false
+    description: "When provided, the execlogs will be uploaded as an artifact with the specified name."
   BAZEL_COMMAND:
     required: true
     default: 'test'
@@ -62,7 +65,9 @@ runs:
           BRANCH_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         with:
           GPG_PASSPHRASE: ${{ inputs.GPG_PASSPHRASE }}
+          execlogs-artifact-name: ${{ inputs.execlogs-artifact-name }}
           run: |
+            set -euo pipefail
 
             diff_only='${{ inputs.diff-only }}'
             release_build='${{ inputs.release-build }}'

.github/actions/bazel-upload-checksums/action.yaml

@@ -6,6 +6,9 @@ inputs:
     required: true
     description: |
       The commands to run. Will be evaluated with bash.
+  execlogs-artifact-name:
+    required: false
+    description: "When provided, the execlogs will be uploaded as an artifact with the specified name."
   GPG_PASSPHRASE:
     required: true
     description: "GPG key to encrypt build events. Upload can be disabled by explicitly setting the input to an empty string."
@@ -70,6 +73,55 @@ runs:
           ${{ steps.metrics-tmpdir.outputs.dir }}/bazel-bep-*.pb.gpg
           ${{ steps.metrics-tmpdir.outputs.dir }}/profile-*.json
 
+
+      # Remove unnecessary entries from the JSON execlogs and create a CSV out of them
+    - name: Clean up execlogs
+      shell: bash
+      if: (success() || failure()) && inputs.execlogs-artifact-name != ''
+      run: |
+        execlogs_json_in='${{ steps.metrics-tmpdir.outputs.dir }}/execlogs.json'
+        find '${{ steps.metrics-tmpdir.outputs.dir }}' -name 'execlog-*.json' \
+          -exec cat {} \; > "$execlogs_json_in"
+
+        execlogs_csv_out='${{ steps.metrics-tmpdir.outputs.dir }}/execlogs.csv'
+        if ! [ -s "$execlogs_json_in" ]; then
+          echo "no execlogs found"
+          exit 0
+        fi
+
+        # this filters out some entries from the input JSON execlogs. Without this,
+        # jq takes 3+ minutes to parse the execlogs.
+        # Effectively removes a full line of "commandArgs", and takes care of skipping
+        # multiple lines for some multiline array values we don't care about
+        prog='
+          BEGIN { arr = 0; }
+          arr == 1 && /  }],?/ { arr = 0; next }
+          arr == 1 { next }
+          /  "commandArgs":.*/ { next }
+          /  "environmentVariables": \[\{/ { arr = 1; next; }
+          /  "inputs": \[\{/ { arr = 1; next; }
+          { print }
+          '
+
+        # Turns the JSON execlogs into a CSV with the following columns:
+        #   <label>,<output path>,<hash>
+        # only the target labels matching "whitelist_pat" are considered
+        whitelist_pat='^//'
+        time (cat "$execlogs_json_in" \
+          | awk "$prog" \
+          | jq -cMr --arg whitelist_pat "$whitelist_pat" \
+              'select(.targetLabel | test($whitelist_pat)) | .targetLabel as $targetLabel | .actualOutputs | map(. | $targetLabel+","+.path+","+.digest.hash) | .[]' \
+          >"$execlogs_csv_out")
+
+    - name: Upload execution log
+      if: (success() || failure()) && inputs.execlogs-artifact-name != ''
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.execlogs-artifact-name }}
+        if-no-files-found: ignore
+        path: |
+          ${{ steps.metrics-tmpdir.outputs.dir }}/execlogs.csv
+
     - name: Cleanup
       shell: bash
       if: always()

.github/actions/bazel/action.yaml

@@ -58,6 +58,10 @@ fi
 if [[ $bazel_command == "build" ]] || [[ $bazel_command == "test" ]]; then
     command_timestamp=$(date +%s)
     bazel_args+=(
+
+        # write execlogs, mostly for build reproducibility checks
+        --execution_log_json_file="$BAZEL_ACTION_METRICS_OUT/execlog-$command_timestamp.json"
+
         --verbose_failures=true
 
         # enables BES upload (see config)

.github/actions/bazel/bin/bazel

@@ -193,18 +193,14 @@ jobs:
         id: bazel-test-all
         uses: ./.github/actions/bazel-test-all/
         with:
+          execlogs-artifact-name: execlogs-bazel-test-all
           diff-only: ${{ needs.config.outputs.diff_only }}
           release-build: ${{ needs.config.outputs.release_build }}
           BAZEL_COMMAND: test --config=ci ${{ steps.bazel-extra-args.outputs.BAZEL_EXTRA_ARGS }}
           BAZEL_TARGETS: //...
           CLOUD_CREDENTIALS_CONTENT: ${{ secrets.CLOUD_CREDENTIALS_CONTENT }}
           GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
 
-      - name: Upload SHA256SUMS (cache)
-        uses: ./.github/actions/bazel-upload-checksums/
-        with:
-          artifact-name: shasums-cache
-
   bazel-test-macos-intel:
     name: Bazel Test macOS Intel
     timeout-minutes: 130
@@ -300,43 +296,38 @@ jobs:
       - <<: *checkout
       - name: Run Build IC
         id: build-ic
-        run: ./ci/scripts/run-build-ic.sh
+        uses: ./.github/actions/bazel
+        with:
+          execlogs-artifact-name: execlogs-build-ic
+          run: ./ci/scripts/run-build-ic.sh
         env:
-          BAZEL_COMMAND: build --config=ci
-          BAZEL_TARGETS: //...
           MERGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           BRANCH_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           RUN_ON_DIFF_ONLY: ${{ needs.config.outputs.diff_only }}
           RELEASE_BUILD: ${{ needs.config.outputs.release_build }}
-      - name: Upload SHA256SUMS (nocache)
-        uses: ./.github/actions/bazel-upload-checksums/
-        with:
-          artifact-name: shasums-nocache
 
   build-determinism:
     name: Build Determinism
     runs-on: ubuntu-latest
-    timeout-minutes: 30
-    # NOTE: this expects "build-ic" to have built the same set of targets
-    # as "bazel-test-all"
     needs: [build-ic, bazel-test-all]
     steps:
-      - <<: *checkout
-      - name: Download SHA256SUMS (cache)
+      - name: Download execution logs (cache)
         uses: actions/download-artifact@v4
         with:
-          name: shasums-cache
-          path: shasums-cache
-      - name: Download SHA256SUMS (nocache)
+          name: execlogs-bazel-test-all
+          path: execlogs-cache
+      - name: Download execution logs (nocache)
         uses: actions/download-artifact@v4
         with:
-          name: shasums-nocache
-          path: shasums-nocache
+          name: execlogs-build-ic
+          path: execlogs-nocache
 
       - name: Build Determinism Test
         run: |
-          n_lines_cache=$(cat shasums-cache/SHA256SUMS | wc -l)
-          n_lines_nocache=$(cat shasums-nocache/SHA256SUMS | wc -l)
+          set -euo pipefail
+
+          n_lines_cache=$(cat execlogs-cache/execlogs.csv | wc -l)
+          n_lines_nocache=$(cat execlogs-nocache/execlogs.csv | wc -l)
           echo "comparing $n_lines_cache (cache) and $n_lines_nocache (nocache) lines"
 
           # running tests may not pull all targets locally. If that's the case,
@@ -346,17 +337,28 @@ jobs:
             exit 0
           fi
 
-          # this checks that all lines in the first argument (subset) are identical
-          # in the second argument (set)
-          difference=$(comm -23 <(sort shasums-nocache/SHA256SUMS | uniq) <(sort shasums-cache/SHA256SUMS | uniq))
-          if [ -n "$difference" ]; then
-            echo "Build Determinism Check Failed! Please contact IDX."
-            echo "The following artifacts were different:"
-            echo "$difference"
-            exit 1
-          fi
+          # sort the files by the field we join on (artifact path), see below
+          sponge=$(mktemp)
+
+          sort -t, -k2 <execlogs-cache/execlogs.csv >"$sponge"
+          cp "$sponge" execlogs-cache/execlogs.csv
+
+          sort -t, -k2 <execlogs-nocache/execlogs.csv >"$sponge"
+          cp "$sponge" execlogs-nocache/execlogs.csv
+
+          rm "$sponge"
 
-          echo "Build Determinism Check Successful"
+          # join the CSVs (separator ',') and compare the hashes. This creates a table with the following columns:
+          #   //rs/foo,bazel-out/path/to-artifact,deadbeef,deafb33f
+          # target label (1.1), artifact path (1.2), and hashes (1.3 & 2.3). The join is done
+          # on the artifact path, second field on input one (-1) and input two (-2) :'-12 -22'
+          # The output is then compared with awk, printing mismatches, and keeping track of how many mismatches we
+          # encountered.
+          join \
+            -t, -o 1.1,1.2,1.3,2.3 -12 -22 \
+            execlogs-cache/execlogs.csv \
+            execlogs-nocache/execlogs.csv \
+            | awk -F, 'BEGIN { N_BAD=0; } $3 != $4 { print $1 " " $2 ": " $3 " != " $4; N_BAD++; } END { if (N_BAD) { print N_BAD " mismatches found"; exit 1; } else { print "No mismatches"; }; }'
 
   cargo-clippy-linux:
     name: Cargo Clippy Linux

.github/workflows-source/ci-main.yml

@@ -54,15 +54,15 @@ jobs:
           filters: |
             fuzzers:
               - '.github/workflows/ci-pr-only.yml'
-              - 'bin/build-all-fuzzers.sh'
+              - 'bin/fuzzing/build-all-fuzzers.sh'
               - 'bazel/fuzz_testing.bzl'
       - name: Run Bazel Build Fuzzers Archives
         id: bazel-build-fuzzers-archives
         if: steps.filter.outputs.fuzzers == 'true'
         shell: bash
         run: |
           set -euo pipefail
-          cd "${GITHUB_WORKSPACE}"/bin
+          cd "${GITHUB_WORKSPACE}"/bin/fuzzing/
           ./build-all-fuzzers.sh --zip
 
   lock-generate:
@@ -95,7 +95,8 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.head_ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.ref }}
           token: ${{ steps.app-token.outputs.token }}
       - name: Run Lock Generate
         id: lock-generate

.github/workflows-source/ci-pr-only.yml

@@ -73,10 +73,10 @@ jobs:
       - <<: *checkout
       - name: Run Libfuzzer targets
         shell: bash
-        run: ./bin/run-all-fuzzers.sh --libfuzzer 100
+        run: ./bin/fuzzing/run-all-fuzzers.sh --libfuzzer 100
       - name: Run AFL targets
         shell: bash
-        run: ./bin/run-all-fuzzers.sh --afl 100
+        run: ./bin/fuzzing/run-all-fuzzers.sh --afl 100
       - name: Post Slack Notification
         uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
         if: failure()