Merge #7047: ci: run multiprocess / tsan on arm runners

ecbce32 fix: resolve quotes (pasta)
72b77f3 ci: fix check-cache job to run on correct runner architecture (pasta)
7ef6f3d chore: move suppressions to "external libraries" (pasta)
64d5fb4 test: add comprehensive TSAN suppression for BDB pthread mutex (pasta)
1fa7b64 test: add TSAN deadlock suppression for DatabaseBatch (pasta)
550e588 test: add TSAN deadlock suppression for BerkeleyBatch (pasta)
6b0ed8b ci: run multiprocess job exclusively on ARM runners (pasta)
6afb8aa ci: enable multi-arch Docker image builds for x86 and ARM (pasta)

Pull request description:

  ## Issue being fixed or feature implemented
  run long running tsan / multiprocess builds on arm

  ## What was done?
  before tsan took 11:09 https://github.com/dashpay/dash/actions/runs/19945280778/job/57193769222
  after tsan took 9:52 https://github.com/PastaPastaPasta/dash/actions/runs/19967178415/job/57305624043

  before multiprocess took 48:45 https://github.com/dashpay/dash/actions/runs/19945280778/job/57193769225
  after multiprocess took 42:04 https://github.com/PastaPastaPasta/dash/actions/runs/19967178415/job/57305624116

  (still waiting on the functional test runs, I expect these to also improve, but not sure)

  ## How Has This Been Tested?
  CI looks ~ good here: https://github.com/PastaPastaPasta/dash/actions/runs/19967178415/job/57305624118

  ## Breaking Changes

  ## Checklist:
    _Go over all the following points, and put an `x` in all the boxes that apply._
  - [ ] I have performed a self-review of my own code
  - [ ] I have commented my code, particularly in hard-to-understand areas
  - [ ] I have added or updated relevant unit/integration/functional/e2e tests
  - [ ] I have made corresponding changes to the documentation
  - [x] I have assigned this pull request to a milestone _(for repository code-owners and collaborators only)_

ACKs for top commit:
  UdjinM6:
    utACK ecbce32

Tree-SHA512: 065ea0217cb92dfc49e863a8359110cbd7f4b1c13c50db29c25a3b6c2d2472159b7b008b109d089999d108fe6c5fa15f83edcb4cd974b8be05b19ca80ee2e02a

Author: PastaPastaPasta

.github/workflows/build-container.yml

@@ -18,15 +18,16 @@ on:
     outputs:
       path:
         description: "Path to built container"
-        value: ghcr.io/${{ jobs.build.outputs.repo }}/${{ inputs.name }}:${{ jobs.build.outputs.tag }}
+        value: ghcr.io/${{ jobs.build-amd64.outputs.repo }}/${{ inputs.name }}:${{ jobs.build-amd64.outputs.tag }}
 
 jobs:
-  build:
-    name: Build container
+  build-amd64:
+    name: Build container (amd64)
     runs-on: ubuntu-24.04
     outputs:
       tag: ${{ steps.prepare.outputs.tag }}
       repo: ${{ steps.prepare.outputs.repo }}
+      digest: ${{ steps.build.outputs.digest }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -38,8 +39,8 @@ jobs:
         run: |
           BRANCH_NAME=$(echo "${GITHUB_REF##*/}" | tr '[:upper:]' '[:lower:]')
           REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
-          echo "tag=${BRANCH_NAME}" >> $GITHUB_OUTPUT
-          echo "repo=${REPO_NAME}" >> $GITHUB_OUTPUT
+          echo "tag=${BRANCH_NAME}" >> "$GITHUB_OUTPUT"
+          echo "repo=${REPO_NAME}" >> "$GITHUB_OUTPUT"
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -52,17 +53,99 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Docker image
+        id: build
         uses: docker/build-push-action@v6
         with:
           context: ${{ inputs.context }}
           file: ${{ inputs.file }}
           push: true
+          platforms: linux/amd64
           tags: |
-            ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:${{ hashFiles(inputs.file) }}
-            ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:${{ steps.prepare.outputs.tag }}
-            ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:latest
+            ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:${{ hashFiles(inputs.file) }}-amd64
           cache-from: |
-            type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:${{ hashFiles(inputs.file) }}
+            type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:${{ hashFiles(inputs.file) }}-amd64
             type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:${{ steps.prepare.outputs.tag }}
-            type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:latest
           cache-to: type=inline
+
+  build-arm64:
+    name: Build container (arm64)
+    runs-on: ubuntu-24.04-arm
+    outputs:
+      digest: ${{ steps.build.outputs.digest }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Prepare variables
+        id: prepare
+        run: |
+          BRANCH_NAME=$(echo "${GITHUB_REF##*/}" | tr '[:upper:]' '[:lower:]')
+          REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
+          echo "tag=${BRANCH_NAME}" >> "$GITHUB_OUTPUT"
+          echo "repo=${REPO_NAME}" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ inputs.context }}
+          file: ${{ inputs.file }}
+          push: true
+          platforms: linux/arm64
+          tags: |
+            ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:${{ hashFiles(inputs.file) }}-arm64
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:${{ hashFiles(inputs.file) }}-arm64
+            type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo }}/${{ inputs.name }}:${{ steps.prepare.outputs.tag }}
+          cache-to: type=inline
+
+  create-manifest:
+    name: Create multi-arch manifest
+    runs-on: ubuntu-24.04-arm
+    needs: [build-amd64, build-arm64]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push multi-arch manifest
+        run: |
+          REPO="ghcr.io/${{ needs.build-amd64.outputs.repo }}/${{ inputs.name }}"
+          TAG="${{ needs.build-amd64.outputs.tag }}"
+          HASH_TAG="${{ hashFiles(inputs.file) }}"
+
+          # Create manifest from arch-specific images
+          docker buildx imagetools create -t "${REPO}:${HASH_TAG}" \
+            "${REPO}:${HASH_TAG}-amd64" \
+            "${REPO}:${HASH_TAG}-arm64"
+
+          docker buildx imagetools create -t "${REPO}:${TAG}" \
+            "${REPO}:${HASH_TAG}-amd64" \
+            "${REPO}:${HASH_TAG}-arm64"
+
+          docker buildx imagetools create -t "${REPO}:latest" \
+            "${REPO}:${HASH_TAG}-amd64" \
+            "${REPO}:${HASH_TAG}-arm64"

.github/workflows/build-depends.yml

@@ -11,6 +11,11 @@ on:
         description: "Path to built container at registry"
         required: true
         type: string
+      runs-on:
+        description: "Runner label to use (e.g., ubuntu-24.04 or ubuntu-24.04-arm)"
+        required: false
+        default: ubuntu-24.04
+        type: string
     outputs:
       key:
         description: "Key needed for restoring depends cache"
@@ -25,7 +30,7 @@ on:
 jobs:
   check-cache:
     name: Check cache
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.runs-on }}
     outputs:
       cache-hit: ${{ steps.cache-check.outputs.cache-hit }}
       cache-key: ${{ steps.setup.outputs.cache-key }}
@@ -52,11 +57,12 @@ jobs:
           source ./ci/dash/matrix.sh
           echo "DEP_OPTS=${DEP_OPTS}" >> "${GITHUB_OUTPUT}"
           echo "HOST=${HOST}" >> "${GITHUB_OUTPUT}"
+          echo "RUNNER_ARCH=$(uname -m)" >> "${GITHUB_OUTPUT}"
           DEP_HASH="$(echo -n "${BUILD_TARGET}" "${DEP_OPTS}" "${HOST}" | sha256sum | head -c 64)"
           echo "DEP_HASH=${DEP_HASH}" >> "${GITHUB_OUTPUT}"
           DOCKERFILE_HASH="${{ hashFiles('contrib/containers/ci/ci.Dockerfile', 'contrib/containers/ci/ci-slim.Dockerfile') }}"
           PACKAGES_HASH="${{ hashFiles('depends/packages/*', 'depends/Makefile') }}"
-          CACHE_KEY="depends-${DOCKERFILE_HASH}-${{ inputs.build-target }}-${DEP_HASH}-${PACKAGES_HASH}"
+          CACHE_KEY="depends-${DOCKERFILE_HASH}-${{ inputs.runs-on }}-${{ inputs.build-target }}-${DEP_HASH}-${PACKAGES_HASH}"
           echo "cache-key=${CACHE_KEY}" >> "${GITHUB_OUTPUT}"
           echo "Cache key: ${CACHE_KEY}"
         shell: bash
@@ -73,7 +79,7 @@ jobs:
     name: Build depends
     needs: [check-cache]
     if: needs.check-cache.outputs.cache-hit != 'true'
-    runs-on: ubuntu-24.04
+    runs-on: ${{ inputs.runs-on }}
     container:
       image: ${{ inputs.container-path }}
       options: --user root

.github/workflows/build-src.yml

@@ -24,6 +24,11 @@ on:
         required: false
         type: string
         default: ""
+      runs-on:
+        description: "Runner label to use (e.g., ubuntu-24.04 or ubuntu-24.04-arm)"
+        required: false
+        default: ubuntu-24.04
+        type: string
     outputs:
       key:
         description: "Key needed for restoring artifacts bundle"
@@ -32,7 +37,7 @@ on:
 jobs:
   build-src:
     name: Build source
-    runs-on: ubuntu-24.04
+    runs-on: ${{ inputs.runs-on }}
     outputs:
       key:  ${{ steps.bundle.outputs.key }}
     container:

.github/workflows/build.yml

@@ -86,7 +86,7 @@ jobs:
       container-path: ${{ needs.container.outputs.path }}
 
   depends-linux64_multiprocess:
-    name: x86_64-pc-linux-gnu_multiprocess
+    name: linux64_multiprocess
     uses: ./.github/workflows/build-depends.yml
     needs: [container, cache-sources]
     if: |
@@ -95,6 +95,7 @@ jobs:
     with:
       build-target: linux64_multiprocess
       container-path: ${{ needs.container.outputs.path }}
+      runs-on: ubuntu-24.04-arm
 
   depends-linux64_nowallet:
     name: x86_64-pc-linux-gnu_nowallet
@@ -176,6 +177,7 @@ jobs:
       depends-key: ${{ needs.depends-linux64_multiprocess.outputs.key }}
       depends-host: ${{ needs.depends-linux64_multiprocess.outputs.host }}
       depends-dep-opts: ${{ needs.depends-linux64_multiprocess.outputs.dep-opts }}
+      runs-on: ubuntu-24.04-arm
 
   src-linux64_nowallet:
     name: linux64_nowallet-build
@@ -211,6 +213,7 @@ jobs:
       depends-key: ${{ needs.depends-linux64_multiprocess.outputs.key }}
       depends-host: ${{ needs.depends-linux64_multiprocess.outputs.host }}
       depends-dep-opts: ${{ needs.depends-linux64_multiprocess.outputs.dep-opts }}
+      runs-on: ubuntu-24.04-arm
 
   src-linux64_ubsan:
     name: linux64_ubsan-build
@@ -263,6 +266,7 @@ jobs:
       bundle-key: ${{ needs.src-linux64_multiprocess.outputs.key }}
       build-target: linux64_multiprocess
       container-path: ${{ needs.container-slim.outputs.path }}
+      runs-on: ubuntu-24.04-arm
 
   test-linux64_nowallet:
     name: linux64_nowallet-test
@@ -290,6 +294,7 @@ jobs:
       bundle-key: ${{ needs.src-linux64_tsan.outputs.key }}
       build-target: linux64_tsan
       container-path: ${{ needs.container-slim.outputs.path }}
+      runs-on: ubuntu-24.04-arm
 
   test-linux64_ubsan:
     name: linux64_ubsan-test

.github/workflows/test-src.yml

@@ -15,6 +15,11 @@ on:
         description: "Path to built container at registry"
         required: true
         type: string
+      runs-on:
+        description: "Runner label to use (e.g., ubuntu-24.04 or ubuntu-24.04-arm)"
+        required: false
+        default: ubuntu-24.04
+        type: string
 
 env:
   INTEGRATION_TESTS_ARGS: "--extended --exclude feature_pruning,feature_dbcrash"
@@ -23,7 +28,7 @@ env:
 jobs:
   test-src:
     name: Test source
-    runs-on: ubuntu-24.04
+    runs-on: ${{ inputs.runs-on }}
     container:
       image: ${{ inputs.container-path }}
       options: --user root

ci/test/00_setup_env_native_multiprocess.sh

@@ -7,7 +7,24 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_multiprocess
-export HOST=x86_64-pc-linux-gnu
+case "$(uname -m)" in
+  aarch64)
+    export HOST=aarch64-linux-gnu
+    ;;
+  x86_64)
+    export HOST=x86_64-pc-linux-gnu
+    ;;
+  *)
+    if command -v dpkg >/dev/null 2>&1; then
+      arch="$(dpkg --print-architecture)"
+      if [ "${arch}" = "arm64" ]; then
+        export HOST=aarch64-linux-gnu
+      elif [ "${arch}" = "amd64" ]; then
+        export HOST=x86_64-pc-linux-gnu
+      fi
+    fi
+    ;;
+esac
 export PACKAGES="cmake python3 llvm clang"
 export DEP_OPTS="MULTIPROCESS=1 CC=clang-19 CXX=clang++-19"
 export RUN_TIDY=true

ci/test/00_setup_env_native_tsan.sh

@@ -7,6 +7,24 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_tsan
+case "$(uname -m)" in
+  aarch64)
+    export HOST=aarch64-linux-gnu
+    ;;
+  x86_64)
+    export HOST=x86_64-pc-linux-gnu
+    ;;
+  *)
+    if command -v dpkg >/dev/null 2>&1; then
+      arch="$(dpkg --print-architecture)"
+      if [ "${arch}" = "arm64" ]; then
+        export HOST=aarch64-linux-gnu
+      elif [ "${arch}" = "amd64" ]; then
+        export HOST=x86_64-pc-linux-gnu
+      fi
+    fi
+    ;;
+esac
 export PACKAGES="clang-19 llvm-19 libclang-rt-19-dev libc++abi-19-dev libc++-19-dev python3-zmq"
 export DEP_OPTS="CC=clang-19 CXX='clang++-19 -stdlib=libc++'"
 export TEST_RUNNER_EXTRA="--extended --exclude feature_pruning,feature_dbcrash,wallet_multiwallet.py" # Temporarily suppress ASan heap-use-after-free (see issue #14163)

contrib/containers/ci/ci-slim.Dockerfile

@@ -19,6 +19,7 @@ RUN set -ex; \
 
 # Main image
 FROM ubuntu:noble
+ARG TARGETARCH
 
 # Include built assets
 COPY --from=cppcheck-builder /src/cppcheck/build/bin/cppcheck /usr/local/bin/cppcheck
@@ -107,7 +108,16 @@ RUN set -ex; \
 
 ARG SHELLCHECK_VERSION=v0.8.0
 RUN set -ex; \
-    curl -fL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" -o /tmp/shellcheck.tar.xz; \
+    ARCH_INFERRED="${TARGETARCH}"; \
+    if [ -z "${ARCH_INFERRED}" ]; then \
+        ARCH_INFERRED="$(dpkg --print-architecture || true)"; \
+    fi; \
+    case "${ARCH_INFERRED}" in \
+        amd64|x86_64) SC_ARCH="x86_64" ;; \
+        arm64|aarch64) SC_ARCH="aarch64" ;; \
+        *) echo "Unsupported architecture for ShellCheck: ${ARCH_INFERRED}"; exit 1 ;; \
+    esac; \
+    curl -fL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.${SC_ARCH}.tar.xz" -o /tmp/shellcheck.tar.xz; \
     mkdir -p /opt/shellcheck && tar -xf /tmp/shellcheck.tar.xz -C /opt/shellcheck --strip-components=1 && rm /tmp/shellcheck.tar.xz
 ENV PATH="/opt/shellcheck:${PATH}"
 

test/sanitizer_suppressions/tsan

@@ -41,6 +41,13 @@ race:libzmq
 # Race in headers only Boost Test
 race:std::__1::ios_base::flags
 
+# BerkeleyDB internal lock ordering - BDB uses its own locking protocol
+# with pthread rwlocks that TSAN cannot properly reason about
+deadlock:BerkeleyBatch
+deadlock:DatabaseBatch
+deadlock:__db_pthread_mutex
+
+
 # Intermittent issues
 # -------------------
 #