TheCharlatan
diff --git a/‎.github/actions/configure-docker/action.yml‎
Lines changed: 12 additions & 3 deletions b/‎.github/actions/configure-docker/action.yml‎
Lines changed: 12 additions & 3 deletions
diff --git a/‎.github/workflows/ci.yml‎
Lines changed: 16 additions & 7 deletions b/‎.github/workflows/ci.yml‎
Lines changed: 16 additions & 7 deletions
diff --git a/‎ci/test/00_setup_env.sh‎
Lines changed: 1 addition & 1 deletion b/‎ci/test/00_setup_env.sh‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎ci/test/00_setup_env_mac_native.sh‎
Lines changed: 1 addition & 3 deletions b/‎ci/test/00_setup_env_mac_native.sh‎
Lines changed: 1 addition & 3 deletions
diff --git a/‎ci/test/00_setup_env_native_alpine_musl.sh‎
Lines changed: 1 addition & 1 deletion b/‎ci/test/00_setup_env_native_alpine_musl.sh‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh‎
Lines changed: 4 additions & 4 deletions b/‎ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh‎
Lines changed: 4 additions & 4 deletions
diff --git a/‎ci/test/00_setup_env_native_tidy.sh‎
Lines changed: 1 addition & 1 deletion b/‎ci/test/00_setup_env_native_tidy.sh‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎ci/test/01_base_install.sh‎
Lines changed: 1 addition & 1 deletion b/‎ci/test/01_base_install.sh‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎ci/test/02_run_container.py‎
Lines changed: 92 additions & 4 deletions b/‎ci/test/02_run_container.py‎
Lines changed: 92 additions & 4 deletions
diff --git a/‎ci/test/02_run_container.sh‎
Lines changed: 0 additions & 71 deletions b/‎ci/test/02_run_container.sh‎
Lines changed: 0 additions & 71 deletions
@@ -4,12 +4,21 @@ inputs:
   cache-provider:
     description: 'gha or cirrus cache provider'
     required: true
-    options:
-    - gh
-    - cirrus
 runs:
   using: 'composite'
   steps:
+    - name: Check inputs
+      shell: bash
+      run: |
+        # We expect only gha or cirrus as inputs to cache-provider
+        case "${{ inputs.cache-provider }}" in
+          gha|cirrus)
+            ;;
+          *)
+            echo "::warning title=Unknown input to configure docker action::Provided value was ${{ inputs.cache-provider }}"
+            ;;
+        esac
+
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
       with:
 
@@ -35,6 +35,14 @@ jobs:
     outputs:
       provider: ${{ steps.runners.outputs.provider }}
     steps:
+      - name: Annotate with pull request number
+        # This annotation is machine-readable and can be used to assign a check
+        # run to its corresponding pull request. Running in one check run is
+        # sufficient for each check suite.
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            echo "::notice title=debug_pull_request_number_str::${{ github.event.number }}"
+          fi
       - id: runners
         run: |
           if [[ "${REPO_USE_CIRRUS_RUNNERS}" == "${{ github.repository }}" ]]; then
@@ -157,11 +165,6 @@ jobs:
           brew install --quiet python@3 || brew link --overwrite python@3
           brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent zeromq qt@6 qrencode capnp
 
-      - name: Install Python packages
-        run: |
-          git clone -b v2.1.0 https://github.com/capnproto/pycapnp
-          pip3 install ./pycapnp -C force-bundled-libcapnp=True --break-system-packages
-
       - name: Set Ccache directory
         run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV"
 
@@ -531,6 +534,12 @@ jobs:
             timeout-minutes: 120
             file-env: './ci/test/00_setup_env_native_tsan.sh'
 
+          - name: 'MSan, fuzz'
+            cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-md'
+            fallback-runner: 'ubuntu-24.04'
+            timeout-minutes: 150
+            file-env: './ci/test/00_setup_env_native_fuzz_with_msan.sh'
+
           - name: 'MSan, depends'
             cirrus-runner: 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-lg'
             fallback-runner: 'ubuntu-24.04'
@@ -559,7 +568,7 @@ jobs:
         run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh
 
       - name: Set mmap_rnd_bits
-        if: ${{ env.CONTAINER_NAME == 'ci_native_tsan' || env.CONTAINER_NAME == 'ci_native_msan' }}
+        if: ${{ env.CONTAINER_NAME == 'ci_native_tsan' || env.CONTAINER_NAME == 'ci_native_msan' || env.CONTAINER_NAME == 'ci_native_fuzz_msan' }}
         # Prevents crashes due to high ASLR entropy
         run: sudo sysctl -w vm.mmap_rnd_bits=28
 
@@ -572,7 +581,7 @@ jobs:
   lint:
     name: 'lint'
     needs: runners
-    runs-on: ${{ needs.runners.outputs.use-cirrus-runners == 'true' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }}
+    runs-on: ${{ needs.runners.outputs.provider == 'cirrus' && 'ghcr.io/cirruslabs/ubuntu-runner-amd64:24.04-xs' || 'ubuntu-24.04' }}
     if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }}
     timeout-minutes: 20
     env:
 
@@ -64,7 +64,7 @@ export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out}
 # The folder for previous release binaries.
 # This folder exists only on the ci guest, and on the ci host as a volume.
 export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases}
-export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkgconf curl ca-certificates ccache python3 rsync git procps bison e2fsprogs cmake ninja-build}
+export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkgconf curl ca-certificates ccache python3-dev rsync git procps bison e2fsprogs cmake ninja-build}
 export GOAL=${GOAL:-install}
 export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets}
 export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"}
 
@@ -6,10 +6,8 @@
 
 export LC_ALL=C.UTF-8
 
-# Homebrew's [email protected] is marked as externally managed (PEP 668).
-# Therefore, `--break-system-packages` is needed.
 export CONTAINER_NAME="ci_mac_native"  # macos does not use a container, but the env var is needed for logging
-export PIP_PACKAGES="--break-system-packages zmq"
+export PIP_PACKAGES="--break-system-packages pycapnp zmq"
 export GOAL="install deploy"
 export CMAKE_GENERATOR="Ninja"
 export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DBUILD_KERNEL_LIB=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_TEST=ON -DREDUCE_EXPORTS=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'"
 
@@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_alpine_musl
 export CI_IMAGE_NAME_TAG="mirror.gcr.io/alpine:3.22"
-export CI_BASE_PACKAGES="build-base musl-dev pkgconf curl ccache make ninja git python3 py3-pip which patch xz procps rsync util-linux bison e2fsprogs cmake dash linux-headers"
+export CI_BASE_PACKAGES="build-base musl-dev pkgconf curl ccache make ninja git python3-dev py3-pip which patch xz procps rsync util-linux bison e2fsprogs cmake dash linux-headers"
 export PIP_PACKAGES="--break-system-packages pyzmq pycapnp"
 export DEP_OPTS="DEBUG=1"
 export GOAL="install"
 
@@ -7,10 +7,10 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel
-export CI_IMAGE_NAME_TAG="mirror.gcr.io/debian:bookworm"
-# Use minimum supported python3.10 (or best-effort 3.11) and clang-16, see doc/dependencies.md
-export PACKAGES="python3-zmq python3-pip clang-16 llvm-16 libc++abi-16-dev libc++-16-dev"
+export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04"
+# Use minimum supported python3.10 (or best-effort 3.12) and clang-17, see doc/dependencies.md
+export PACKAGES="python3-zmq python3-pip clang-17 llvm-17 libc++abi-17-dev libc++-17-dev"
 export PIP_PACKAGES="--break-system-packages pycapnp"
-export DEP_OPTS="NO_WALLET=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'"
+export DEP_OPTS="NO_WALLET=1 CC=clang-17 CXX='clang++-17 -stdlib=libc++'"
 export GOAL="install"
 export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_LIB=ON -DBUILD_KERNEL_TEST=ON -DBUILD_SHARED_LIBS=ON"
@@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8
 
 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04"
 export CONTAINER_NAME=ci_native_tidy
-export TIDY_LLVM_V="20"
+export TIDY_LLVM_V="21"
 export APT_LLVM_V="${TIDY_LLVM_V}"
 export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libzmq3-dev systemtap-sdt-dev qt6-base-dev qt6-tools-dev qt6-l10n-tools libqrencode-dev libsqlite3-dev libcapnp-dev capnproto"
 export NO_DEPENDS=1
 
@@ -57,7 +57,7 @@ if [ -n "$PIP_PACKAGES" ]; then
 fi
 
 if [[ -n "${USE_INSTRUMENTED_LIBCPP}" ]]; then
-  ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-21.1.1" /llvm-project
+  ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-21.1.5" /llvm-project
 
   cmake -G Ninja -B /cxx_build/ \
     -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
 
@@ -3,16 +3,19 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or https://opensource.org/license/mit/.
 
+from pathlib import Path
 import os
 import shlex
 import subprocess
 import sys
+import time
 
 
 def run(cmd, **kwargs):
     print("+ " + shlex.join(cmd), flush=True)
+    kwargs.setdefault("check", True)
     try:
-        return subprocess.run(cmd, check=True, **kwargs)
+        return subprocess.run(cmd, **kwargs)
     except Exception as e:
         sys.exit(e)
 
@@ -36,8 +39,8 @@ def main():
     # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME
     # to allow support starting multiple runs simultaneously by the same user.
     env_file = "/tmp/env-{u}-{c}".format(
-        u=os.getenv("USER"),
-        c=os.getenv("CONTAINER_NAME"),
+        u=os.environ["USER"],
+        c=os.environ["CONTAINER_NAME"],
     )
     with open(env_file, "w", encoding="utf8") as file:
         for k, v in os.environ.items():
@@ -64,7 +67,92 @@ def main():
         cmd_build += [os.environ["BASE_READ_ONLY_DIR"]]
 
         print(f"Building {os.environ['CONTAINER_NAME']} image tag to run in")
-        run(cmd_build)
+        if run(cmd_build, check=False).returncode != 0:
+            print(f"Retry building {os.environ['CONTAINER_NAME']} image tag after failure")
+            time.sleep(3)
+            run(cmd_build)
+
+        for suffix in ["ccache", "depends", "depends_sources", "previous_releases"]:
+            run(["docker", "volume", "create", f"{os.environ['CONTAINER_NAME']}_{suffix}"], check=False)
+
+        CI_CCACHE_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_ccache,dst={os.environ['CCACHE_DIR']}"
+        CI_DEPENDS_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends,dst={os.environ['DEPENDS_DIR']}/built"
+        CI_DEPENDS_SOURCES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_depends_sources,dst={os.environ['DEPENDS_DIR']}/sources"
+        CI_PREVIOUS_RELEASES_MOUNT = f"type=volume,src={os.environ['CONTAINER_NAME']}_previous_releases,dst={os.environ['PREVIOUS_RELEASES_DIR']}"
+        CI_BUILD_MOUNT = []
+
+        if os.getenv("DANGER_CI_ON_HOST_FOLDERS"):
+            # ensure the directories exist
+            for create_dir in [
+                    os.environ["CCACHE_DIR"],
+                    f"{os.environ['DEPENDS_DIR']}/built",
+                    f"{os.environ['DEPENDS_DIR']}/sources",
+                    os.environ["PREVIOUS_RELEASES_DIR"],
+                    os.environ["BASE_BUILD_DIR"],  # Unset by default, must be defined externally
+            ]:
+                Path(create_dir).mkdir(parents=True, exist_ok=True)
+
+            CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
+            CI_DEPENDS_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/built,dst={os.environ['DEPENDS_DIR']}/built"
+            CI_DEPENDS_SOURCES_MOUNT = f"type=bind,src={os.environ['DEPENDS_DIR']}/sources,dst={os.environ['DEPENDS_DIR']}/sources"
+            CI_PREVIOUS_RELEASES_MOUNT = f"type=bind,src={os.environ['PREVIOUS_RELEASES_DIR']},dst={os.environ['PREVIOUS_RELEASES_DIR']}"
+            CI_BUILD_MOUNT = [f"--mount=type=bind,src={os.environ['BASE_BUILD_DIR']},dst={os.environ['BASE_BUILD_DIR']}"]
+
+        if os.getenv("DANGER_CI_ON_HOST_CCACHE_FOLDER"):
+            if not os.path.isdir(os.environ["CCACHE_DIR"]):
+                print(f"Error: Directory '{os.environ['CCACHE_DIR']}' must be created in advance.")
+                sys.exit(1)
+            CI_CCACHE_MOUNT = f"type=bind,src={os.environ['CCACHE_DIR']},dst={os.environ['CCACHE_DIR']}"
+
+        run(["docker", "network", "create", "--ipv6", "--subnet", "1111:1111::/112", "ci-ip6net"], check=False)
+
+        if os.getenv("RESTART_CI_DOCKER_BEFORE_RUN"):
+            print("Restart docker before run to stop and clear all containers started with --rm")
+            run(["podman", "container", "rm", "--force", "--all"])  # Similar to "systemctl restart docker"
+
+            # Still prune everything in case the filtered pruning doesn't work, or if labels were not set
+            # on a previous run. Belt and suspenders approach, should be fine to remove in the future.
+            # Prune images used by --external containers (e.g. build containers) when
+            # using podman.
+            print("Prune all dangling images")
+            run(["podman", "image", "prune", "--force", "--external"])
+
+        print(f"Prune all dangling {CI_IMAGE_LABEL} images")
+        # When detecting podman-docker, `--external` should be added.
+        run(["docker", "image", "prune", "--force", "--filter", f"label={CI_IMAGE_LABEL}"])
+
+        cmd_run = ["docker", "run", "--rm", "--interactive", "--detach", "--tty"]
+        cmd_run += [
+            "--cap-add=LINUX_IMMUTABLE",
+            *shlex.split(os.getenv("CI_CONTAINER_CAP", "")),
+            f"--mount=type=bind,src={os.environ['BASE_READ_ONLY_DIR']},dst={os.environ['BASE_READ_ONLY_DIR']},readonly",
+            f"--mount={CI_CCACHE_MOUNT}",
+            f"--mount={CI_DEPENDS_MOUNT}",
+            f"--mount={CI_DEPENDS_SOURCES_MOUNT}",
+            f"--mount={CI_PREVIOUS_RELEASES_MOUNT}",
+            *CI_BUILD_MOUNT,
+            f"--env-file={env_file}",
+            f"--name={os.environ['CONTAINER_NAME']}",
+            "--network=ci-ip6net",
+            f"--platform={os.environ['CI_IMAGE_PLATFORM']}",
+            os.environ["CONTAINER_NAME"],
+        ]
+
+        container_id = run(
+            cmd_run,
+            stdout=subprocess.PIPE,
+            text=True,
+        ).stdout.strip()
+        os.environ["CI_CONTAINER_ID"] = container_id
+
+    # GNU getopt is required for the CI_RETRY_EXE script
+    if os.getenv("CI_OS_NAME") == "macos":
+        prefix = run(
+            ["brew", "--prefix", "gnu-getopt"],
+            stdout=subprocess.PIPE,
+            text=True,
+        ).stdout.strip()
+        os.environ["IN_GETOPT_BIN"] = f"{prefix}/bin/getopt"
 
     run(["./ci/test/02_run_container.sh"])  # run the remainder
 
 
@@ -5,76 +5,10 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 export LC_ALL=C.UTF-8
-export CI_IMAGE_LABEL="bitcoin-ci-test"
 
 set -o errexit -o pipefail -o xtrace
 
 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
-  docker volume create "${CONTAINER_NAME}_ccache" || true
-  docker volume create "${CONTAINER_NAME}_depends" || true
-  docker volume create "${CONTAINER_NAME}_depends_sources" || true
-  docker volume create "${CONTAINER_NAME}_previous_releases" || true
-
-  CI_CCACHE_MOUNT="type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR"
-  CI_DEPENDS_MOUNT="type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR/built"
-  CI_DEPENDS_SOURCES_MOUNT="type=volume,src=${CONTAINER_NAME}_depends_sources,dst=$DEPENDS_DIR/sources"
-  CI_PREVIOUS_RELEASES_MOUNT="type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR"
-  CI_BUILD_MOUNT=""
-
-  if [ "$DANGER_CI_ON_HOST_FOLDERS" ]; then
-    # ensure the directories exist
-    mkdir -p "${CCACHE_DIR}"
-    mkdir -p "${DEPENDS_DIR}/built"
-    mkdir -p "${DEPENDS_DIR}/sources"
-    mkdir -p "${PREVIOUS_RELEASES_DIR}"
-    mkdir -p "${BASE_BUILD_DIR}"  # Unset by default, must be defined externally
-
-    CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=$CCACHE_DIR"
-    CI_DEPENDS_MOUNT="type=bind,src=${DEPENDS_DIR}/built,dst=$DEPENDS_DIR/built"
-    CI_DEPENDS_SOURCES_MOUNT="type=bind,src=${DEPENDS_DIR}/sources,dst=$DEPENDS_DIR/sources"
-    CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR"
-    CI_BUILD_MOUNT="--mount type=bind,src=${BASE_BUILD_DIR},dst=${BASE_BUILD_DIR}"
-  fi
-
-  if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then
-    if [ ! -d "${CCACHE_DIR}" ]; then
-      echo "Error: Directory '${CCACHE_DIR}' must be created in advance."
-      exit 1
-    fi
-    CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}"
-  fi
-
-  docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true
-
-  if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
-    echo "Restart docker before run to stop and clear all containers started with --rm"
-    podman container rm --force --all  # Similar to "systemctl restart docker"
-
-    # Still prune everything in case the filtered pruning doesn't work, or if labels were not set
-    # on a previous run. Belt and suspenders approach, should be fine to remove in the future.
-    # Prune images used by --external containers (e.g. build containers) when
-    # using podman.
-    echo "Prune all dangling images"
-    podman image prune --force --external
-  fi
-  echo "Prune all dangling $CI_IMAGE_LABEL images"
-  # When detecting podman-docker, `--external` should be added.
-  docker image prune --force --filter "label=$CI_IMAGE_LABEL"
-
-  # shellcheck disable=SC2086
-  CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \
-                  --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \
-                  --mount "${CI_CCACHE_MOUNT}" \
-                  --mount "${CI_DEPENDS_MOUNT}" \
-                  --mount "${CI_DEPENDS_SOURCES_MOUNT}" \
-                  --mount "${CI_PREVIOUS_RELEASES_MOUNT}" \
-                  ${CI_BUILD_MOUNT} \
-                  --env-file /tmp/env-$USER-$CONTAINER_NAME \
-                  --name "$CONTAINER_NAME" \
-                  --network ci-ip6net \
-                  --platform="${CI_IMAGE_PLATFORM}" \
-                  "$CONTAINER_NAME")
-  export CI_CONTAINER_ID
   export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}"
 else
   echo "Running on host system without docker wrapper"
@@ -83,11 +17,6 @@ else
   mkdir -p "${PREVIOUS_RELEASES_DIR}"
 fi
 
-if [ "$CI_OS_NAME" == "macos" ]; then
-  IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt"
-  export IN_GETOPT_BIN
-fi
-
 CI_EXEC () {
   $CI_EXEC_CMD_PREFIX bash -c "export PATH=\"/path_with space:${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH\" && cd \"${BASE_ROOT_DIR}\" && $*"
 }