Merge bitcoin/bitcoin#28087: ci: Use qemu-user through container engine

fad0b67 ci: Use qemu-user through container engine (MarcoFalke)

Pull request description:

  Currently the CI containers always run on the host architecture, and only wrap `bitcoind` into `qemu-user` when needed. This has many issues:

  * The `i386` tasks can not be run on non-x86 hosts.
  * `config.guess` isn't present when building the CI image, which is fine. But it prints a warning, see bitcoin/bitcoin#27739 (review)
  * The python tests are run on the host architecture, making it harder to find architecture specific bugs. See for example bitcoin/bitcoin#27529 (comment)
  * All modern container engines support automatic dispatch to qemu-user, so it seems redundant to re-invent the wheel.

  Fix all issues by:
  * removing `HOST` from `ci/test/00_setup_env.sh`.
  * removing `QEMU_USER_CMD` and `ci/test/wrap-qemu.sh`.
  * removing `DPKG_ADD_ARCH` where possible, and pruning `PACKAGES` where possible.
  * specifying the architecture in `CI_IMAGE_NAME_TAG` to be used by the container engine.

ACKs for top commit:
  fanquake:
    ACK fad0b67 - this seems ok to me, and removes complexity from our CI system.

Tree-SHA512: 85e79f9f570e292d70a629d112fd4a6e6217d96226a1b665ed13485f616d84720ad2126b7d4b22fc603049f72fa7f2163b56a6bc276319fcd8b0496304ea4157

Author: fanquake

.cirrus.yml

@@ -30,6 +30,7 @@ persistent_worker_template_env: &PERSISTENT_WORKER_TEMPLATE_ENV
 #
 # The following specific types should exist, with the following requirements:
 # - lunar: For a machine running the Linux kernel shipped with Ubuntu Lunar 23.04. The machine is recommended to have 4 CPUs and 16 GB of memory.
+# - arm64: For an aarch64 machine, recommended to have 2 CPUs and 8 GB of memory.
 persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE
   persistent_worker: {}  # Only use this if the task does not care about the type at all
 
@@ -206,13 +207,11 @@ task:
 task:
   name: 'ARM [unit tests, no functional tests] [bullseye]'
   << : *GLOBAL_TASK_TEMPLATE
-  container:
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: debian:bullseye
-      FILE_ENV: "./ci/test/00_setup_env_arm.sh"
-  << : *CREDITS_TEMPLATE
+  persistent_worker:
+    labels:
+      type: arm64  # Use arm64 worker to sidestep qemu and avoid a slow CI: https://github.com/bitcoin/bitcoin/pull/28087#issuecomment-1649399453
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
+    FILE_ENV: "./ci/test/00_setup_env_arm.sh"
 
 task:
   name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]'
@@ -230,7 +229,7 @@ task:
   << : *GLOBAL_TASK_TEMPLATE
   container:
     docker_arguments:
-      CI_IMAGE_NAME_TAG: quay.io/centos/centos:stream9
+      CI_IMAGE_NAME_TAG: "quay.io/centos/amd64:stream9"
       FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
   # For faster CI feedback, immediately schedule one task that runs all tests
   << : *CREDITS_TEMPLATE
@@ -304,7 +303,7 @@ task:
     cpu: 4
     memory: 16G  # The default memory is too small, so double everything
     docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:focal
+      CI_IMAGE_NAME_TAG: "docker.io/amd64/ubuntu:focal"
       FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV

ci/test/00_setup_env.sh

@@ -32,10 +32,6 @@ fi
 echo "Fallback to default values in env (if not yet set)"
 # The number of parallel jobs to pass down to make and test_runner.py
 export MAKEJOBS=${MAKEJOBS:--j4}
-# What host to compile for. See also ./depends/README.md
-# Tests that need cross-compilation export the appropriate HOST.
-# Tests that run natively guess the host
-export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")}
 # Whether to prefer BusyBox over GNU utilities
 export USE_BUSY_BOX=${USE_BUSY_BOX:-false}
 
@@ -61,12 +57,12 @@ export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
 # This folder exists only on the ci guest, and on the ci host as a volume.
 export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache}
 # Folder where the build result is put (bin and lib).
-export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
+export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out}
 # Folder where the build is done (dist and out-of-tree build).
 export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
 # The folder for previous release binaries.
 # This folder exists only on the ci guest, and on the ci host as a volume.
-export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST}
+export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}
 export GOAL=${GOAL:-install}

ci/test/00_setup_env_arm.sh

@@ -7,18 +7,10 @@
 export LC_ALL=C.UTF-8
 
 export HOST=arm-linux-gnueabihf
-# The host arch is unknown, so we run the tests through qemu.
-# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
-if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi
 export DPKG_ADD_ARCH="armhf"
 export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf"
-if [ -n "$QEMU_USER_CMD" ]; then
-  # Likely cross-compiling, so install the needed gcc and qemu-user
-  export PACKAGES="$PACKAGES qemu-user"
-fi
 export CONTAINER_NAME=ci_arm_linux
-# Use debian to avoid 404 apt errors when cross compiling
-export CI_IMAGE_NAME_TAG="debian:bullseye"
+export CI_IMAGE_NAME_TAG="docker.io/arm64v8/debian:bookworm"
 export USE_BUSY_BOX=true
 export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=false

ci/test/00_setup_env_i686_centos.sh

@@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8
 
 export HOST=i686-pc-linux-gnu
 export CONTAINER_NAME=ci_i686_centos
-export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream9"
+export CI_IMAGE_NAME_TAG="quay.io/centos/amd64:stream9"
 export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison util-linux"
 export PIP_PACKAGES="pyzmq"
 export GOAL="install"

ci/test/00_setup_env_i686_multiprocess.sh

@@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8
 
 export HOST=i686-pc-linux-gnu
 export CONTAINER_NAME=ci_i686_multiprocess
-export CI_IMAGE_NAME_TAG=ubuntu:20.04
+export CI_IMAGE_NAME_TAG="docker.io/amd64/ubuntu:20.04"
 export PACKAGES="cmake llvm clang g++-multilib"
 export DEP_OPTS="DEBUG=1 MULTIPROCESS=1"
 export GOAL="install"

ci/test/00_setup_env_s390x.sh

@@ -7,18 +7,9 @@
 export LC_ALL=C.UTF-8
 
 export HOST=s390x-linux-gnu
-# The host arch is unknown, so we run the tests through qemu.
-# If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
-if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi
 export PACKAGES="python3-zmq"
-if [ -n "$QEMU_USER_CMD" ]; then
-  # Likely cross-compiling, so install the needed gcc and qemu-user
-  export DPKG_ADD_ARCH="s390x"
-  export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x"
-fi
-# Use debian to avoid 404 apt errors
 export CONTAINER_NAME=ci_s390x
-export CI_IMAGE_NAME_TAG="debian:bookworm"
+export CI_IMAGE_NAME_TAG="docker.io/s390x/debian:bookworm"
 export TEST_RUNNER_ENV="LC_ALL=C"
 export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra"  # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547
 export RUN_FUNCTIONAL_TESTS=true

ci/test/04_install.sh

@@ -6,10 +6,6 @@
 
 export LC_ALL=C.UTF-8
 
-if [[ $QEMU_USER_CMD == qemu-s390* ]]; then
-  export LC_ALL=C
-fi
-
 # Create folders that are mounted into the docker
 mkdir -p "${CCACHE_DIR}"
 mkdir -p "${PREVIOUS_RELEASES_DIR}"

ci/test/06_script_b.sh

@@ -19,6 +19,11 @@ fi
 echo "Free disk space:"
 df -h
 
+# What host to compile for. See also ./depends/README.md
+# Tests that need cross-compilation export the appropriate HOST.
+# Tests that run natively guess the host
+export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")}
+
 if [ "$RUN_FUZZ_TESTS" = "true" ]; then
   export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
   if [ ! -d "$DIR_FUZZ_IN" ]; then
@@ -124,13 +129,6 @@ if [[ $HOST = *-mingw32 ]]; then
   "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
 fi
 
-if [ -n "$QEMU_USER_CMD" ]; then
-  # Generate all binaries, so that they can be wrapped
-  make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
-  make "$MAKEJOBS" -C src minisketch/test VERBOSE=1
-  "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh"
-fi
-
 if [ -n "$USE_VALGRIND" ]; then
   "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
 fi