Merge #17767: ci: Fix qemu issues

fa569e1 ci: Set LC_ALL=C when running in qemu-s390x (MarcoFalke)
fa3d776 ci: Use debian to avoid apt install 404 errors (MarcoFalke)
fabb946 ci: Install needed gcc and qemu-user iff cross-compiling (MarcoFalke)
faba467 ci: Fix QEMU_USER_CMD parse issues (MarcoFalke)
fa5d709 ci: Move wrap-qemu into separate script (MarcoFalke)

Pull request description:

  Now that the ci system no longer propagates files from the docker container back to the host, the wrap-qemu script is broken.

  To fix it, every statement in the script needs to be executed in the docker (with `DOCKER_EXEC`). Instead of juggling with triple escape sequences like `\\\"`, just move the script to a separate file and call it with `DOCKER_EXEC`.

  Also, fix a bunch of other bugs that prevent running the ci system in qemu

  See the `ci/README.md` on how to test. TLDR: Can be tested with (replace `arm` with `s390x` to run the s390x build):

  ```
  FILE_ENV="./ci/test/00_setup_env_arm.sh" MAKEJOBS="-j9" ./ci/test_run_all.sh
  ```

ACKs for top commit:
  laanwj:
    Code review ACK fa569e1

Tree-SHA512: 84ebc44a4f0261ee6c29605a6896a1833ff6c81d729e6d08dd111941f570ce73221422bd3303e1108a266ec5eab2148bd5ee1cf6bc01477d8cc9a6c5bf2b34c2

Author: laanwj

.travis.yml

@@ -82,14 +82,14 @@ jobs:
         - set -o errexit; source ./ci/lint/06_script.sh
 
     - stage: test
-      name: 'ARM  [GOAL: install]  [bionic]  [unit tests, functional tests]'
+      name: 'ARM  [GOAL: install]  [buster]  [unit tests, functional tests]'
       arch: arm64
       env: >-
         FILE_ENV="./ci/test/00_setup_env_arm.sh"
         QEMU_USER_CMD=""  # Can run the tests natively without qemu
 
     - stage: test
-      name: 'S390x  [GOAL: install]  [bionic]  [unit tests, functional tests]'
+      name: 'S390x  [GOAL: install]  [buster]  [unit tests, functional tests]'
       arch: s390x
       env: >-
         FILE_ENV="./ci/test/00_setup_env_s390x.sh"

ci/test/00_setup_env.sh

@@ -51,7 +51,7 @@ export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
 export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export WINEDEBUG=${WINEDEBUG:-fixme-all}
-export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git}
+export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps}
 export GOAL=${GOAL:-install}
 export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets}
 export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH

ci/test/00_setup_env_arm.sh

@@ -9,11 +9,15 @@ export LC_ALL=C.UTF-8
 export HOST=arm-linux-gnueabihf
 # The host arch is unknown, so we run the tests through qemu.
 # If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
-export QEMU_USER_CMD="${QEMU_USER_CMD:"qemu-arm -L /usr/arm-linux-gnueabihf/"}"
-# We don't know whether the host can run the cross compiled binaries. To run them, either qemu-user or libc6:armhf for
-# the target is required, so install both.
+if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi
 export DPKG_ADD_ARCH="armhf"
-export PACKAGES="python3 g++-arm-linux-gnueabihf busybox qemu-user libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf"
+export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf"
+if [ -n "$QEMU_USER_CMD" ]; then
+  # Likely cross-compiling, so install the needed gcc and qemu-user
+  export PACKAGES="$PACKAGES qemu-user"
+fi
+# Use debian to avoid 404 apt errors when cross compiling
+export DOCKER_NAME_TAG="debian:buster"
 export USE_BUSY_BOX=true
 export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=true

ci/test/00_setup_env_s390x.sh

@@ -9,8 +9,15 @@ export LC_ALL=C.UTF-8
 export HOST=s390x-linux-gnu
 # The host arch is unknown, so we run the tests through qemu.
 # If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
-export QEMU_USER_CMD="${QEMU_USER_CMD:"qemu-s390x"}"
-export PACKAGES="python3-zmq bsdmainutils qemu-user"
+if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi
+export PACKAGES="python3-zmq"
+if [ -n "$QEMU_USER_CMD" ]; then
+  # Likely cross-compiling, so install the needed gcc and qemu-user
+  export DPKG_ADD_ARCH="s390x"
+  export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x libfontconfig1:s390x libxcb1:s390x"
+fi
+# Use debian to avoid 404 apt errors
+export DOCKER_NAME_TAG="debian:buster"
 export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=true
 export GOAL="install"

ci/test/04_install.sh

@@ -9,6 +9,9 @@ export LC_ALL=C.UTF-8
 if [[ $DOCKER_NAME_TAG == centos* ]]; then
   export LC_ALL=en_US.utf8
 fi
+if [[ $QEMU_USER_CMD == qemu-s390* ]]; then
+  export LC_ALL=C
+fi
 
 if [ "$TRAVIS_OS_NAME" == "osx" ]; then
   set +o errexit
@@ -42,7 +45,7 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=
 export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan"
 export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
-env | grep -E '^(BITCOIN_CONFIG|BASE_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
+env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_ADMIN="--cap-add SYS_ADMIN"
 elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
@@ -73,16 +76,6 @@ else
   }
 fi
 
-if [ "$TRAVIS_OS_NAME" == "osx" ]; then
-  top -l 1 -s 0 | awk ' /PhysMem/ {print}'
-  echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
-else
-  DOCKER_EXEC free -m -h
-  DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
-  DOCKER_EXEC echo "Free disk space:"
-  DOCKER_EXEC df -h
-fi
-
 if [ -n "$DPKG_ADD_ARCH" ]; then
   DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH"
 fi
@@ -95,6 +88,16 @@ elif [ "$TRAVIS_OS_NAME" != "osx" ]; then
   ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
 fi
 
+if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+  top -l 1 -s 0 | awk ' /PhysMem/ {print}'
+  echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
+else
+  DOCKER_EXEC free -m -h
+  DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
+  DOCKER_EXEC echo "Free disk space:"
+  DOCKER_EXEC df -h
+fi
+
 if [ ! -d ${DIR_QA_ASSETS} ]; then
   DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
 fi

ci/test/06_script_b.sh

@@ -11,16 +11,7 @@ if [ -n "$QEMU_USER_CMD" ]; then
   # Generate all binaries, so that they can be wrapped
   DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1
   DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1
-  for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do
-    # shellcheck disable=SC2044
-    for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do
-      echo "Wrap $b ..."
-      DOCKER_EXEC mv "$b" "${b}_orig"
-      DOCKER_EXEC echo "\#\!/usr/bin/env bash" \> "$b"
-      DOCKER_EXEC echo "$QEMU_USER_CMD \\\"${b}_orig\\\" \\\"\\\$@\\\"" \>\> "$b"
-      DOCKER_EXEC chmod +x "$b"
-    done
-  done
+  DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh"
   END_FOLD
 fi
 

ci/test/wrap-qemu.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do
+    # shellcheck disable=SC2044
+    for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do
+      echo "Wrap $b ..."
+      mv "$b" "${b}_orig"
+      echo '#!/usr/bin/env bash' > "$b"
+      echo "$QEMU_USER_CMD \"${b}_orig\" \"\$@\"" >> "$b"
+      chmod +x "$b"
+    done
+done