Merge #19267: ci: Upgrade most ci configs to focal

fa05f44 ci: Upgrade most ci configs to focal (MarcoFalke)
fad6720 doc: move doc to ci readme (MarcoFalke)
fa88077 ci: Have one config run in xenial to test against python3.5 (MarcoFalke)
fa6ddb2 travis: Always run multiprocess build (MarcoFalke)

Pull request description:

  Generally developers compile with recent compilers, so bumping the ci configs to a recent OS should be uncontroversial. Older OSes (especially with compiler sanitizers) need workarounds that can be dropped by running on a more recent OS.

  This pull changes the asan sanitizer and the experimental multiprocess build to use focal.
  Also, it runs the no_wallet config on xenial to test against python 3.5, according to `doc/dependencies.md`.

  Finally, all configs that mimic gitian (win and mac) will stay at bionic.

ACKs for top commit:
  Sjors:
    ACK fa05f44, assuming Travis passes
  hebasto:
    ACK fa05f44

Tree-SHA512: 55ec56c71ba2280d27c1a8856a1e6c310b1fbf469d5a8a1dde228063e3892e1dd1e51408ecff7a3d77ac2ae018daa9e9bbbb60598cdeaab8c32a146b11b3e7c4

Author: MarcoFalke

.cirrus.yml

@@ -48,9 +48,9 @@ global_task_template: &GLOBAL_TASK_TEMPLATE
 #    - choco install python --version=3.7.7 -y
 
 task:
-  name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]'
+  name: 'x86_64 Linux  [GOAL: install]  [focal]  [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:bionic
+    image: ubuntu:focal
   env:
     FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"

.travis.yml

@@ -1,18 +1,4 @@
-# The test build matrix (stage: test) is constructed to test a wide range of
-# configurations, rather than a single pass/fail. This helps to catch build
-# failures and logic errors that present on platforms other than the ones the
-# author has tested.
-#
-# Some builders use the dependency-generator in `./depends`, rather than using
-# apt-get to install build dependencies. This guarantees that the tester is
-# using the same versions as Gitian, so the build results are nearly identical
-# to what would be found in a final release.
-#
-# In order to avoid rebuilding all dependencies for each build, the binaries
-# are cached and re-used when possible. Changes in the dependency-generator
-# will trigger cache-invalidation and rebuilds as necessary.
-#
-# These caches can be manually removed if necessary. This is one of the very
+# Travis caches can be manually removed if necessary. This is one of the very
 # few manual operations that is possible with Travis, and it can be done by a
 # Bitcoin Core GitHub member via the Travis web interface [0].
 #
@@ -27,7 +13,7 @@
 
 version: ~> 1.0
 
-dist: xenial
+dist: bionic
 os: linux
 language: minimal
 arch: amd64
@@ -126,13 +112,12 @@ jobs:
         FILE_ENV="./ci/test/00_setup_env_native_fuzz.sh"
 
     - stage: test
-      name: 'x86_64 Linux [GOAL: install]  [bionic]  [multiprocess]'
-      if: type != pull_request OR commit_message =~ /depends:|multiprocess:/ # Skip on non-depends, non-multiprocess PRs
+      name: 'x86_64 Linux [GOAL: install]  [focal]  [multiprocess]'
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh"
 
     - stage: test
-      name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no wallet]'
+      name: 'x86_64 Linux  [GOAL: install]  [xenial]  [no wallet]'
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh"
 

ci/README.md

@@ -1,12 +1,8 @@
-## ci scripts
+## CI Scripts
 
 This directory contains scripts for each build step in each build stage.
 
-Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the
-[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named
-and numbered according to which stage and lifecycle step it belongs to.
-
-### Running a stage locally
+### Running a Stage Locally
 
 Be aware that the tests will be built and run in-place, so please run at your own risk.
 If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
@@ -36,3 +32,34 @@ To run the test stage with a specific configuration,
 ```
 FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
 ```
+
+### Configurations
+
+The test files (`FILE_ENV`) are constructed to test a wide range of
+configurations, rather than a single pass/fail. This helps to catch build
+failures and logic errors that present on platforms other than the ones the
+author has tested.
+
+Some builders use the dependency-generator in `./depends`, rather than using
+the system package manager to install build dependencies. This guarantees that
+the tester is using the same versions as the release builds, which also use
+`./depends`.
+
+If no `FILE_ENV` has been specified or values are left out, `00_setup_env.sh`
+is used as the default configuration with fallback values.
+
+It is also possible to force a specific configuration without modifying the
+file. For example,
+
+```
+MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
+```
+
+The files starting with `0n` (`n` greater than 0) are the scripts that are run
+in order.
+
+### Cache
+
+In order to avoid rebuilding all dependencies for each build, the binaries are
+cached and re-used when possible. Changes in the dependency-generator will
+trigger cache-invalidation and rebuilds as necessary.

ci/test/00_setup_env_mac_host.sh

@@ -7,6 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export HOST=x86_64-apple-darwin16
+export DOCKER_NAME_TAG=ubuntu:18.04  # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well)
 export PIP_PACKAGES="zmq"
 export GOAL="install"
 export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror"

ci/test/00_setup_env_native_asan.sh

@@ -7,8 +7,8 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_asan
-export PACKAGES="clang-8 llvm-8 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
-# Use clang-8 instead of default clang (which is clang-6 on Bionic) to avoid spurious segfaults when running on ppc64le
+export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
+export DOCKER_NAME_TAG=ubuntu:20.04
 export NO_DEPENDS=1
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang-8 CXX=clang++-8"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"

ci/test/00_setup_env_native_multiprocess.sh

@@ -7,6 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_multiprocess
+export DOCKER_NAME_TAG=ubuntu:20.04
 export PACKAGES="cmake python3"
 export DEP_OPTS="MULTIPROCESS=1"
 export GOAL="install"

ci/test/00_setup_env_native_nowallet.sh

@@ -7,6 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_nowallet
+export DOCKER_NAME_TAG=ubuntu:16.04  # Use xenial to have one config run the tests in python3.5, see doc/dependencies.md
 export PACKAGES="python3-zmq"
 export DEP_OPTS="NO_WALLET=1"
 export GOAL="install"

ci/test/00_setup_env_native_qt5.sh

@@ -7,6 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_qt5
+export DOCKER_NAME_TAG=ubuntu:18.04  # Check that bionic can compile our c++17 and run our functional tests in python3
 export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev"
 export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
 export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash

ci/test/00_setup_env_win64.sh

@@ -7,6 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_win64
+export DOCKER_NAME_TAG=ubuntu:18.04  # Check that bionic can cross-compile to win64 (bionic is used in the gitian build as well)
 export HOST=x86_64-w64-mingw32
 export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64"
 export RUN_FUNCTIONAL_TESTS=false