Merge #16582: test: Rework ci (Use travis only as fallback env)

faeacf3 ci: Add note that this assumes a fresh git clone (MarcoFalke)
fa6cbdc ci: Use ./ci/ on non-travis host (MarcoFalke)
fa31bc3 ci: Remove dependence on travis, use it as fallback env (MarcoFalke)
fa0aac0 ci: Add retry (MarcoFalke)
fafe78f ci: Rename .travis/ to ./ci/ (MarcoFalke)

Pull request description:

  This moves the `.travis` folder to `ci` and removes dependence on travis, so that the test script can be run anywhere.

Top commit has no ACKs.

Tree-SHA512: 4d8c82f3eb4e9e047444b0e0f700485e929a3c4d27fc8777a95b8847f23ed507d2701cc92730198b14d1e753cbb558ffac883da558fc2ec72b8a12c4eaec9000

Author: MarcoFalke

.cirrus.yml

@@ -27,3 +27,27 @@ task:
     - gmake check ${MAKEJOBS} VERBOSE=1
   functional_test_script:
     - ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast
+task:
+  name: "x86_64 Linux  [GOAL: install]  [bionic]  [Using ./ci/ system]"
+  container:
+    image: ubuntu:18.04
+    cpu: 8
+    memory: 8G
+  timeout_in: 60m
+  env:
+    MAKEJOBS: "-j9"
+    RUN_CI_ON_HOST: "1"
+    CCACHE_SIZE: "200M"
+    CCACHE_DIR: "/tmp/ccache_dir"
+  ccache_cache:
+    folder: "/tmp/ccache_dir"
+  depends_built_cache:
+    folder: "/tmp/cirrus-ci-build/depends/built"
+  depends_sdk_cache:
+    folder: "/tmp/cirrus-ci-build/depends/sdk-sources"
+  install_script:
+    - apt-get update
+    - apt-get -y install git bash ccache
+    - ccache --max-size=${CCACHE_SIZE}
+  ci_script:
+    - ./ci/test_run_all.sh

.travis.yml

@@ -40,38 +40,25 @@ stages:
   - extended-lint
 env:
   global:
-    - MAKEJOBS=-j3
-    - RUN_UNIT_TESTS=true
-    - RUN_FUNCTIONAL_TESTS=true
-    - RUN_FUZZ_TESTS=false
-    - DOCKER_NAME_TAG=ubuntu:18.04
-    - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
-    - CCACHE_SIZE=100M
-    - CCACHE_TEMPDIR=/tmp/.ccache-temp
-    - CCACHE_COMPRESS=1
-    - CCACHE_DIR=$HOME/.ccache
-    - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out
-    - SDK_URL=https://bitcoincore.org/depends-sources/sdks
-    - WINEDEBUG=fixme-all
-    - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache"
+    - CI_RETRY_EXE="travis_retry"
     - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved."
 before_install:
-  - set -o errexit; source .travis/test_03_before_install.sh
+  - set -o errexit; source ./ci/test/00_setup_env.sh
+  - set -o errexit; source ./ci/test/03_before_install.sh
 install:
-  - set -o errexit; source .travis/test_04_install.sh
+  - set -o errexit; source ./ci/test/04_install.sh
 before_script:
-  - set -o errexit; source .travis/test_05_before_script.sh
+  - set -o errexit; source ./ci/test/05_before_script.sh
 script:
   - export CONTINUE=1
   - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi  # Likely the depends build took very long
   - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi  # Whitelisted repo (90 minutes build time)
-  - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
+  - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
   - if [ $SECONDS -gt 2000 ]; then export CONTINUE=0; fi  # Likely the build took very long; The tests take about 1000s, so we should abort if we have less than 50*60-1000=2000s left
   - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi  # Whitelisted repo (90 minutes build time)
-  - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
+  - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
 after_script:
   - echo $TRAVIS_COMMIT_RANGE
-  - echo $TRAVIS_COMMIT_LOG
 jobs:
   include:
 
@@ -82,11 +69,11 @@ jobs:
       language: python
       python: '3.5' # Oldest supported version according to doc/dependencies.md
       install:
-        - set -o errexit; source .travis/lint_04_install.sh
+        - set -o errexit; source ./ci/lint/04_install.sh
       before_script:
-        - set -o errexit; source .travis/lint_05_before_script.sh
+        - set -o errexit; source ./ci/lint/05_before_script.sh
       script:
-        - set -o errexit; source .travis/lint_06_script.sh
+        - set -o errexit; source ./ci/lint/06_script.sh
 
     - stage: extended-lint
       name: 'extended lint [runtime >= 60 seconds]'
@@ -95,11 +82,11 @@ jobs:
       language: python
       python: '3.5'
       install:
-        - set -o errexit; source .travis/extended_lint_04_install.sh
+        - set -o errexit; source ./ci/extended_lint/04_install.sh
       before_script:
-        - set -o errexit; source .travis/lint_05_before_script.sh
+        - set -o errexit; source ./ci/lint/05_before_script.sh
       script:
-        - set -o errexit; source .travis/extended_lint_06_script.sh
+        - set -o errexit; source ./ci/extended_lint/06_script.sh
 
     - stage: test
       name: 'ARM  [GOAL: install]  [no unit or functional tests]'

.travis/README.md

@@ -0,0 +1,25 @@
+## ci scripts
+
+This directory contains scripts for each build step in each build stage.
+
+Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the
+[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named
+and numbered according to which stage and lifecycle step it belongs to.
+
+### Running a stage locally
+
+To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
+requires `docker` to be installed. To install all requirements on Ubuntu, run
+
+```
+sudo apt install docker.io ccache bash git
+```
+
+To run the test stage,
+
+```
+./ci/test_run_all.sh
+```
+
+Be aware that the tests will be build and run in-place, so please run at your own risk.
+If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.