Merge #16739: ci: Pass down $MAKEJOBS to test_runner.py, other improvements

fa27372 ci: Move CCACHE_DIR and test_runner tmp dir into ./ci/scratch/ (MarcoFalke)
fa60583 ci: Pass down $MAKEJOBS to test_runner.py (MarcoFalke)

Pull request description:

  Some changes to the ci system:

  * Pass down MAKEJOBS to the test_runner, instead of falling back to the default of 4. Passing it down avoids OOM on weak machines and allows better use of resources on beefy machines.
  * Move CCACHE_DIR to ./ci/scratch/ subfolder: `ccache` is executed with root permissions inside the docker, so the cache files are created with root as owner. So it might be wise to not put them in the $HOME of the host
  * Use the scratch dir as prefix for the test runner, as opposed to `/tmp/`, which is often a ramdisk and thus leads to OOM on the host when either a lot of tests are run in parallel or when a lot of tests fail and the datadirs are not cleaned.

ACKs for top commit:
  fanquake:
    ACK fa27372
  laanwj:
    ACK fa27372

Tree-SHA512: 67834fbab282051ec81c319d460528b32870507e53df2b8a1ce9a1f3f6a685aaf8eb8ba03f5406918ca4a33adf736e6a4adad7134c54cf3a9e47a26c64a13442

Author: laanwj

.travis.yml

@@ -33,7 +33,7 @@ cache:
   directories:
     - $TRAVIS_BUILD_DIR/depends/built
     - $TRAVIS_BUILD_DIR/depends/sdk-sources
-    - $HOME/.ccache
+    - $TRAVIS_BUILD_DIR/ci/scratch/.ccache
 stages:
   - lint
   - test

ci/test/00_setup_env.sh

@@ -11,7 +11,10 @@ echo "Setting default values in env"
 BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
 export BASE_ROOT_DIR
 
-export MAKEJOBS=${MAKEJOBS:--j3}
+# The number of parallel jobs to pass down to make and test_runner.py
+export MAKEJOBS=${MAKEJOBS:--j4}
+# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
+export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/}
 export HOST=${HOST:-x86_64-unknown-linux-gnu}
 export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
 export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
@@ -21,7 +24,7 @@ export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1$TRAVIS_BUILD_ID}
 export CCACHE_SIZE=${CCACHE_SIZE:-100M}
 export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
 export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
-export CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache}
+export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache}
 export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}}
 export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}

ci/test/04_install.sh

@@ -6,6 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
+mkdir -p "${BASE_SCRATCH_DIR}"
 ccache echo "Creating ccache dir if it didn't already exist"
 
 if [ ! -d ${DIR_QA_ASSETS} ]; then

ci/test/06_script_b.sh

@@ -16,7 +16,7 @@ fi
 
 if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
   BEGIN_FOLD functional-tests
-  DOCKER_EXEC test/functional/test_runner.py --ci --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast
+  DOCKER_EXEC test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast
   END_FOLD
 fi