Skip to content

Commit ca4510c

Browse files
author
MarcoFalke
committed
Merge #13863: travis: move script sections to files in .travis/ subject to shellcheck
4143269 use export LC_ALL=C.UTF-8 (Julian Fleischer) 728c82d make script exit if a command fails (Julian Fleischer) 506890b move remaining travis build steps into individual files (Julian Fleischer) 272306e number .travis/ script according to build lifecycle and add README to explain (Julian Fleischer) 519e273 move lint stage up to resemble travis build ui (Julian Fleischer) 86d34f0 abort script in END_FOLD on non-zero exit code (Julian Fleischer) 4f2f88c move script sections info individual files and comply with shellcheck (Julian Fleischer) Pull request description: This PR is extracted from bitcoin/bitcoin#13816 to make that one easier to review. It follows on bitcoin/bitcoin#13849 and bitcoin/bitcoin#13851 In here the shell script parts from `travis.yml` are extracted into `.travis/before_install.sh`, `.travis/install.sh`, `.travis/before_script.sh`, `.travis/script.sh`, and `.travis/lint.sh`. This has the benefit that `test/lint/lint-shell.sh` will also shellcheck these parts. Also it makes the individual script parts more readable. Tree-SHA512: c497e1687ceb1c1d795de177d3fc35af908bc8e3f781a871afabdecf031e581d4db229290627249e35ef7c09952bc34884e4734ea91d40f57b4a9efb85bba2e3
2 parents 51a73c9 + 4143269 commit ca4510c

10 files changed

+218
-63
lines changed

‎.travis.yml

Lines changed: 22 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ language: minimal
55
cache:
66
ccache: true
77
directories:
8-
- depends/built
9-
- depends/sdk-sources
10-
- $HOME/.ccache
8+
- depends/built
9+
- depends/sdk-sources
10+
- $HOME/.ccache
1111
stages:
1212
- lint
1313
- test
@@ -18,7 +18,6 @@ env:
1818
- RUN_FUNCTIONAL_TESTS=true
1919
- RUN_BENCH=false # Set to true for any one job that has debug enabled, to quickly check bench is not crashing or hitting assertions
2020
- DOCKER_NAME_TAG=ubuntu:18.04
21-
- LC_ALL=C.UTF-8
2221
- BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
2322
- CCACHE_SIZE=100M
2423
- CCACHE_TEMPDIR=/tmp/.ccache-temp
@@ -29,46 +28,31 @@ env:
2928
- WINEDEBUG=fixme-all
3029
- DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache"
3130
before_install:
32-
- export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
33-
- BEGIN_FOLD () { echo ""; CURRENT_FOLD_NAME=$1; echo "travis_fold:start:${CURRENT_FOLD_NAME}"; }
34-
- END_FOLD () { RET=$?; echo "travis_fold:end:${CURRENT_FOLD_NAME}"; return $RET; }
31+
- set -o errexit; source .travis/test_03_before_install.sh
3532
install:
36-
- travis_retry docker pull $DOCKER_NAME_TAG
37-
- env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL)' | tee /tmp/env
38-
- if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN"; fi
39-
- DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
40-
- DOCKER_EXEC () { docker exec $DOCKER_ID bash -c "cd $PWD && $*"; }
41-
- if [ -n "$DPKG_ADD_ARCH" ]; then DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi
42-
- travis_retry DOCKER_EXEC apt-get update
43-
- travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
33+
- set -o errexit; source .travis/test_04_install.sh
4434
before_script:
45-
- DOCKER_EXEC echo \> \$HOME/.bitcoin # Make sure default datadir does not exist and is never read by creating a dummy file
46-
- mkdir -p depends/SDKs depends/sdk-sources
47-
- if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
48-
- if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
49-
- if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\); fi
50-
- if [ -z "$NO_DEPENDS" ]; then DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS; fi
35+
- set -o errexit; source .travis/test_05_before_script.sh
5136
script:
52-
- export TRAVIS_COMMIT_LOG=`git log --format=fuller -1`
53-
- OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
54-
- BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"
55-
- if [ -z "$NO_DEPENDS" ]; then DOCKER_EXEC ccache --max-size=$CCACHE_SIZE; fi
56-
- BEGIN_FOLD autogen; test -n "$CONFIG_SHELL" && DOCKER_EXEC "$CONFIG_SHELL" -c "./autogen.sh" || DOCKER_EXEC ./autogen.sh; END_FOLD
57-
- mkdir build && cd build
58-
- BEGIN_FOLD configure; DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false); END_FOLD
59-
- BEGIN_FOLD distdir; DOCKER_EXEC make distdir VERSION=$HOST; END_FOLD
60-
- cd bitcoin-$HOST
61-
- BEGIN_FOLD configure; DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false); END_FOLD
62-
- BEGIN_FOLD build; DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ); END_FOLD
63-
- if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests; DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1; END_FOLD; fi
64-
- if [ "$RUN_BENCH" = "true" ]; then BEGIN_FOLD bench; DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib $OUTDIR/bin/bench_bitcoin -scaling=0.001 ; END_FOLD; fi
65-
- if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then extended="--extended --exclude feature_pruning,feature_dbcrash"; fi
66-
- if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then BEGIN_FOLD functional-tests; DOCKER_EXEC test/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet --failfast ${extended}; END_FOLD; fi
37+
- set -o errexit; source .travis/test_06_script.sh
6738
after_script:
68-
- echo $TRAVIS_COMMIT_RANGE
69-
- echo $TRAVIS_COMMIT_LOG
39+
- echo $TRAVIS_COMMIT_RANGE
40+
- echo $TRAVIS_COMMIT_LOG
7041
jobs:
7142
include:
43+
# lint stage
44+
- stage: lint
45+
env:
46+
sudo: false
47+
cache: false
48+
language: python
49+
python: '3.6'
50+
install:
51+
- set -o errexit; source .travis/lint_04_install.sh
52+
before_script:
53+
- set -o errexit; source .travis/lint_05_before_script.sh
54+
script:
55+
- set -o errexit; source .travis/lint_06_script.sh
7256
# ARM
7357
- stage: test
7458
env: >-
@@ -149,26 +133,3 @@ jobs:
149133
RUN_FUNCTIONAL_TESTS=false
150134
GOAL="all deploy"
151135
BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror"
152-
- stage: lint
153-
env:
154-
sudo: false
155-
cache: false
156-
language: python
157-
python: '3.6'
158-
install:
159-
- travis_retry pip install flake8
160-
before_script:
161-
- git fetch --unshallow
162-
script:
163-
- if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then test/lint/commit-script-check.sh $TRAVIS_COMMIT_RANGE; fi
164-
- test/lint/git-subtree-check.sh src/crypto/ctaes
165-
- test/lint/git-subtree-check.sh src/secp256k1
166-
- test/lint/git-subtree-check.sh src/univalue
167-
- test/lint/git-subtree-check.sh src/leveldb
168-
- test/lint/check-doc.py
169-
- test/lint/check-rpc-mappings.py .
170-
- test/lint/lint-all.sh
171-
- if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_EVENT_TYPE" = "cron" ]; then
172-
while read LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys &&
173-
travis_wait 50 contrib/verify-commits/verify-commits.py;
174-
fi

‎.travis/README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
## travis build scripts
2+
3+
The `.travis` directory contains scripts for each build step in each build stage.
4+
Currently the travis build defines two stages `lint` and `test`. Each stage has
5+
it's own [lifecycle](https://docs.travis-ci.com/user/customizing-the-build/#the-build-lifecycle).
6+
Every script in here is named and numbered according to which stage and lifecycle
7+
step it belongs to.
8+

‎.travis/lint_04_install.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright (c) 2018 The Bitcoin Core developers
4+
# Distributed under the MIT software license, see the accompanying
5+
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6+
7+
export LC_ALL=C
8+
9+
travis_retry pip install flake8

‎.travis/lint_05_before_script.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright (c) 2018 The Bitcoin Core developers
4+
# Distributed under the MIT software license, see the accompanying
5+
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6+
7+
export LC_ALL=C
8+
9+
git fetch --unshallow

‎.travis/lint_06_script.sh

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright (c) 2018 The Bitcoin Core developers
4+
# Distributed under the MIT software license, see the accompanying
5+
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6+
7+
export LC_ALL=C
8+
9+
if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then
10+
test/lint/commit-script-check.sh $TRAVIS_COMMIT_RANGE
11+
fi
12+
13+
test/lint/git-subtree-check.sh src/crypto/ctaes
14+
test/lint/git-subtree-check.sh src/secp256k1
15+
test/lint/git-subtree-check.sh src/univalue
16+
test/lint/git-subtree-check.sh src/leveldb
17+
test/lint/check-doc.py
18+
test/lint/check-rpc-mappings.py .
19+
test/lint/lint-all.sh
20+
21+
if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_EVENT_TYPE" = "cron" ]; then
22+
while read -r LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys &&
23+
travis_wait 50 contrib/verify-commits/verify-commits.py;
24+
fi

‎.travis/test_03_before_install.sh

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright (c) 2018 The Bitcoin Core developers
4+
# Distributed under the MIT software license, see the accompanying
5+
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6+
7+
export LC_ALL=C.UTF-8
8+
9+
PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
10+
export PATH
11+
12+
BEGIN_FOLD () {
13+
echo ""
14+
CURRENT_FOLD_NAME=$1
15+
echo "travis_fold:start:${CURRENT_FOLD_NAME}"
16+
}
17+
18+
END_FOLD () {
19+
RET=$?
20+
echo "travis_fold:end:${CURRENT_FOLD_NAME}"
21+
if [ $RET != 0 ]; then
22+
echo "${CURRENT_FOLD_NAME} failed with status code ${RET}"
23+
fi
24+
}
25+

‎.travis/test_04_install.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright (c) 2018 The Bitcoin Core developers
4+
# Distributed under the MIT software license, see the accompanying
5+
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6+
7+
export LC_ALL=C.UTF-8
8+
9+
travis_retry docker pull "$DOCKER_NAME_TAG"
10+
env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL)' | tee /tmp/env
11+
if [[ $HOST = *-mingw32 ]]; then
12+
DOCKER_ADMIN="--cap-add SYS_ADMIN"
13+
fi
14+
DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
15+
16+
DOCKER_EXEC () {
17+
docker exec $DOCKER_ID bash -c "cd $PWD && $*"
18+
}
19+
20+
if [ -n "$DPKG_ADD_ARCH" ]; then
21+
DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH"
22+
fi
23+
24+
travis_retry DOCKER_EXEC apt-get update
25+
travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
26+

‎.travis/test_05_before_script.sh

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright (c) 2018 The Bitcoin Core developers
4+
# Distributed under the MIT software license, see the accompanying
5+
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6+
7+
export LC_ALL=C.UTF-8
8+
9+
DOCKER_EXEC echo \> \$HOME/.bitcoin # Make sure default datadir does not exist and is never read by creating a dummy file
10+
11+
mkdir -p depends/SDKs depends/sdk-sources
12+
13+
if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
14+
curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
15+
fi
16+
if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
17+
tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
18+
fi
19+
if [[ $HOST = *-mingw32 ]]; then
20+
DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)
21+
fi
22+
if [ -z "$NO_DEPENDS" ]; then
23+
DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
24+
fi
25+

‎.travis/test_06_script.sh

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright (c) 2018 The Bitcoin Core developers
4+
# Distributed under the MIT software license, see the accompanying
5+
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6+
7+
export LC_ALL=C.UTF-8
8+
9+
TRAVIS_COMMIT_LOG=$(git log --format=fuller -1)
10+
export TRAVIS_COMMIT_LOG
11+
12+
OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
13+
BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"
14+
if [ -z "$NO_DEPENDS" ]; then
15+
DOCKER_EXEC ccache --max-size=$CCACHE_SIZE
16+
fi
17+
18+
BEGIN_FOLD autogen
19+
if [ -n "$CONFIG_SHELL" ]; then
20+
DOCKER_EXEC "$CONFIG_SHELL" -c "./autogen.sh"
21+
else
22+
DOCKER_EXEC ./autogen.sh
23+
fi
24+
END_FOLD
25+
26+
mkdir build
27+
cd build || (echo "could not enter build directory"; exit 1)
28+
29+
BEGIN_FOLD configure
30+
DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
31+
END_FOLD
32+
33+
BEGIN_FOLD distdir
34+
DOCKER_EXEC make distdir VERSION=$HOST
35+
END_FOLD
36+
37+
cd "bitcoin-$HOST" || (echo "could not enter distdir bitcoin-$HOST"; exit 1)
38+
39+
BEGIN_FOLD configure
40+
DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
41+
END_FOLD
42+
43+
BEGIN_FOLD build
44+
DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false )
45+
END_FOLD
46+
47+
if [ "$RUN_TESTS" = "true" ]; then
48+
BEGIN_FOLD unit-tests
49+
DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1
50+
END_FOLD
51+
fi
52+
53+
if [ "$RUN_BENCH" = "true" ]; then
54+
BEGIN_FOLD bench
55+
DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib $OUTDIR/bin/bench_bitcoin -scaling=0.001
56+
END_FOLD
57+
fi
58+
59+
if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then
60+
extended="--extended --exclude feature_pruning,feature_dbcrash"
61+
fi
62+
63+
if [ "$RUN_TESTS" = "true" ]; then
64+
BEGIN_FOLD functional-tests
65+
DOCKER_EXEC test/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet --failfast ${extended}
66+
END_FOLD
67+
fi

‎test/lint/lint-shell-locale.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
66
#
77
# Make sure all shell scripts:
8-
# a.) explicitly opt out of locale dependence using "export LC_ALL=C", or
8+
# a.) explicitly opt out of locale dependence using
9+
# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or
910
# b.) explicitly opt in to locale dependence using the annotation below.
1011

1112
export LC_ALL=C
@@ -16,7 +17,7 @@ for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|univalue
1617
continue
1718
fi
1819
FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*|)$' "${SHELL_SCRIPT}" | head -1)
19-
if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" ]]; then
20+
if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then
2021
echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}"
2122
EXIT_CODE=1
2223
fi

0 commit comments

Comments
 (0)