Skip to content

Commit 3a63ef5

Browse files
fanquakefanquake
committed
Merge bitcoin/bitcoin#27616: ci: Remove CI_EXEC bloat
fa01c3c ci: Remove CI_EXEC bloat (MarcoFalke) fa8a428 move-only: Move almost all CI_EXEC code to 06_script_b.sh (MarcoFalke) Pull request description: `CI_EXEC` has many issues: * It is roughly equivalent to `bash -c "$*"`, meaning that the full command will be treated as a single string, ignoring tokens. * It must be put in front of (almost) every command, making it easy to forget, hard to debug the resulting failure, and the code verbose. Fix all issues by removing it almost completely. ACKs for top commit: TheCharlatan: ACK fa01c3c Tree-SHA512: 4a65d61f5c35ca945d31f270dba3e96305fd83333a7713f0452c67f02a78e1901113e9f18d21e1dc016403c0033eb32038a9308d0a0ded7ee6b970d18381a1c2
2 parents 29c36f0 + fa01c3c commit 3a63ef5

File tree

5 files changed

+118
-135
lines changed

5 files changed

+118
-135
lines changed

‎ci/test/04_install.sh‎

Lines changed: 0 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -74,39 +74,3 @@ CI_EXEC rsync --archive --stats --human-readable /ro_base/ "${BASE_ROOT_DIR}" ||
7474
CI_EXEC git config --global --add safe.directory \"*\"
7575

7676
CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}"
77-
78-
if [ "$CI_OS_NAME" == "macos" ]; then
79-
top -l 1 -s 0 | awk ' /PhysMem/ {print}'
80-
echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
81-
else
82-
CI_EXEC free -m -h
83-
CI_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
84-
CI_EXEC echo "$(lscpu | grep Endian)"
85-
fi
86-
CI_EXEC echo "Free disk space:"
87-
CI_EXEC df -h
88-
89-
if [ "$RUN_FUZZ_TESTS" = "true" ]; then
90-
export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
91-
if [ ! -d "$DIR_FUZZ_IN" ]; then
92-
CI_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}"
93-
fi
94-
elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
95-
export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/
96-
if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then
97-
CI_EXEC mkdir -p "$DIR_UNIT_TEST_DATA"
98-
CI_EXEC curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json"
99-
fi
100-
fi
101-
102-
CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
103-
104-
if [ "$USE_BUSY_BOX" = "true" ]; then
105-
echo "Setup to use BusyBox utils"
106-
# tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)
107-
# ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed)
108-
# shellcheck disable=SC1010
109-
CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \)\; do ln -s \$\(command -v busybox\) "${BINS_SCRATCH_DIR}/\$util"\; done
110-
# Print BusyBox version
111-
CI_EXEC patch --help
112-
fi

‎ci/test/05_before_script.sh‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎ci/test/06_script_a.sh‎

Lines changed: 0 additions & 68 deletions
This file was deleted.

‎ci/test/06_script_b.sh‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,124 @@ export LC_ALL=C.UTF-8
88

99
set -ex
1010

11+
if [ "$CI_OS_NAME" == "macos" ]; then
12+
top -l 1 -s 0 | awk ' /PhysMem/ {print}'
13+
echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
14+
else
15+
free -m -h
16+
echo "Number of CPUs \(nproc\):" \$\(nproc\)
17+
lscpu | grep Endian
18+
fi
19+
echo "Free disk space:"
20+
df -h
21+
22+
if [ "$RUN_FUZZ_TESTS" = "true" ]; then
23+
export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
24+
if [ ! -d "$DIR_FUZZ_IN" ]; then
25+
git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}"
26+
fi
27+
elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
28+
export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/
29+
if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then
30+
mkdir -p "$DIR_UNIT_TEST_DATA"
31+
curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json"
32+
fi
33+
fi
34+
35+
mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
36+
37+
if [ "$USE_BUSY_BOX" = "true" ]; then
38+
echo "Setup to use BusyBox utils"
39+
# tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)
40+
# ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed)
41+
for util in $(busybox --list | grep -v "^ar$" | grep -v "^tar$" ); do ln -s "$(command -v busybox)" "${BINS_SCRATCH_DIR}/$util"; done
42+
# Print BusyBox version
43+
patch --help
44+
fi
45+
46+
# Make sure default datadir does not exist and is never read by creating a dummy file
47+
if [ "$CI_OS_NAME" == "macos" ]; then
48+
echo > "${HOME}/Library/Application Support/Bitcoin"
49+
else
50+
echo > "${HOME}/.bitcoin"
51+
fi
52+
53+
if [ -z "$NO_DEPENDS" ]; then
54+
if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then
55+
# CentOS has problems building the depends if the config shell is not explicitly set
56+
# (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to
57+
# an error as the first command is executed)
58+
SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/dash"
59+
else
60+
SHELL_OPTS="CONFIG_SHELL="
61+
fi
62+
bash -c "$SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS LOG=1"
63+
fi
64+
if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then
65+
test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR"
66+
fi
67+
68+
BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking"
69+
if [ -z "$NO_DEPENDS" ]; then
70+
BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site"
71+
fi
72+
if [ -z "$NO_WERROR" ]; then
73+
BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror"
74+
fi
75+
76+
ccache --zero-stats --max-size="${CCACHE_SIZE}"
77+
PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats"
78+
79+
if [ -n "$ANDROID_TOOLS_URL" ]; then
80+
make distclean || true
81+
./autogen.sh
82+
bash -c "./configure $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false)
83+
make "${MAKEJOBS}" && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk
84+
bash -c "${PRINT_CCACHE_STATISTICS}"
85+
exit 0
86+
fi
87+
88+
BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR"
89+
90+
if [ -n "$CONFIG_SHELL" ]; then
91+
"$CONFIG_SHELL" -c "./autogen.sh"
92+
else
93+
./autogen.sh
94+
fi
95+
96+
mkdir -p "${BASE_BUILD_DIR}"
97+
cd "${BASE_BUILD_DIR}"
98+
99+
bash -c "${BASE_ROOT_DIR}/configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false)
100+
101+
make distdir VERSION="$HOST"
102+
103+
cd "${BASE_BUILD_DIR}/bitcoin-$HOST"
104+
105+
bash -c "./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG" || ( (cat config.log) && false)
106+
107+
set -o errtrace
108+
trap 'bash -c "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR
109+
110+
if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
111+
# MemorySanitizer (MSAN) does not support tracking memory initialization done by
112+
# using the Linux getrandom syscall. Avoid using getrandom by undefining
113+
# HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for
114+
# details.
115+
grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h
116+
fi
117+
118+
if [[ "${RUN_TIDY}" == "true" ]]; then
119+
MAYBE_BEAR="bear --config src/.bear-tidy-config"
120+
MAYBE_TOKEN="--"
121+
fi
122+
123+
bash -c "${MAYBE_BEAR} ${MAYBE_TOKEN} make $MAKEJOBS $GOAL" || ( echo "Build failure. Verbose build follows." && make "$GOAL" V=1 ; false )
124+
125+
bash -c "${PRINT_CCACHE_STATISTICS}"
126+
du -sh "${DEPENDS_DIR}"/*/
127+
du -sh "${PREVIOUS_RELEASES_DIR}"
128+
11129
if [[ $HOST = *-mingw32 ]]; then
12130
# Generate all binaries, so that they can be wrapped
13131
make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1

‎ci/test_run_all.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ export LC_ALL=C.UTF-8
88

99
set -o errexit; source ./ci/test/00_setup_env.sh
1010
set -o errexit; source ./ci/test/04_install.sh
11-
set -o errexit; source ./ci/test/05_before_script.sh
12-
set -o errexit; source ./ci/test/06_script_a.sh
1311
set -o errexit
1412
CI_EXEC "${BASE_ROOT_DIR}/ci/test/06_script_b.sh"
1513

0 commit comments

Comments
 (0)