Skip to content

Commit 883766f

Browse files
fanquakefanquake
committed
Merge bitcoin/bitcoin#27573: ci: Remove CI_EXEC bloat in test/06_script_b.sh
fa1dbd0 ci: Remove CI_EXEC bloat in test/06_script_b.sh (MarcoFalke) fae8de9 ci: Move CI container kill out of 06_script_b.sh (MarcoFalke) fa7d755 ci: Pass full env to CI pod to avoid missing a var (MarcoFalke) Pull request description: `CI_EXEC` has many issues: * It is roughly equivalent to `bash -c "$*"`, meaning that the full command will be treated as a single string, ignoring tokens. * It must be put in front of (almost) every command, making it easy to forget, hard to debug the resulting failure, and the code verbose. Fix all issues in one script by removing it. ACKs for top commit: fanquake: ACK fa1dbd0 - this conflicts with #27125, but that is going to be rebased soon, and this could be merged in the interim. cc TheCharlatan TheCharlatan: ACK fa1dbd0 Tree-SHA512: e5ab5503a05a787f2bc6ca25e71ad3dc166aade57e25d9677e72b1ca4e5fb6045c058dfd55f47ac93f710538e62d57c12cd7eb9d1260c6f55f3c8091908dc70d
2 parents 3777c75 + fa1dbd0 commit 883766f

File tree

4 files changed

+78
-73
lines changed

4 files changed

+78
-73
lines changed

‎ci/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ testing compared to other parts of the codebase. If you want to keep the work tr
1414
system in a virtual machine with a Linux operating system of your choice.
1515

1616
To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
17-
requires `docker` to be installed. To install all requirements on Ubuntu, run
17+
requires `bash`, `docker`, and `python3` to be installed. To install all requirements on Ubuntu, run
1818

1919
```
20-
sudo apt install docker.io bash
20+
sudo apt install bash docker.io python3
2121
```
2222

2323
To run the default test stage,

‎ci/test/04_install.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=
1818
export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
1919
export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan"
2020
export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
21-
env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env
2221
if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
2322
CI_CONTAINER_CAP="--cap-add SYS_PTRACE"
2423
fi
@@ -27,6 +26,9 @@ export P_CI_DIR="$PWD"
2726
export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/"
2827

2928
if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
29+
# Export all env vars to avoid missing some.
30+
# Though, exclude those with newlines to avoid parsing problems.
31+
python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value]' | tee /tmp/env
3032
echo "Creating $CI_IMAGE_NAME_TAG container to run in"
3133
DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \
3234
--file "${BASE_ROOT_DIR}/ci/test_imagefile" \

‎ci/test/06_script_b.sh

Lines changed: 66 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -6,101 +6,98 @@
66

77
export LC_ALL=C.UTF-8
88

9+
set -ex
10+
911
if [[ $HOST = *-mingw32 ]]; then
1012
# Generate all binaries, so that they can be wrapped
11-
CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
12-
CI_EXEC make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1
13-
CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
13+
make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
14+
make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1
15+
"${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
1416
fi
1517

1618
if [ -n "$QEMU_USER_CMD" ]; then
1719
# Generate all binaries, so that they can be wrapped
18-
CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
19-
CI_EXEC make "$MAKEJOBS" -C src minisketch/test VERBOSE=1
20-
CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh"
20+
make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1
21+
make "$MAKEJOBS" -C src minisketch/test VERBOSE=1
22+
"${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh"
2123
fi
2224

2325
if [ -n "$USE_VALGRIND" ]; then
24-
CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
26+
"${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
2527
fi
2628

2729
if [ "$RUN_UNIT_TESTS" = "true" ]; then
28-
CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" make "$MAKEJOBS" check VERBOSE=1
30+
bash -c "${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib make $MAKEJOBS check VERBOSE=1"
2931
fi
3032

3133
if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then
32-
CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}/bin/test_bitcoin" --catch_system_errors=no -l test_suite
34+
bash -c "${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib ${BASE_OUTDIR}/bin/test_bitcoin --catch_system_errors=no -l test_suite"
3335
fi
3436

3537
if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
36-
CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast
38+
bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix ${BASE_SCRATCH_DIR}/test_runner/ --ansi --combinedlogslen=99999999 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast"
3739
fi
3840

3941
if [ "${RUN_TIDY}" = "true" ]; then
4042
set -eo pipefail
41-
export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/"
42-
( CI_EXEC run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error"
43-
export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/"
44-
CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\
45-
" src/common/args.cpp"\
46-
" src/common/config.cpp"\
47-
" src/common/init.cpp"\
48-
" src/common/url.cpp"\
49-
" src/compat"\
50-
" src/dbwrapper.cpp"\
51-
" src/init"\
52-
" src/kernel"\
53-
" src/node/chainstate.cpp"\
54-
" src/node/chainstatemanager_args.cpp"\
55-
" src/node/mempool_args.cpp"\
56-
" src/node/minisketchwrapper.cpp"\
57-
" src/node/utxo_snapshot.cpp"\
58-
" src/node/validation_cache_args.cpp"\
59-
" src/policy/feerate.cpp"\
60-
" src/policy/packages.cpp"\
61-
" src/policy/settings.cpp"\
62-
" src/primitives/transaction.cpp"\
63-
" src/random.cpp"\
64-
" src/rpc/fees.cpp"\
65-
" src/rpc/signmessage.cpp"\
66-
" src/test/fuzz/string.cpp"\
67-
" src/test/fuzz/txorphan.cpp"\
68-
" src/test/fuzz/util/"\
69-
" src/test/util/coins.cpp"\
70-
" src/uint256.cpp"\
71-
" src/util/bip32.cpp"\
72-
" src/util/bytevectorhash.cpp"\
73-
" src/util/check.cpp"\
74-
" src/util/error.cpp"\
75-
" src/util/exception.cpp"\
76-
" src/util/getuniquepath.cpp"\
77-
" src/util/hasher.cpp"\
78-
" src/util/message.cpp"\
79-
" src/util/moneystr.cpp"\
80-
" src/util/serfloat.cpp"\
81-
" src/util/spanparsing.cpp"\
82-
" src/util/strencodings.cpp"\
83-
" src/util/string.cpp"\
84-
" src/util/syserror.cpp"\
85-
" src/util/threadinterrupt.cpp"\
86-
" src/zmq"\
87-
" -p . ${MAKEJOBS}"\
88-
" -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp"\
89-
" |& tee /tmp/iwyu_ci.out"
90-
export P_CI_DIR="${BASE_ROOT_DIR}/src"
91-
CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/fix_includes.py --nosafe_headers < /tmp/iwyu_ci.out"
92-
CI_EXEC "git --no-pager diff"
43+
cd "${BASE_BUILD_DIR}/bitcoin-$HOST/src/"
44+
( run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error"
45+
cd "${BASE_BUILD_DIR}/bitcoin-$HOST/"
46+
python3 "${DIR_IWYU}/include-what-you-use/iwyu_tool.py" \
47+
src/common/args.cpp \
48+
src/common/config.cpp \
49+
src/common/init.cpp \
50+
src/common/url.cpp \
51+
src/compat \
52+
src/dbwrapper.cpp \
53+
src/init \
54+
src/kernel \
55+
src/node/chainstate.cpp \
56+
src/node/chainstatemanager_args.cpp \
57+
src/node/mempool_args.cpp \
58+
src/node/minisketchwrapper.cpp \
59+
src/node/utxo_snapshot.cpp \
60+
src/node/validation_cache_args.cpp \
61+
src/policy/feerate.cpp \
62+
src/policy/packages.cpp \
63+
src/policy/settings.cpp \
64+
src/primitives/transaction.cpp \
65+
src/random.cpp \
66+
src/rpc/fees.cpp \
67+
src/rpc/signmessage.cpp \
68+
src/test/fuzz/string.cpp \
69+
src/test/fuzz/txorphan.cpp \
70+
src/test/fuzz/util \
71+
src/test/util/coins.cpp \
72+
src/uint256.cpp \
73+
src/util/bip32.cpp \
74+
src/util/bytevectorhash.cpp \
75+
src/util/check.cpp \
76+
src/util/error.cpp \
77+
src/util/exception.cpp \
78+
src/util/getuniquepath.cpp \
79+
src/util/hasher.cpp \
80+
src/util/message.cpp \
81+
src/util/moneystr.cpp \
82+
src/util/serfloat.cpp \
83+
src/util/spanparsing.cpp \
84+
src/util/strencodings.cpp \
85+
src/util/string.cpp \
86+
src/util/syserror.cpp \
87+
src/util/threadinterrupt.cpp \
88+
src/zmq \
89+
-p . "${MAKEJOBS}" \
90+
-- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" \
91+
2>&1 | tee /tmp/iwyu_ci.out
92+
cd "${BASE_ROOT_DIR}/src"
93+
python3 "${DIR_IWYU}/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out
94+
git --no-pager diff
9395
fi
9496

9597
if [ "$RUN_SECURITY_TESTS" = "true" ]; then
96-
CI_EXEC make test-security-check
98+
make test-security-check
9799
fi
98100

99101
if [ "$RUN_FUZZ_TESTS" = "true" ]; then
100-
CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}"
101-
fi
102-
103-
if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
104-
echo "Stop and remove CI container by ID"
105-
docker container kill "${CI_CONTAINER_ID}"
102+
bash -c "LD_LIBRARY_PATH=${DEPENDS_DIR}/${HOST}/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN}"
106103
fi

‎ci/test_run_all.sh

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,10 @@ set -o errexit; source ./ci/test/00_setup_env.sh
1010
set -o errexit; source ./ci/test/04_install.sh
1111
set -o errexit; source ./ci/test/05_before_script.sh
1212
set -o errexit; source ./ci/test/06_script_a.sh
13-
set -o errexit; source ./ci/test/06_script_b.sh
13+
set -o errexit
14+
CI_EXEC "${BASE_ROOT_DIR}/ci/test/06_script_b.sh"
15+
16+
if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
17+
echo "Stop and remove CI container by ID"
18+
docker container kill "${CI_CONTAINER_ID}"
19+
fi

0 commit comments

Comments
 (0)