Merge bitcoin/bitcoin#33099: ci: allow for any libc++ intrumentation & use it for TSAN

7aa5b67 ci: remove DEBUG_LOCKORDER from TSAN job (fanquake)
b09af2c ci: instrument libc++ in TSAN job (fanquake)
6653caf ci: allow libc++ instrumentation other than msan (fanquake)

Pull request description:

  Allow for instrumenting libc++ with a sanitizer other than MemoryWithOrigins.

  Would also close #33087, as with the extra instrumentation, the issue from bitcoin/bitcoin#32862 (comment) is avoided (also see bitcoin/bitcoin#33081), and we can drop `DEBUG_LOCKORDER`.

ACKs for top commit:
  maflcko:
    re-ACK 7aa5b67 🦀
  dergoegge:
    utACK 7aa5b67

Tree-SHA512: 95f123e37da5e81d571244e4b1cd7658107676f1ea763ff16e5b69f4dfadb88236a577bb2ee52230ff542872c1da151c88fc50aba0f32540e765080120cec55e

Author: fanquake

.cirrus.yml

@@ -143,7 +143,7 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_native_previous_releases.sh"
 
 task:
-  name: 'TSan, depends, gui'
+  name: 'TSan, depends, no gui'
   << : *GLOBAL_TASK_TEMPLATE
   persistent_worker:
     labels:

ci/test/00_setup_env_native_fuzz_with_msan.sh

@@ -7,7 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04"
-LIBCXX_DIR="/msan/cxx_build/"
+LIBCXX_DIR="/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
@@ -25,7 +25,7 @@ export BITCOIN_CONFIG="\
  -DSANITIZERS=fuzzer,memory \
  -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE -U_FORTIFY_SOURCE' \
 "
-export USE_MEMORY_SANITIZER="true"
+export USE_INSTRUMENTED_LIBCPP="MemoryWithOrigins"
 export RUN_UNIT_TESTS="false"
 export RUN_FUNCTIONAL_TESTS="false"
 export RUN_FUZZ_TESTS=true

ci/test/00_setup_env_native_msan.sh

@@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8
 
 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04"
 export APT_LLVM_V="20"
-LIBCXX_DIR="/msan/cxx_build/"
+LIBCXX_DIR="/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
@@ -27,4 +27,4 @@ export BITCOIN_CONFIG="\
  -DSANITIZERS=memory \
  -DAPPEND_CPPFLAGS='-U_FORTIFY_SOURCE' \
 "
-export USE_MEMORY_SANITIZER="true"
+export USE_INSTRUMENTED_LIBCPP="MemoryWithOrigins"

ci/test/00_setup_env_native_tsan.sh

@@ -9,9 +9,12 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_native_tsan
 export CI_IMAGE_NAME_TAG="mirror.gcr.io/ubuntu:24.04"
 export APT_LLVM_V="20"
-export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libc++abi-${APT_LLVM_V}-dev libc++-${APT_LLVM_V}-dev python3-zmq"
-export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'"
+LIBCXX_DIR="/cxx_build/"
+LIBCXX_FLAGS="-fsanitize=thread -nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
+export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} llvm-${APT_LLVM_V}-dev libclang-${APT_LLVM_V}-dev libclang-rt-${APT_LLVM_V}-dev python3-zmq"
+export DEP_OPTS="CC=clang CXX=clang++ CXXFLAGS='${LIBCXX_FLAGS}' NO_QT=1"
 export GOAL="install"
 export CI_LIMIT_STACK_SIZE=1
 export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DSANITIZERS=thread \
--DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES'"
+-DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES'"
+export USE_INSTRUMENTED_LIBCPP="Thread"

ci/test/01_base_install.sh

@@ -55,45 +55,45 @@ if [ -n "$PIP_PACKAGES" ]; then
   ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
 fi
 
-if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
+if [[ -n "${USE_INSTRUMENTED_LIBCPP}" ]]; then
   if [ -n "${APT_LLVM_V}" ]; then
-    ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-$( clang --version | sed --silent 's@.*clang version \([0-9.]*\).*@\1@p' )" /msan/llvm-project
+    ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-$( clang --version | sed --silent 's@.*clang version \([0-9.]*\).*@\1@p' )" /llvm-project
   else
-    ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-20.1.8" /msan/llvm-project
+    ${CI_RETRY_EXE} git clone --depth=1 https://github.com/llvm/llvm-project -b "llvmorg-20.1.8" /llvm-project
 
-    cmake -G Ninja -B /msan/clang_build/ \
+    cmake -G Ninja -B /clang_build/ \
       -DLLVM_ENABLE_PROJECTS="clang" \
       -DCMAKE_BUILD_TYPE=Release \
       -DLLVM_TARGETS_TO_BUILD=Native \
       -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \
-      -S /msan/llvm-project/llvm
+      -S /llvm-project/llvm
 
-    ninja -C /msan/clang_build/ "$MAKEJOBS"
-    ninja -C /msan/clang_build/ install-runtimes
+    ninja -C /clang_build/ "$MAKEJOBS"
+    ninja -C /clang_build/ install-runtimes
 
-    update-alternatives --install /usr/bin/clang++ clang++ /msan/clang_build/bin/clang++ 100
-    update-alternatives --install /usr/bin/clang clang /msan/clang_build/bin/clang 100
-    update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100
+    update-alternatives --install /usr/bin/clang++ clang++ /clang_build/bin/clang++ 100
+    update-alternatives --install /usr/bin/clang clang /clang_build/bin/clang 100
+    update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /clang_build/bin/llvm-symbolizer 100
   fi
 
-  cmake -G Ninja -B /msan/cxx_build/ \
+  cmake -G Ninja -B /cxx_build/ \
     -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
     -DCMAKE_BUILD_TYPE=Release \
-    -DLLVM_USE_SANITIZER=MemoryWithOrigins \
+    -DLLVM_USE_SANITIZER="${USE_INSTRUMENTED_LIBCPP}" \
     -DCMAKE_C_COMPILER=clang \
     -DCMAKE_CXX_COMPILER=clang++ \
     -DLLVM_TARGETS_TO_BUILD=Native \
     -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \
     -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \
     -DLIBCXX_ABI_DEFINES="_LIBCPP_ABI_BOUNDED_ITERATORS;_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY;_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING;_LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR;_LIBCPP_ABI_BOUNDED_UNIQUE_PTR" \
     -DLIBCXX_HARDENING_MODE=debug \
-    -S /msan/llvm-project/runtimes
+    -S /llvm-project/runtimes
 
-  ninja -C /msan/cxx_build/ "$MAKEJOBS"
+  ninja -C /cxx_build/ "$MAKEJOBS"
 
   # Clear no longer needed source folder
-  du -sh /msan/llvm-project
-  rm -rf /msan/llvm-project
+  du -sh /llvm-project
+  rm -rf /llvm-project
 fi
 
 if [[ "${RUN_TIDY}" == "true" ]]; then