Merge bitcoin/bitcoin#27737: ci: compile Clang and compiler-rt in msan jobs

5763b23 ci: return to using Ubuntu 22.04 in MSAN jobs (fanquake)
d3cbcbf ci: compile clang and compiler-rt in MSAN jobs (fanquake)
796bd1d ci: use LLVM 16.0.4 in MSAN jobs (fanquake)
883bc9f ci: remove extra CC & CXX from MSAN jobs (fanquake)
2d4f4b8 ci: standardize custom libc++ usage in MSAN jobs (fanquake)

Pull request description:

  This reworks the MSAN CIs, to first compile Clang and compiler-rt (using GCC 12), and then, compile an MSAN instrumented libc++ using the just-built Clang 16. This fixes the `native_fuzz_with_msan` job, working around https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1005341, by not using the Debian provided Clang/LLVM.

  Also included are changes to streamline how we use our "custom libc++", according to upstream: https://releases.llvm.org/16.0.0/projects/libcxx/docs/UsingLibcxx.html#using-a-custom-built-libc, as well as other minor cleanups in the CI configs.

  An example job is currently running in the qa-assets repo: bitcoin-core/qa-assets#129 (https://cirrus-ci.com/task/4632561431871488).

ACKs for top commit:
  dergoegge:
    utACK 5763b23

Tree-SHA512: 4f2a6e0b796bb1830b8346dd1e55eaa86a79037b8b4f16a336c1e29f4fc460acca2ecba076635459370bcbb4009333cb79d27ef1521c1fb5db7599cd5bdf558c

Author: fanquake

.cirrus.yml

@@ -252,11 +252,11 @@ task:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: '[MSan, depends] [lunar]'
+  name: '[MSan, depends] [jammy]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
     docker_arguments:
-      CI_IMAGE_NAME_TAG: ubuntu:lunar
+      CI_IMAGE_NAME_TAG: ubuntu:jammy
       FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV

ci/test/00_setup_env_native_fuzz_with_msan.sh

@@ -6,18 +6,18 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
-LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/"
+export CI_IMAGE_NAME_TAG="ubuntu:22.04"
+LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
-LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
+LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
 
 export CONTAINER_NAME="ci_native_fuzz_msan"
-export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake"
+export PACKAGES="cmake ninja-build"
 # BDB generates false-positives and will be removed in future
-export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+export DEP_OPTS="NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export USE_MEMORY_SANITIZER="true"
 export RUN_UNIT_TESTS="false"
 export RUN_FUNCTIONAL_TESTS="false"

ci/test/00_setup_env_native_msan.sh

@@ -6,18 +6,18 @@
 
 export LC_ALL=C.UTF-8
 
-export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
-LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/"
+export CI_IMAGE_NAME_TAG="ubuntu:22.04"
+LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
-LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
+LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
 
 export CONTAINER_NAME="ci_native_msan"
-export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake"
+export PACKAGES="cmake ninja-build"
 # BDB generates false-positives and will be removed in future
-export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+export DEP_OPTS="NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
-export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export USE_MEMORY_SANITIZER="true"
 export RUN_FUNCTIONAL_TESTS="false"
 export CCACHE_SIZE=250M

ci/test/01_base_install.sh

@@ -42,11 +42,32 @@ if [ -n "$PIP_PACKAGES" ]; then
 fi
 
 if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
-  update-alternatives --install /usr/bin/clang++ clang++ "$(which clang++-16)" 100
-  update-alternatives --install /usr/bin/clang clang "$(which clang-16)" 100
-  git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.1 "${BASE_SCRATCH_DIR}"/msan/llvm-project
-  cmake -B "${BASE_SCRATCH_DIR}"/msan/build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DLIBCXX_ENABLE_DEBUG_MODE=ON -DLIBCXX_ENABLE_ASSERTIONS=ON -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes
-  make -C "${BASE_SCRATCH_DIR}"/msan/build/ "$MAKEJOBS"
+  git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.4 "${BASE_SCRATCH_DIR}"/msan/llvm-project
+
+  cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/clang_build/ -DLLVM_ENABLE_PROJECTS="clang" \
+                                                            -DCMAKE_BUILD_TYPE=Release \
+                                                            -DLLVM_TARGETS_TO_BUILD=X86 \
+                                                            -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \
+                                                            -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/llvm
+
+  ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ "$MAKEJOBS"
+  ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ install-runtimes
+
+  update-alternatives --install /usr/bin/clang++ clang++ "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang++ 100
+  update-alternatives --install /usr/bin/clang clang "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang 100
+
+  cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/cxx_build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \
+                                                          -DCMAKE_BUILD_TYPE=Release \
+                                                          -DLLVM_USE_SANITIZER=MemoryWithOrigins \
+                                                          -DCMAKE_C_COMPILER=clang \
+                                                          -DCMAKE_CXX_COMPILER=clang++ \
+                                                          -DLLVM_TARGETS_TO_BUILD=X86 \
+                                                          -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \
+                                                          -DLIBCXX_ENABLE_DEBUG_MODE=ON \
+                                                          -DLIBCXX_ENABLE_ASSERTIONS=ON \
+                                                          -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes
+
+  ninja -C "${BASE_SCRATCH_DIR}"/msan/cxx_build/ "$MAKEJOBS"
 fi
 
 if [[ "${RUN_TIDY}" == "true" ]]; then