Merge bitcoin/bitcoin#28185: ci: Use hard-coded root path for CI containers (bugfix)

fanquake · fanquake · commit 80d70cb6b04b · 2023-08-15T11:17:39.000+01:00
fafa17c ci: Use hard-coded root path for CI containers (MarcoFalke) fa084f5 ci: Only create folders when needed (MarcoFalke) fab2712 ci: Drop BASE_SCRATCH_DIR from LIBCXX_DIR (MarcoFalke) Pull request description: Currently the CI system will fail if the git folder that holds the Bitcoin Core source is moved from one location to another. Fix this by using a single hard-coded root path *inside* the CI system containers. Steps to test: * Run the CI system: `MAKEJOBS="-j$(nproc)" FILE_ENV="./ci/test/00_setup_env_win64.sh" ./ci/test_run_all.sh` * Move the git folder: `pwd && cd .. && mv bitcoin_core_folder_1 bitcoin_core_folder_2 && cd ./bitcoin_core_folder_2 && pwd` * Run the CI system again: (same cmd as above) On master (error): ``` STRIPPROG="x86_64-w64-mingw32-strip" /bin/bash /bitcoin_core_folder_2/ci/scratch/build/bitcoin-x86_64-w64-mingw32/build-aux/install-sh -c -s ./src/qt/bitcoin-qt.exe ./release /bitcoin_core_folder_2/ci/scratch/build/bitcoin-x86_64-w64-mingw32/build-aux/install-sh: ./src/qt/bitcoin-qt.exe does not exist. make: *** [Makefile:1258: bitcoin-25.99.0-win64-setup.exe] Error 1 ``` On this pull: (pass). ACKs for top commit: fanquake: ACK bitcoin/bitcoin@fafa17c - somewhat tested. MSAN changes are the same as what we did for tidy. Tree-SHA512: 2ce693a3773c70fcfca062c2a6f0e5a16b94960b34a6145d10cee1a28f79154829d59d014465ccbb80e1cb9dcd5aa043729cee9afd2c4175b05e9bc945364b79
diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh
@@ -8,10 +8,18 @@ export LC_ALL=C.UTF-8
 
 set -ex
 
-# The root dir.
+# The source root dir, usually from git, usually read-only.
 # The ci system copies this folder.
-BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
-export BASE_ROOT_DIR
+BASE_READ_ONLY_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
+export BASE_READ_ONLY_DIR
+# The destination root dir inside the container.
+if [ -z "${DANGER_RUN_CI_ON_HOST}" ] ; then
+  # This folder only exists on the ci guest and will be a copy of BASE_READ_ONLY_DIR
+  export BASE_ROOT_DIR="/ci_container_base"
+else
+  # This folder is equal to BASE_READ_ONLY_DIR and is read-write
+  export BASE_ROOT_DIR="${BASE_READ_ONLY_DIR}"
+fi
 # The depends dir.
 # This folder exists only on the ci guest, and on the ci host as a volume.
 export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh
@@ -7,7 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CI_IMAGE_NAME_TAG="ubuntu:22.04"
-LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/"
+LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh
@@ -7,7 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CI_IMAGE_NAME_TAG="ubuntu:22.04"
-LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/"
+LIBCXX_DIR="/msan/cxx_build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh
@@ -42,33 +42,35 @@ if [ -n "$PIP_PACKAGES" ]; then
 fi
 
 if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
-  git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.6 "${BASE_SCRATCH_DIR}"/msan/llvm-project
-
-  cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/clang_build/ -DLLVM_ENABLE_PROJECTS="clang" \
-                                                            -DCMAKE_BUILD_TYPE=Release \
-                                                            -DLLVM_TARGETS_TO_BUILD=Native \
-                                                            -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \
-                                                            -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/llvm
-
-  ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ "$MAKEJOBS"
-  ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ install-runtimes
-
-  update-alternatives --install /usr/bin/clang++ clang++ "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang++ 100
-  update-alternatives --install /usr/bin/clang clang "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang 100
-  update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/llvm-symbolizer 100
-
-  cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/cxx_build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \
-                                                          -DCMAKE_BUILD_TYPE=Release \
-                                                          -DLLVM_USE_SANITIZER=MemoryWithOrigins \
-                                                          -DCMAKE_C_COMPILER=clang \
-                                                          -DCMAKE_CXX_COMPILER=clang++ \
-                                                          -DLLVM_TARGETS_TO_BUILD=Native \
-                                                          -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \
-                                                          -DLIBCXX_ENABLE_DEBUG_MODE=ON \
-                                                          -DLIBCXX_ENABLE_ASSERTIONS=ON \
-                                                          -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes
-
-  ninja -C "${BASE_SCRATCH_DIR}"/msan/cxx_build/ "$MAKEJOBS"
+  git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.6 /msan/llvm-project
+
+  cmake -G Ninja -B /msan/clang_build/ \
+    -DLLVM_ENABLE_PROJECTS="clang" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_TARGETS_TO_BUILD=Native \
+    -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \
+    -S /msan/llvm-project/llvm
+
+  ninja -C /msan/clang_build/ "$MAKEJOBS"
+  ninja -C /msan/clang_build/ install-runtimes
+
+  update-alternatives --install /usr/bin/clang++ clang++ /msan/clang_build/bin/clang++ 100
+  update-alternatives --install /usr/bin/clang clang /msan/clang_build/bin/clang 100
+  update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /msan/clang_build/bin/llvm-symbolizer 100
+
+  cmake -G Ninja -B /msan/cxx_build/ \
+    -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_USE_SANITIZER=MemoryWithOrigins \
+    -DCMAKE_C_COMPILER=clang \
+    -DCMAKE_CXX_COMPILER=clang++ \
+    -DLLVM_TARGETS_TO_BUILD=Native \
+    -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \
+    -DLIBCXX_ENABLE_DEBUG_MODE=ON \
+    -DLIBCXX_ENABLE_ASSERTIONS=ON \
+    -S /msan/llvm-project/runtimes
+
+  ninja -C /msan/cxx_build/ "$MAKEJOBS"
 fi
 
 if [[ "${RUN_TIDY}" == "true" ]]; then
diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh
@@ -6,10 +6,6 @@
 
 export LC_ALL=C.UTF-8
 
-# Create folders that are mounted into the docker
-mkdir -p "${CCACHE_DIR}"
-mkdir -p "${PREVIOUS_RELEASES_DIR}"
-
 export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1"
 export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan"
@@ -23,11 +19,11 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append /tmp/env
   echo "Creating $CI_IMAGE_NAME_TAG container to run in"
   DOCKER_BUILDKIT=1 docker build \
-      --file "${BASE_ROOT_DIR}/ci/test_imagefile" \
+      --file "${BASE_READ_ONLY_DIR}/ci/test_imagefile" \
       --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \
       --build-arg "FILE_ENV=${FILE_ENV}" \
       --tag="${CONTAINER_NAME}" \
-      "${BASE_ROOT_DIR}"
+      "${BASE_READ_ONLY_DIR}"
   docker volume create "${CONTAINER_NAME}_ccache" || true
   docker volume create "${CONTAINER_NAME}_depends" || true
   docker volume create "${CONTAINER_NAME}_previous_releases" || true
@@ -41,7 +37,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
 
   # shellcheck disable=SC2086
   CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \
-                  --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
+                  --mount type=bind,src=$BASE_READ_ONLY_DIR,dst=/ro_base,readonly \
                   --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \
                   --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \
                   --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \
@@ -52,6 +48,9 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}"
 else
   echo "Running on host system without docker wrapper"
+  echo "Create missing folders"
+  mkdir -p "${CCACHE_DIR}"
+  mkdir -p "${PREVIOUS_RELEASES_DIR}"
 fi
 
 CI_EXEC () {
