Merge bitcoin/bitcoin#28138: ci: Keep system env vars as-is (bugfix)

fabc04a ci: Keep system env vars as-is (MarcoFalke)
fa8dcdc ci: Set PATH inside the CI env (MarcoFalke)
fac229a ci: Remove P_CI_DIR and --workdir (MarcoFalke)

Pull request description:

  This fixes a bug where the `$PATH` from the host is used inside the container. This will lead to bugs when the `$PATH` is different. For example on a host of Fedora 38, and a container of `debian:bullseye`.

  This can be tested with the `FILE_ENV=./ci/test/00_setup_env_arm.sh` CI env. On master:

  ```
  Error: crun: executable file `bash` not found in $PATH: No such file or directory: OCI runtime attempted to invoke a command that was not found
  ```

  On this pull:

  (everything passes)

ACKs for top commit:
  TheCharlatan:
    lgtm ACK fabc04a

Tree-SHA512: 51d2affed91624d0e5b43a3ee1e696313f934e05fde6b5a19e8ac4e6666c1e7b667a444bf3de3f77190bcd00e81efb7576196afb0f6b5e788d1a50e7ddb28d7e

Author: fanquake

ci/lint/04_install.sh

@@ -6,6 +6,8 @@
 
 export LC_ALL=C
 
+export PATH=$PWD/ci/retry:$PATH
+
 ${CI_RETRY_EXE} apt-get update
 # Lint dependencies:
 # - curl/xz-utils (to install shellcheck)

ci/test/00_setup_env.sh

@@ -15,9 +15,12 @@ export BASE_ROOT_DIR
 # The depends dir.
 # This folder exists only on the ci guest, and on the ci host as a volume.
 export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
-# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
-# This folder only exists on the ci host.
+# A folder for the ci system to put temporary files (build result, datadirs for tests, ...)
+# This folder only exists on the ci guest.
 export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch}
+# A folder for the ci system to put executables.
+# This folder only exists on the ci guest.
+export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/"
 
 echo "Setting specific values in env"
 if [ -n "${FILE_ENV}" ]; then
@@ -69,5 +72,4 @@ export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}
 export GOAL=${GOAL:-install}
 export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets}
-export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
 export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"}

ci/test/04_install.sh

@@ -22,15 +22,14 @@ if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASa
   CI_CONTAINER_CAP="--cap-add SYS_PTRACE"
 fi
 
-export P_CI_DIR="$PWD"
-export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/"
-
 if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
   # Export all env vars to avoid missing some.
   # Though, exclude those with newlines to avoid parsing problems.
-  python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value and "HOME" not in key]' | tee /tmp/env
+  python3 -c 'import os; [print(f"{key}={value}") for key, value in os.environ.items() if "\n" not in value and "HOME" != key and "PATH" != key and "USER" != key]' | tee /tmp/env
+  # System-dependent env vars must be kept as is. So read them from the container.
+  docker run --rm "${CI_IMAGE_NAME_TAG}" bash -c "env | grep --extended-regexp '^(HOME|PATH|USER)='" | tee --append /tmp/env
   echo "Creating $CI_IMAGE_NAME_TAG container to run in"
-  DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \
+  DOCKER_BUILDKIT=1 docker build \
       --file "${BASE_ROOT_DIR}/ci/test_imagefile" \
       --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \
       --build-arg "FILE_ENV=${FILE_ENV}" \
@@ -53,7 +52,6 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
                   --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \
                   --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \
                   --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \
-                  -w $BASE_ROOT_DIR \
                   --env-file /tmp/env \
                   --name $CONTAINER_NAME \
                   $CONTAINER_NAME)
@@ -64,7 +62,7 @@ else
 fi
 
 CI_EXEC () {
-  $CI_EXEC_CMD_PREFIX bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*"
+  $CI_EXEC_CMD_PREFIX bash -c "export PATH=${BINS_SCRATCH_DIR}:${BASE_ROOT_DIR}/ci/retry:\$PATH && cd \"${BASE_ROOT_DIR}\" && $*"
 }
 export -f CI_EXEC