Merge branch 'master' into acts-45.1.1

Author: wdconinc

.github/workflows/build-push.yml

@@ -353,6 +353,7 @@ jobs:
           runner: ubuntu-latest
           PLATFORM: linux/amd64
           target: final
+          SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3"
         - BUILD_IMAGE: eic_
           BUILD_TYPE: default
           BUILDER_IMAGE: debian_stable_base
@@ -362,6 +363,7 @@ jobs:
           runner: ubuntu-latest
           PLATFORM: linux/amd64/v3
           target: final
+          SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3"
         - BUILD_IMAGE: eic_
           BUILD_TYPE: default
           BUILDER_IMAGE: debian_stable_base
@@ -371,6 +373,7 @@ jobs:
           runner: ubuntu-24.04-arm
           PLATFORM: linux/arm64
           target: final
+          SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3"
         - BUILD_IMAGE: eic_
           BUILD_TYPE: default
           BUILDER_IMAGE: debian_stable_base
@@ -380,6 +383,7 @@ jobs:
           runner: ubuntu-latest
           PLATFORM: linux/amd64
           target: final
+          SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests"
         - BUILD_IMAGE: eic_
           BUILD_TYPE: default
           BUILDER_IMAGE: debian_stable_base
@@ -389,6 +393,7 @@ jobs:
           runner: ubuntu-latest
           PLATFORM: linux/amd64/v3
           target: final
+          SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests"
         - BUILD_IMAGE: eic_
           BUILD_TYPE: default
           BUILDER_IMAGE: debian_stable_base
@@ -398,6 +403,7 @@ jobs:
           runner: ubuntu-24.04-arm
           PLATFORM: linux/arm64
           target: final
+          SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests"
         - BUILD_IMAGE: eic_
           BUILD_TYPE: default
           BUILDER_IMAGE: cuda_devel
@@ -407,6 +413,7 @@ jobs:
           runner: ubuntu-latest
           PLATFORM: linux/amd64
           target: builder_concretization_default
+          SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3"
         - BUILD_IMAGE: eic_
           BUILD_TYPE: default
           BUILDER_IMAGE: cuda_devel
@@ -415,7 +422,8 @@ jobs:
           arch: amd64
           runner: ubuntu-latest
           PLATFORM: linux/amd64
-          target: builder_concretization_default
+          target: final
+          SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3"
       fail-fast: false
     steps:
       - name: Free Disk Space (Ubuntu)
@@ -537,6 +545,7 @@ jobs:
             RUNTIME_IMAGE=${{ matrix.RUNTIME_IMAGE }}
             INTERNAL_TAG=${{ env.INTERNAL_TAG }}
             ENV=${{ matrix.ENV }}
+            SPACK_DUPLICATE_ALLOWLIST=${{ matrix.SPACK_DUPLICATE_ALLOWLIST }}
           cache-from: |
             type=registry,ref=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/buildcache:${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }}-${{ matrix.BUILD_TYPE }}-${{ env.GITHUB_REF_POINT_SLUG }}-${{ matrix.arch }}
             type=registry,ref=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/buildcache:${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }}-${{ matrix.BUILD_TYPE }}-${{ env.GITHUB_BASE_REF_SLUG }}-${{ matrix.arch }}

.gitlab-ci.yml

@@ -477,6 +477,16 @@ eic:
                      --build-arg JUGGLER_SHA=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION:-main})
                    }
                    --build-arg ENV=${ENV}
+                   --build-arg SPACK_DUPLICATE_ALLOWLIST=$(
+                     case "${ENV}" in
+                       ci|ci_without_acts|cuda|dbg|jl|prod)
+                         echo "epic|llvm|py-setuptools|py-urllib3" ;;
+                       xl|tf)
+                         echo "epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests" ;;
+                       *)
+                         echo "epic|llvm|py-setuptools|py-urllib3" ;;
+                     esac
+                   )
                    --build-arg jobs=${JOBS}
                    --build-context spack-environment=spack-environment
                    --secret id=mirrors,src=mirrors.yaml

containers/debian/Dockerfile

@@ -237,7 +237,7 @@ if [ -n "${SPACKPACKAGES_CHERRYPICKS}" ] ; then
   done
 fi
 git -C ${SPACKPACKAGES_ROOT} gc --prune=all --aggressive
-spack repo add --scope site "${SPACKPACKAGES_ROOT}/repos/spack_repo/builtin"
+spack repo add --scope spack "${SPACKPACKAGES_ROOT}/repos/spack_repo/builtin"
 EOF
 
 ## Setup build configuration
@@ -252,17 +252,18 @@ declare -A target=(
   ["linux/arm64"]="aarch64"
 )
 target=${target[${TARGETPLATFORM}]}
-spack config --scope site add "packages:all:require:[target=${target}]"
-spack config --scope site add "packages:all:target:[${target}]"
+spack config --scope spack add "packages:all:require:[target=${target}]"
+spack config --scope spack add "packages:all:target:[${target}]"
 spack config blame packages
+mkdir -p $HOME/.spack/  # workaround for Spack not creating config directory automatically in some versions (see https://github.com/spack/spack/issues/51564)
 spack config --scope user add "config:suppress_gpg_warnings:true"
 spack config --scope user add "config:build_jobs:${jobs}"
 spack config --scope user add "config:db_lock_timeout:${jobs}00"
 spack config --scope user add "config:source_cache:/var/cache/spack"
 spack config --scope user add "config:install_tree:root:/opt/software"
 spack config --scope user add "config:ccache:true"
 spack config blame config
-spack compiler find --scope site
+spack compiler find --scope spack
 # Ensure GCC externals have LTO‑safe binutils configured.
 # 
 # 1. LTO issue being worked around:
@@ -296,9 +297,9 @@ EOF
 ## - allow llvm to be buildable for py-numba > py-llvmlite
 RUN <<EOF
 set -e
-spack external find --scope site llvm
-spack external find --scope site --not-buildable gcc
-spack external find --scope site --not-buildable --path /usr/local/cuda/bin cuda
+spack external find --scope spack llvm
+spack external find --scope spack --not-buildable gcc
+spack external find --scope spack --not-buildable --path /usr/local/cuda/bin cuda
 spack config blame packages
 EOF
 
@@ -307,8 +308,8 @@ EOF
 ## - the write-enabled mirror is provided later as a secret mount
 RUN --mount=type=cache,target=/var/cache/spack <<EOF
 set -e
-spack mirror add --scope site --signed spack-${SPACK_VERSION} https://binaries.spack.io/${SPACK_VERSION}
-spack mirror add --scope site --unsigned ghcr-${SPACKPACKAGES_VERSION} oci://ghcr.io/eic/spack-${SPACKPACKAGES_VERSION}
+spack mirror add --scope spack --signed spack-${SPACK_VERSION} https://binaries.spack.io/${SPACK_VERSION}
+spack mirror add --scope spack --unsigned ghcr-${SPACKPACKAGES_VERSION} oci://ghcr.io/eic/spack-${SPACKPACKAGES_VERSION}
 spack mirror list
 EOF
 
@@ -321,7 +322,7 @@ RUN <<EOF
 set -e
 git clone --filter=tree:0 https://github.com/${KEY4HEPSPACK_ORGREPO}.git ${KEY4HEPSPACK_ROOT}
 git -C ${KEY4HEPSPACK_ROOT} checkout ${KEY4HEPSPACK_SHA:-${KEY4HEPSPACK_VERSION}}
-spack repo add --scope site "${KEY4HEPSPACK_ROOT}"
+spack repo add --scope spack "${KEY4HEPSPACK_ROOT}"
 EOF
 
 ## Setup eic-spack (no need for cherry-picks)
@@ -333,5 +334,5 @@ RUN <<EOF
 set -e
 git clone --filter=tree:0 https://github.com/${EICSPACK_ORGREPO}.git ${EICSPACK_ROOT}
 git -C ${EICSPACK_ROOT} checkout ${EICSPACK_SHA:-${EICSPACK_VERSION}}
-spack repo add --scope site "${EICSPACK_ROOT}/spack_repo/eic"
+spack repo add --scope spack "${EICSPACK_ROOT}/spack_repo/eic"
 EOF

containers/eic/Dockerfile

@@ -46,6 +46,7 @@ ARG ENV=xl
 ENV SPACK_ENV=/opt/spack-environment/${ENV}
 ARG SPACK_FLAGS="--backtrace"
 ARG SPACK_INSTALL_FLAGS="--no-check-signature --show-log-on-error --yes-to-all"
+ARG SPACK_DUPLICATE_ALLOWLIST=""
 ENV SPACK_COLOR="always"
 ENV GIT_TERMINAL_PROMPT=0
 
@@ -55,11 +56,19 @@ set -e
 spack env activate --without-view --dir ${SPACK_ENV}
 spack concretize --force
 spack --color=never find --long --no-groups --show-concretized --format "{name}" \
-| uniq -D -f2 | grep -v -w -e "\(epic\|llvm\|py-setuptools\|py-urllib3\)" \
-| tee /tmp/duplicates.txt
-if [ -s /tmp/duplicates.txt ] ; then
-  echo "Duplicate packages found"
-  cat /tmp/duplicates.txt | while read status hash spec ; do
+| uniq -D -f2 \
+| tee /tmp/duplicates_all.txt
+if [ -n "${SPACK_DUPLICATE_ALLOWLIST}" ] ; then
+  grep -Evw "(${SPACK_DUPLICATE_ALLOWLIST})" /tmp/duplicates_all.txt \
+  | tee /tmp/duplicates_disallowed.txt
+else
+  cat /tmp/duplicates_all.txt \
+  | tee /tmp/duplicates_disallowed.txt
+fi
+if [ -s /tmp/duplicates_disallowed.txt ] ; then
+  echo "Duplicate packages found:"
+  cat /tmp/duplicates_disallowed.txt
+  cat /tmp/duplicates_disallowed.txt | while read status hash spec ; do
     spack --backtrace spec --long /${hash}
     if [ "$prevspec" = "$spec" ] ; then
       spack diff /${hash} /${prevhash}
@@ -192,11 +201,19 @@ if [ -n "${JUGGLER_SHA}" ] ; then
 fi
 spack concretize --force
 spack --color=never find --long --no-groups --show-concretized --format "{name}" \
-| uniq -D -f2 | grep -v -w -e "\(epic\|llvm\|py-setuptools\|py-urllib3\)" \
-| tee /tmp/duplicates.txt
-if [ -s /tmp/duplicates.txt ] ; then
-  echo "Duplicate packages found"
-  cat /tmp/duplicates.txt | while read status hash spec ; do
+| uniq -D -f2 \
+| tee /tmp/duplicates_all.txt
+if [ -n "${SPACK_DUPLICATE_ALLOWLIST}" ] ; then
+  grep -Evw "(${SPACK_DUPLICATE_ALLOWLIST})" /tmp/duplicates_all.txt \
+  | tee /tmp/duplicates_disallowed.txt
+else
+  cat /tmp/duplicates_all.txt \
+  | tee /tmp/duplicates_disallowed.txt
+fi
+if [ -s /tmp/duplicates_disallowed.txt ] ; then
+  echo "Duplicate packages found:"
+  cat /tmp/duplicates_disallowed.txt
+  cat /tmp/duplicates_disallowed.txt | while read status hash spec ; do
     spack --backtrace spec --long /${hash}
     if [ "$prevspec" = "$spec" ] ; then
       spack diff /${hash} /${prevhash}
@@ -359,13 +376,13 @@ RUN ldconfig
 ENV SPACK_DISABLE_LOCAL_CONFIG="true"
 RUN <<EOF
 set -e
-spack config --scope site add "config:install_tree:root:~/spack"
-spack config --scope site add "config:source_cache:~/.spack/cache"
-spack config --scope site add "config:binary_index_root:~/.spack"
-spack config --scope site add "config:environments_root:~/.spack/env"
-spack config --scope site add "config:suppress_gpg_warnings:true"
+spack config --scope spack add "config:install_tree:root:~/spack"
+spack config --scope spack add "config:source_cache:~/.spack/cache"
+spack config --scope spack add "config:binary_index_root:~/.spack"
+spack config --scope spack add "config:environments_root:~/.spack/env"
+spack config --scope spack add "config:suppress_gpg_warnings:true"
 spack config blame config
-spack config --scope site add "upstreams:eic-shell:install_tree:/opt/software"
+spack config --scope spack add "upstreams:eic-shell:install_tree:/opt/software"
 spack config blame upstreams
 EOF
 

docs/spack-environment.md

@@ -149,7 +149,7 @@ Spack versions can include cherry-picks from newer commits:
 
 ```bash
 # From spack.sh
-SPACK_VERSION="v1.0.2"
+SPACK_VERSION="v1.1.0"
 SPACK_CHERRYPICKS=$(cat <<- EOF
 09f75ee426a2e05e0543570821582480ff823ba5
 a462612b64e97fa7dfe461c32c58553fd6ec63c5

spack-environment/packages.yaml

@@ -26,15 +26,19 @@ packages:
     #- any_of: [+ipo, '@:']
     - any_of: [build_system=cmake, '@:']
     - any_of: [build_type=Release, '@:']
+  abseil-cpp:
+    require:
+    # Without constraints on abseil-cpp, we end up with cxxstd=14 branches
+    - cxxstd=17
   acts:
     require:
     - '@45.1.1'
     - cxxstd=20 +dd4hep +edm4hep +examples +fatras +geant4 +json +onnx +podio +python +svg +root +pr4496 +pr4502 +pr4620
     # ACTS requires same compiler as DD4hep since compiler options are reused
     - spec: '%gcc'
-      when: '^dd4hep%gcc'
+      when: '^dd4hep %gcc'
     - spec: '%clang'
-      when: '^dd4hep%clang'
+      when: '^dd4hep %clang'
   actsvg:
     require:
     - '@0.4.56'
@@ -242,7 +246,7 @@ packages:
     - '@1.0.10'
   irt2:
     require:
-    - '@2.1.2'
+    - '@2.1.3'
   iwyu:
     require:
     - '@0.23:'
@@ -279,9 +283,9 @@ packages:
     require:
     - '@6.5.5'
     - +python
-  llvm:
-    require:
-    - any_of: [~gold, '@:']
+  #llvm:
+    #require:
+    #- any_of: [~gold, '@:']
     # FIXME any ipo requirement breaks llvm external reuse
     #- any_of: [~ipo, '@:']
   madx:
@@ -453,7 +457,7 @@ packages:
     - '@0.61.0:'
   py-numpy:
     require:
-    - '@2.0.2:'
+    - '@2.2'  # pin to avoid duplicates in eic_tf
   py-onnx:
     require:
     - '@1.17.0'
@@ -579,6 +583,9 @@ packages:
   strace:
     require:
     - -mpers
+  swig:
+    require:
+    - '@4.1:'  # constrain to avoid duplicates in eic_tf
   tbb:
     require:
     - 'intel-tbb'

spack-environment/tf/epic/spack.yaml

@@ -3,6 +3,8 @@ spack:
   - ../../concretizer.yaml
   - ../../packages.yaml
   - ../../view.yaml
+  concretizer:
+    unify: when_possible  # included environment: py-numba needs a different llvm than the one provided as external
   include_concrete:
   - /opt/spack-environment/tf
   specs:

spack-environment/tf/spack.yaml

@@ -3,6 +3,8 @@ spack:
   - ../concretizer.yaml
   - ../packages.yaml
   - ../view.yaml
+  concretizer:
+    unify: when_possible  # py-numba needs a different llvm than the one provided as external
   specs:
   - edm4hep
   - onnx

spack.sh

@@ -3,7 +3,7 @@ SPACK_ORGREPO="spack/spack"
 
 ## Spack github version, e.g. v0.18.1 or commit hash
 ## note: nightly builds will use e.g. releases/v1.0
-SPACK_VERSION="v1.0.4"
+SPACK_VERSION="v1.1.0"
 
 ## Space-separated list of spack cherry-picks
 read -r -d '' SPACK_CHERRYPICKS <<- \