stacks: rework e4s to enforce a single AMD architecture

.ci/env

@@ -1,2 +1,2 @@
-SPACK_CHECKOUT_VERSION=a429ffcb690fc2098657e6ed14e12ad6af48c928
+SPACK_CHECKOUT_VERSION=06f3ac473591a77e6a623838fc2b164b7c06023b
 SPACK_CHECKOUT_REPO=spack/spack

stacks/e4s/spack.yaml

@@ -74,6 +74,50 @@ spack:
       - cxxstd=17
       - target=x86_64_v3 %gcc
 
+  definitions:
+  - cuda_specs:
+    - amrex
+    - arborx ^kokkos +wrapper
+    - axom
+    - cabana ^kokkos +wrapper +cuda_lambda
+    - caliper
+    - chai ^umpire ~shared
+    - chapel
+    - cusz
+    - dealii
+    - ecp-data-vis-sdk  +adios2 ~ascent +hdf5 +vtkm +zfp ~paraview
+    - exago +mpi +python +raja +hiop  ~ipopt ^hiop@1.0.0 ~sparse +mpi +raja
+    - fftx
+    - flecsi
+    - ginkgo
+    - gromacs
+    - heffte
+    - hpx
+    - hypre
+    - kokkos +wrapper
+    - kokkos-kernels ^kokkos +wrapper
+    - kokkos-fft device_backend=cufft +tests ^kokkos +wrapper
+    - libceed
+    - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard ^cusz
+    - magma
+    - mfem
+    - mgard +serial +openmp +timing +unstructured
+    - omega-h
+    - parsec
+    - petsc
+    - py-torch
+    - raja
+    - slate
+    - slepc
+    - strumpack ~slate
+    - sundials
+    - superlu-dist
+    - tasmanian
+    - trilinos
+    - umpire ~shared
+    - adios2
+    - vtk-m
+    - zfp
   specs:
   # CPU
   - adios
@@ -252,160 +296,91 @@ spack:
   - papi +cuda
   - tau +mpi +cuda +syscall
 
-  # CUDA 80
-  - amrex +cuda cuda_arch=80
-  - arborx +cuda cuda_arch=80 ^kokkos +wrapper
-  - axom +cuda cuda_arch=80
-  - cabana +cuda cuda_arch=80 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=80
-  - caliper +cuda cuda_arch=80
-  - chai +cuda cuda_arch=80 ^umpire ~shared
-  - chapel +cuda cuda_arch=80
-  - cusz +cuda cuda_arch=80
-  - dealii +cuda cuda_arch=80
-  - ecp-data-vis-sdk ~rocm +adios2 ~ascent +hdf5 +vtkm +zfp ~paraview +cuda cuda_arch=80 # +paraview: FAILED: VTK/ThirdParty/vtkm/vtkvtkm/vtk-m/vtkm/filter/flow/CMakeFiles/vtkm_filter_flow.dir/StreamSurface.cxx.o
-  - exago +mpi +python +raja +hiop ~rocm +cuda cuda_arch=80 ~ipopt ^hiop@1.0.0 ~sparse +mpi +raja ~rocm +cuda cuda_arch=80
-  - fftx +cuda cuda_arch=80
-  - flecsi +cuda cuda_arch=80
-  - ginkgo +cuda cuda_arch=80
-  - gromacs +cuda cuda_arch=80
-  - heffte +cuda cuda_arch=80
-  - hpx +cuda cuda_arch=80
-  - hypre +cuda cuda_arch=80
-  - kokkos +wrapper +cuda cuda_arch=80
-  - kokkos-kernels +cuda cuda_arch=80 ^kokkos +wrapper +cuda cuda_arch=80
-  - kokkos-fft device_backend=cufft +tests ^kokkos +wrapper +cuda cuda_arch=80
-  - libceed +cuda cuda_arch=80
-  - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=80 ^cusz +cuda cuda_arch=80
-  - magma +cuda cuda_arch=80
-  - mfem +cuda cuda_arch=80
-  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=80
-  - omega-h +cuda cuda_arch=80
-  - parsec +cuda cuda_arch=80
-  - petsc +cuda cuda_arch=80
-  - py-torch +cuda cuda_arch=80
-  - raja +cuda cuda_arch=80
-  - slate +cuda cuda_arch=80
-  - slepc +cuda cuda_arch=80
-  - strumpack ~slate +cuda cuda_arch=80
-  - sundials +cuda cuda_arch=80
-  - superlu-dist +cuda cuda_arch=80
-  - tasmanian +cuda cuda_arch=80
-  - trilinos +cuda cuda_arch=80
-  - umpire ~shared +cuda cuda_arch=80
-  # INCLUDED IN ECP DAV CUDA
-  - adios2 +cuda cuda_arch=80
-  # - ascent +cuda cuda_arch=80     # ascent: https://github.com/spack/spack/issues/38045
-  # - paraview +cuda cuda_arch=80   # paraview: FAILED: VTK/ThirdParty/vtkm/vtkvtkm/vtk-m/vtkm/filter/flow/CMakeFiles/vtkm_filter_flow.dir/StreamSurface.cxx.o
-  - vtk-m +cuda cuda_arch=80
-  - zfp +cuda cuda_arch=80
-  # --
-  # - lammps +cuda cuda_arch=80     # lammps: needs NVIDIA driver
-  # - lbann +cuda cuda_arch=80      # dihydrogen: hydrogen/device/gpu/CUB.hpp:41:5: error: reference to 'CachingDeviceAllocator' is ambiguous
-  # - upcxx +cuda cuda_arch=80      # upcxx: needs NVIDIA driver
+  - group: "cuda-80"
+    specs:
+    - $cuda_specs
+    override:
+      packages:
+        all:
+          prefer:
+          - ~rocm
+          - +cuda
+          - cuda_arch=80
 
-  # CUDA 90
-  - amrex +cuda cuda_arch=90
-  - arborx +cuda cuda_arch=90 ^kokkos +wrapper
-  - axom +cuda cuda_arch=90
-  - cabana +cuda cuda_arch=90 ^kokkos +wrapper +cuda_lambda +cuda cuda_arch=90
-  - caliper +cuda cuda_arch=90
-  - chai +cuda cuda_arch=90 ^umpire ~shared
-  - chapel +cuda cuda_arch=90
-  - cusz +cuda cuda_arch=90
-  - ecp-data-vis-sdk ~rocm +adios2 ~ascent +hdf5 +vtkm +zfp ~paraview +cuda cuda_arch=90 # +paraview: issue with cuda@12; +ascent: # ascent: https://github.com/spack/spack/issues/38045; +paraview: VTK/ThirdParty/vtkm/vtkvtkm/vtk-m/vtkm/exec/cuda/internal/ExecutionPolicy.h(121): error: namespace "thrust" has no member "sort"
-  - fftx +cuda cuda_arch=90
-  - flecsi +cuda cuda_arch=90
-  - ginkgo +cuda cuda_arch=90
-  - gromacs +cuda cuda_arch=90
-  - heffte +cuda cuda_arch=90
-  - hpx +cuda cuda_arch=90
-  - hypre +cuda cuda_arch=90
-  - kokkos +wrapper +cuda cuda_arch=90
-  - kokkos-kernels +cuda cuda_arch=90 ^kokkos +wrapper +cuda cuda_arch=90
-  - kokkos-fft device_backend=cufft +tests ^kokkos +wrapper +cuda cuda_arch=90
-  - libceed +cuda cuda_arch=90
-  - libpressio +bitgrooming +bzip2 +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +cusz +mgard +cuda cuda_arch=90 ^cusz +cuda cuda_arch=90
-  - magma +cuda cuda_arch=90
-  - mfem +cuda cuda_arch=90
-  - mgard +serial +openmp +timing +unstructured +cuda cuda_arch=90
-  - omega-h +cuda cuda_arch=90
-  - parsec +cuda cuda_arch=90
-  - petsc +cuda cuda_arch=90
-  - py-torch +cuda cuda_arch=90
-  - raja +cuda cuda_arch=90
-  - slate +cuda cuda_arch=90
-  - slepc +cuda cuda_arch=90
-  - strumpack ~slate +cuda cuda_arch=90
-  - sundials +cuda cuda_arch=90
-  - superlu-dist +cuda cuda_arch=90
-  - tasmanian +cuda cuda_arch=90
-  - trilinos +cuda cuda_arch=90
-  - umpire ~shared +cuda cuda_arch=90
-  # INCLUDED IN ECP DAV CUDA
-  - adios2 +cuda cuda_arch=90
-  # - ascent +cuda cuda_arch=90       # ascent: https://github.com/spack/spack/issues/38045
-  # - paraview +cuda cuda_arch=90     # paraview: Cannot satisfy 'cuda@:11' and 'cuda@12.0:
-  - vtk-m +cuda cuda_arch=90
-  - zfp +cuda cuda_arch=90
-  # --
-  # - dealii +cuda cuda_arch=90       # dealii: conflicts with '+cuda ^cuda@12:'; dealii: https://github.com/spack/spack/issues/39532
-  # - lammps +cuda cuda_arch=90       # lammps: needs NVIDIA driver
-  # - lbann +cuda cuda_arch=90       # dihydrogen: hydrogen/device/gpu/CUB.hpp:41:5: error: reference to 'CachingDeviceAllocator' is ambiguous
-  # - upcxx +cuda cuda_arch=90        # upcxx: needs NVIDIA driver
+  - group: "cuda-90"
+    specs:
+      - $cuda_specs
+    override:
+      packages:
+        all:
+          prefer:
+          - ~rocm
+          - +cuda
+          - cuda_arch=80
 
-  # ROCM NOARCH
-  - hpctoolkit +rocm
-  - tau +mpi +rocm +syscall           # tau: has issue with `spack env depfile` build
+  - group: rocm
+    specs:
+    # ROCM NOARCH
+    - hpctoolkit +rocm
+    - tau +mpi +rocm +syscall           # tau: has issue with `spack env depfile` build
 
-  # ROCM 90a
-  - adios2 +kokkos +rocm amdgpu_target=gfx90a
-  - amrex +rocm amdgpu_target=gfx90a
-  - arborx +rocm amdgpu_target=gfx90a
-  - cabana +rocm amdgpu_target=gfx90a
-  - caliper +rocm amdgpu_target=gfx90a
-  - chai +rocm amdgpu_target=gfx90a
-  - ecp-data-vis-sdk ~paraview +vtkm +rocm amdgpu_target=gfx90a # +paraview: CMake Error at VTK/ThirdParty/vtkm/vtkvtkm/vtk-m/CMakeLists.txt:272 (find_package): Could not find a package configuration file provided by "rocthrust" with any of the following names: rocthrustConfig.cmake rocthrust-config.cmake
-  - gasnet +rocm amdgpu_target=gfx90a
-  - ginkgo +rocm amdgpu_target=gfx90a
-  - heffte +rocm amdgpu_target=gfx90a
-  - hpx +rocm amdgpu_target=gfx90a
-  - hypre +rocm amdgpu_target=gfx90a
-  - kokkos +rocm amdgpu_target=gfx90a
-  - lammps +rocm amdgpu_target=gfx90a
-  - legion +rocm amdgpu_target=gfx90a
-  - libceed +rocm amdgpu_target=gfx90a
-  - magma ~cuda +rocm amdgpu_target=gfx90a
-  - mfem +rocm amdgpu_target=gfx90a
-  - raja ~openmp +rocm amdgpu_target=gfx90a
-  - slate +rocm amdgpu_target=gfx90a
-  - strumpack ~slate +rocm amdgpu_target=gfx90a
-  - sundials +rocm amdgpu_target=gfx90a
-  - superlu-dist +rocm amdgpu_target=gfx90a
-  - tasmanian ~openmp +rocm amdgpu_target=gfx90a
-  - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long +rocm amdgpu_target=gfx90a
-  - umpire +rocm amdgpu_target=gfx90a
-  - upcxx +rocm amdgpu_target=gfx90a
-  # INCLUDED IN ECP DAV ROCM
-  # - hdf5
-  # - hdf5-vol-async
-  # - hdf5-vol-cache
-  # - hdf5-vol-log
-  # - libcatalyst
-  # - paraview +rocm amdgpu_target=gfx90a # paraview: CMake Error at VTK/ThirdParty/vtkm/vtkvtkm/vtk-m/CMakeLists.txt:272 (find_package): Could not find a package configuration file provided by "rocthrust" with any of the following names: rocthrustConfig.cmake rocthrust-config.cmake
-  - vtk-m ~openmp +rocm amdgpu_target=gfx90a  # vtk-m: https://github.com/spack/spack/issues/40268
-  # --
-  # - chapel +rocm amdgpu_target=gfx90a         # chapel: lld: error: undefined symbol: llvm.amdgcn.readfirstlane.i32
-  # - exago +mpi +python +raja +hiop +rocm amdgpu_target=gfx90a ~ipopt cxxflags="-Wno-error=non-pod-varargs" ^hiop ~sparse +mpi +raja +rocm amdgpu_target=gfx90a # hiop: CMake Error at cmake/FindHiopHipLibraries.cmake:23 (find_package); By not providing "Findhipfft.cmake" in CMAKE_MODULE_PATH this project has asked CMake to find a package configuration file provided by "hipfft", but CMake did not find one.
-  # - fftx +rocm amdgpu_target=gfx90a           # fftx: https://github.com/spack/spack/issues/47034
-  # - lbann ~cuda +rocm amdgpu_target=gfx90a    # concretize: Cannot select a single "version" for package "hiptt"
-  # - papi +rocm amdgpu_target=gfx90a           # papi: https://github.com/spack/spack/issues/27898
-  # - petsc +rocm amdgpu_target=gfx90a          # petsc: hipblas-6.3.0-brvfa2h47yfpawguzbiac6ijtj3b6bcn/include/hipblas/hipblas.h:40:10: fatal error: 'hipblas-common/hipblas-common.h' file not found
-  # - slepc +rocm amdgpu_target=gfx90a ^petsc +rocm amdgpu_target=gfx90a # slepc: ERROR: Unable to link with PETSc
+    # ROCM 90a
+    - adios2 +kokkos
+    - amrex
+    - arborx
+    - cabana
+    - caliper
+    - chai
+    - ecp-data-vis-sdk ~paraview +vtkm # +paraview: CMake Error at VTK/ThirdParty/vtkm/vtkvtkm/vtk-m/CMakeLists.txt:272 (find_package): Could not find a package configuration file provided by "rocthrust" with any of the following names: rocthrustConfig.cmake rocthrust-config.cmake
+    - gasnet
+    - ginkgo
+    - heffte
+    - hpx
+    - hypre
+    - kokkos
+    - lammps
+    - legion
+    - libceed
+    - magma ~cuda
+    - mfem
+    - raja ~openmp
+    - slate
+    - strumpack ~slate
+    - sundials
+    - superlu-dist
+    - tasmanian ~openmp
+    - trilinos +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext +ifpack ~ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu +nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu ~stokhos +stratimikos +teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
+    - umpire
+    - upcxx
+    # INCLUDED IN ECP DAV ROCM
+    # - hdf5
+    # - hdf5-vol-async
+    # - hdf5-vol-cache
+    # - hdf5-vol-log
+    # - libcatalyst
+    # - paraview # paraview: CMake Error at VTK/ThirdParty/vtkm/vtkvtkm/vtk-m/CMakeLists.txt:272 (find_package): Could not find a package configuration file provided by "rocthrust" with any of the following names: rocthrustConfig.cmake rocthrust-config.cmake
+    - vtk-m ~openmp  # vtk-m: https://github.com/spack/spack/issues/40268
+    # --
+    # - chapel         # chapel: lld: error: undefined symbol: llvm.amdgcn.readfirstlane.i32
+    # - exago +mpi +python +raja +hiop ~ipopt cxxflags="-Wno-error=non-pod-varargs" ^hiop ~sparse +mpi +raja # hiop: CMake Error at cmake/FindHiopHipLibraries.cmake:23 (find_package); By not providing "Findhipfft.cmake" in CMAKE_MODULE_PATH this project has asked CMake to find a package configuration file provided by "hipfft", but CMake did not find one.
+    # - fftx           # fftx: https://github.com/spack/spack/issues/47034
+    # - lbann ~cuda    # concretize: Cannot select a single "version" for package "hiptt"
+    # - papi           # papi: https://github.com/spack/spack/issues/27898
+    # - petsc          # petsc: hipblas-6.3.0-brvfa2h47yfpawguzbiac6ijtj3b6bcn/include/hipblas/hipblas.h:40:10: fatal error: 'hipblas-common/hipblas-common.h' file not found
+    # - slepc ^petsc # slepc: ERROR: Unable to link with PETSc
+
+    # - cp2k +mpi +cuda cuda_arch=80            # cp2k: libxsmm_gemm.c:238: undefined reference to `sgemv_'
+    # - cp2k +mpi +cuda cuda_arch=90            # cp2k: libxsmm_gemm.c:238: undefined reference to `sgemv_'
+    # - cp2k +mpi +rocm amdgpu_target=gfx908    # cp2k: exts/build_dbcsr//Makefile:290: calculate_norms.o] Error 127 (ignored)
+    # - cp2k +mpi    # cp2k: exts/build_dbcsr//Makefile:290: calculate_norms.o] Error 127 (ignored)
+    override:
+      packages:
+        all:
+          prefer:
+            - ~cuda
+            - +rocm
+            - amdgpu_target=gfx90a
 
-  # - cp2k +mpi +cuda cuda_arch=80            # cp2k: libxsmm_gemm.c:238: undefined reference to `sgemv_'
-  # - cp2k +mpi +cuda cuda_arch=90            # cp2k: libxsmm_gemm.c:238: undefined reference to `sgemv_'
-  # - cp2k +mpi +rocm amdgpu_target=gfx908    # cp2k: exts/build_dbcsr//Makefile:290: calculate_norms.o] Error 127 (ignored)
-  # - cp2k +mpi +rocm amdgpu_target=gfx90a    # cp2k: exts/build_dbcsr//Makefile:290: calculate_norms.o] Error 127 (ignored)
 
   ci:
     pipeline-gen: