Unpin or update many packages (mostly Python) in configs/common/packages.yaml, fix S4 site config (#1384)

In preparation for spack-stack-1.9.0, this PR unpins or updates several packages in configs/common/packages.yaml (mostly Python packages). Most notably, py-shapely and py-numpy are updated.

The py-numpy update may require bug fixes with the Intel classic compiler that @DavidHuber-NOAA worked on and that are currently under review in spack develop (see #1276).

Included is an update of the S4 site config, which had several flaws that prevented building and testing this PR.

Author: climbfuji

configs/common/packages.yaml

@@ -26,7 +26,16 @@ packages:
   cairo:
     require: '+pic'
   cdo:
-    require: '@2.3.0 ~openmp'
+    require:
+    - any_of: ['@2.3.0 ~openmp']
+      when: '%intel@2021:'
+      message: '2.3.0 is the last version to use C++17, use with Intel Classic'
+    - any_of: ['@2.3.0 ~openmp']
+      when: '%apple-clang@:14'
+      message: '2.3.0 is the last version to use C++17, use with Apple Clang 14 and earlier'
+    - any_of: ['@2.0.5 ~openmp']
+      when: '%intel@:2020'
+      message: '2.0.5 is the last version to use C++14, use with Intel Classic 2020 and earlier'
   cmake:
     version: ['3.27.9']
     require: '+ownlibs'
@@ -142,11 +151,6 @@ packages:
   mapl:
     require: '@2.46.3 ~shared ~f2py'
     variants: '+pflogger'
-  # To avoid duplicate packages being built
-  meson:
-    require:
-    - '@1.5.1:'
-  # If making changes here, also check the Discover site configs and the CI workflows
   met:
     require: '@11.1.1 +python +grib2'
   metplus:
@@ -156,7 +160,7 @@ packages:
   mpich:
     require: '~hwloc'
   nco:
-    require: '@5.1.6 ~doc'
+    require: '@5.2.4 ~doc'
   # ncview - when adding information here, also check Orion
   # and Discover site configs
   nemsio:
@@ -202,9 +206,6 @@ packages:
     require: '~tiff'
   python:
     require: '@3.11.7'
-  py-attrs:
-    # https://github.com/JCSDA/spack-stack/issues/740
-    require: '@21.4.0'
   py-cartopy:
     require: '+plotting'
   # Avoid full rust dependency: version 43+ requires py-maturin
@@ -226,42 +227,23 @@ packages:
     require: '@3.8.0'
   py-h5py:
     require: '~mpi'
-  ## To avoid duplicate packages
-  #py-jinja2:
-  #  require: '@3.1.2'
   # https://github.com/JCSDA/spack-stack/issues/1276
   py-matplotlib:
      require: '@3.7.4'
-  py-meson-python:
-     require: '@0.15.0'
-  # Comment out for now until build problems are solved
-  # https://github.com/jcsda/spack-stack/issues/522
-  # see also ewok-env virtual package and container
-  # README.md
-  #py-mysql-connector-python:
-  #  require: '@8.0.32'
+  # Variant ~mpi not working for latest py-netcdf4
+  # https://github.com/spack/spack/issues/47652
   py-netcdf4:
-    require: '@1.5.8 ~mpi'
-  # py-numpy@1.26 causes many build problems with older Python packages
-  # also check Nautilus site config when making changes here
-  # https://github.com/JCSDA/spack-stack/issues/1276
-  py-numpy:
-    require:
-    - '@:1.23.5'
+    require: '@1.7.1 +mpi'
   py-pandas:
     require: '+excel'
-  py-pybind11:
-    require: '@2.11.0'
   # To avoid duplicate packages
   py-ruamel-yaml:
     require: '@0.17.16'
   # Pin the py-setuptools version to avoid duplicate Python packages
   py-setuptools:
-    require: '@63.4.3'
+    require: '@69'
   py-setuptools-rust:
     require: '+rust_bootstrap'
-  py-shapely:
-    require: '@1.8.0'
   py-torch:
     require: '+custom-protobuf ~mkldnn'
   # To avoid duplicate packages
@@ -297,4 +279,4 @@ packages:
   wrf-io:
     require: '@1.2.0'
   zstd:
-    require: '@1.5.2 +programs'
+    require: '@1.5.6 +programs'

configs/sites/tier1/aws-pcluster/packages_intel.yaml

@@ -31,5 +31,5 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH

configs/sites/tier1/derecho/packages_intel.yaml

@@ -21,4 +21,3 @@ packages:
   # Newer versions of patchelf (tested 0.18.0) don't build with Intel on Derecho
   patchelf:
     version:: ['0.17.2']
-

configs/sites/tier1/gaea-c5/packages.yaml

@@ -35,7 +35,7 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH
 
 ### All other external packages listed alphabetically

configs/sites/tier1/gaea-c6/packages.yaml

@@ -34,7 +34,7 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH
 
 ### All other external packages listed alphabetically

configs/sites/tier1/hera/packages_intel.yaml

@@ -30,7 +30,7 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH
   zlib-ng:
     require:

configs/sites/tier1/hercules/packages_intel.yaml

@@ -30,4 +30,4 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'

configs/sites/tier1/nautilus/packages_gcc.yaml

@@ -31,4 +31,4 @@ packages:
     - '@1.2.1 +mkl'
   py-numpy:
     require::
-    - '@:1.23 ^intel-oneapi-mkl'
+    - '^intel-oneapi-mkl'

configs/sites/tier1/noaa-aws/packages_intel.yaml

@@ -30,5 +30,5 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH

configs/sites/tier1/noaa-azure/packages_intel.yaml

@@ -30,5 +30,5 @@ packages:
     - '@1.2.1 ~mkl'
   py-numpy:
     require::
-    - '@:1.23 ^openblas'
+    - '^openblas'
   # *DH