NCO/WCOSS2-related updates from release/1.9.0 to develop (#1690)

* Add fetch_go_deps.py, with ability to specify packages
* NCO template: pin [email protected], don't specify compilers for individual packages
* Substantial updates to WCOSS2 site config for 1.9.2 release, including oneAPI compilers.

Author: AlexanderRichert-NOAA

configs/sites/tier1/wcoss2/compilers.yaml

@@ -43,3 +43,49 @@ compilers:
         CONFIG_SITE: ''
       unset: [PYTHONPATH]
     extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: cc
+      cxx: CC
+      f77: ftn
+      fc: ftn
+    flags:
+      cflags: '--gcc-toolchain=/opt/cray/pe/gcc/12.1.0/snos/'
+      fflags: '-gcc-name=/opt/cray/pe/gcc/12.1.0/snos/bin/gcc -gxx-name=/opt/cray/pe/gcc/12.1.0/snos/bin/g++'
+      cxxflags: '--gcc-toolchain=/opt/cray/pe/gcc/12.1.0/snos/'
+    operating_system: sles15
+    modules:
+    - PrgEnv-intel/8.3.3
+    - craype/2.7.17
+    - intel-oneapi/2024.2
+    - libfabric
+    environment:
+      prepend_path:
+        PATH: /opt/cray/pe/gcc/12.1.0/snos/bin
+        LD_LIBRARY_PATH: /opt/cray/pe/gcc/12.1.0/snos/lib64
+        LIBRARY_PATH: /opt/cray/pe/gcc/12.1.0/snos/lib64
+        CPATH: /opt/cray/pe/gcc/12.1.0/snos/include
+      set:
+        # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so
+        # Automake-based builds are installed in lib64
+        # which confuses some packages.
+        CONFIG_SITE: ''
+    extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /opt/cray/pe/gcc/12.1.0/snos/bin/gcc
+      cxx: /opt/cray/pe/gcc/12.1.0/snos/bin/g++
+      f77: /opt/cray/pe/gcc/12.1.0/snos/bin/gfortran
+      fc: /opt/cray/pe/gcc/12.1.0/snos/bin/gfortran
+    flags: {}
+    operating_system: sles15
+    modules: []
+    environment:
+      set:
+        # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so
+        # Automake-based builds are installed in lib64
+        # which confuses some packages.
+        CONFIG_SITE: ''
+    extra_rpaths: []

configs/sites/tier1/wcoss2/packages.yaml

@@ -1,14 +1,6 @@
   packages:
     #
-    ### Set compiler.
-    #
-    all:
-      prefer: ['%[email protected]']
-      compiler:: [[email protected]]
-      providers:
-        mpi: [cray-mpich]
-    #
-    ### Strictly set virtual package providers.
+    ### Strictly set virtual package providers (+disable mkl).
     #
     mpi:
       buildable: False
@@ -21,16 +13,8 @@
       require: openblas
     jpeg:
       require: libjpeg-turbo
-    #
-    ### External packages.
-    #
-    cray-mpich:
-      externals:
-      - spec: [email protected]~wrappers
-        modules:
-        - libfabric
-        - craype-network-ofi
-        - cray-mpich/8.1.9
+    intel-oneapi-mkl:
+      buildable: false
     #
     ### Individual package settings.
     # Use `require::` to override require's from {common,site}/packages.yaml,
@@ -44,7 +28,6 @@
     cdo:
       require:
       - 'grib2=none'   # avoids eccodes/grib-api dependency
-      - '%intel'
       - any_of: ["@2.2.2"]
         when: "%[email protected]"
         message: "2.3.0 is the last version to use C++17"
@@ -72,13 +55,13 @@
       - snapshot=none   # make sure spack-stack doesn't accidentally give us a beta snapshot
       - '~python'
     flex:
-      require: '@2.6.4'
-    gcc-runtime:
-      require: '%gcc'
+      require: ['@2.6.4']
     gdal:
       variants: ~curl
     gettext:
-      version: ['0.19.7']
+      require:
+      - any_of: ['@0.19.7']
+        when: '%[email protected]'
     git-lfs:
       require: '%gcc'
     glib:
@@ -88,37 +71,47 @@
     gsibec:
       require::
       - '@1.2.1 ~mkl'
-    harfbuzz:
-      require: '%gcc'
+    icu4c:
+      require: ['%gcc']
     libffi:
-      require: '@3.3'
+      require:
+      - any_of: ['@3.3']
+        when: '%intel'
     libunistring:
-      require: '@:1.0'
+      require:
+      - any_of: ['@:1.0']
+        when: '%intel'
     mapl:
       require:
       - ~pflogger ~fargparse ~extdata2g #   minimize dependencies
     netcdf-c:
       'require:': +mpi ~parallel-netcdf ~dap ~blosc ~szip build_system=autotools # disabling dap prevents network access; disabling blosc and szip reduces dependencies
     patchelf:
-      version: ['0.13.1']
+      require:
+      - any_of: ['@0.13.1']
+        when: '%intel'
     py-cython::
-      require: '@0.29.36'
-    py-numpy:
-      require::
-      - '^[virtuals=lapack,blas] openblas'
-      - '@:1.25'
-      - any_of: ['@:1.24']
-        when: '%intel@:19'
+      require:
+      - any_of: ['@0.29.36']
+        when: '%intel'
     py-pandas:
       'require:': ~excel #   minimize dependencies
     py-scipy:
       require:
       - any_of: ["@1.10.1"]
         when: '%intel@:19'
     py-setuptools:
-      require: '@63.4.3' #   avoid duplicate build deps
+      require: ['@63.4.3'] #   avoid duplicate build deps
     rhash:
-      version: ['1.3.5']
+      require:
+      - any_of: ['@1.3.5']
+        when: '%intel'
     subversion:
       require:
       - ~serf +pic #   avoid serf dependency
+    zlib-ng:
+      require: ['+compat ~new_strategies']
+    crtm:                                                     
+      require: ['@2.4.0.1 +fix']
+    scotch:
+      require:: ['@7.0.7 +mpi+metis~shared~threads~mpi_thread+noarch+esmumps build_system=cmake']

configs/sites/tier1/wcoss2/packages_intel.yaml

@@ -0,0 +1,40 @@
+packages:
+    #
+    ### Set compilers & MPI provider for metamodule generation. 
+    # 
+    all: 
+      compiler:: [[email protected]]
+      providers:
+        mpi:: [cray-mpich]
+    #
+    ### External packages.
+    # 
+    cray-mpich:
+      externals:
+      - spec: [email protected]~wrappers
+        modules:
+        - libfabric
+        - craype-network-ofi
+        - cray-mpich/8.1.9
+    #
+    ### Set GCC version.
+    #
+    gcc-runtime:
+      require: ['%[email protected]']
+    #
+    ### Package settings.
+    #
+    harfbuzz:
+      require: ['%gcc']
+    ecflow:
+      require::
+      - '@5.11.4'
+      - '+ui'
+      - 'cflags="-no-ipo" cxxflags="-no-ipo"'
+      - '^boost %gcc'
+      - '%intel'
+    py-numpy::
+      require::
+      - '^[virtuals=lapack,blas] openblas'
+      - '@1.26'
+      - '%gcc'

configs/sites/tier1/wcoss2/packages_oneapi.yaml

@@ -0,0 +1,52 @@
+packages:
+  #
+  ### Set compilers & MPI provider for metamodule generation.
+  #
+  all:
+    compiler:: [[email protected]]
+    providers:
+      mpi:: [cray-mpich]
+  gcc-runtime:
+    require: '%[email protected]'
+  cray-mpich:
+    externals:
+    - spec: [email protected]~wrappers
+      modules:
+      - libfabric
+      - craype-network-ofi
+      - cray-mpich/8.1.29
+      - cray-pals
+  bison:
+    require: ['%gcc']
+  cairo:
+    require: ['+fc']
+  ectrans:
+    require::
+    - '@1.2.0 ~mkl +fftw'
+  elfutils:
+    require: ['@0.191']
+  gsibec:
+    require::
+    - '@1.2.1 ~mkl'
+  subversion:
+    require: ['%gcc']
+  cmake:
+    require: ['%gcc']
+  proj:
+    require: ['@9.4.1']
+  libtiff:
+    require: ['build_system=cmake']
+  ninja:
+    require: ['~re2c %gcc']
+  pixman:
+    require: ['build_system=meson']
+  apr:
+    require: ['@:1.6', '%gcc']
+  glib:
+    require: ['%gcc']
+  zlib-ng:
+    require: ['cflags="-static-intel"', 'cxxflags="-static-intel"']
+  py-numpy::
+    require::
+    - '^[virtuals=lapack,blas] openblas'
+    - '@1.26'

configs/templates/nco/spack.yaml

@@ -14,18 +14,18 @@ spack:
   specs:
   - awscli
   - bacio
-  - boost %intel
+  - boost
   - bufr
   - cdo
   - cfitsio
   - cmake
-  - crtm
+  - crtm@=2.4.0.1
   - curl
-  - ecflow ^boost%gcc
+  - ecflow
   - eckit
   - ecmwf-atlas
   - eigen
-  - esmf
+  - esmf@8.8.0
   - fckit
   - fms
   - g2
@@ -53,7 +53,7 @@ spack:
   - libxml2
   - libxrender
   - madis
-  - mapl
+  - mapl@2.53.4 ^[email protected]
   - mbedtls
   - met
   - metis