Sync develop into main

.github/workflows/push-to-develop.yml

@@ -11,9 +11,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          filter: blob:none
       - name: Run the action
         uses: devops-infra/action-pull-request@v0.5.5
         with:

.github/workflows/validate_yaml_files.yml

@@ -15,7 +15,11 @@ jobs:
   validate-YAML:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.3.0
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          filter: blob:none
       - id: yaml-lint
         name: yaml-lint
         uses: ibiqlik/action-yamllint@v3
@@ -24,7 +28,7 @@ jobs:
           format: colored
           config_file: .yamllint.yml
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: yamllint-logfile

gcm_setup

@@ -420,18 +420,28 @@ if ( $SITE == 'NCCS' ) then
    if ("$BUILT_ON_SLES15" == "TRUE") then
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
       echo "   ${C2}mil  (Milan)${CN} (default)"
+      echo "   ${C2}cas  (Cascade Lake)${CN}"
       echo " "
       set MODEL = `echo $<`
       set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
       if ( .$MODEL == .) then
          set MODEL = 'mil'
       endif
 
-      if( $MODEL != 'mil' ) goto ASKPROC
+      if( $MODEL != 'mil' & \
+          $MODEL != 'cas' ) goto ASKPROC
 
       if ($MODEL == 'mil') then
          # We save a couple processes for the kernel
          set NCPUS_PER_NODE = 126
+      else if ($MODEL == 'cas') then
+         # NCCS currently recommends that users do not run with
+         # 48 cores per node on SCU16 due to OS issues and
+         # recommends that CPU-intensive works run with 46 or less
+         # cores. As 45 is a multiple of 3, it's the best value
+         # that doesn't waste too much
+         #set NCPUS_PER_NODE = 48
+         set NCPUS_PER_NODE = 45
       endif
    else
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
@@ -2279,6 +2289,9 @@ if ( $SITE == 'NCCS' ) then
 
 if ("$BUILT_ON_SLES15" == "TRUE") then
 cat >> $HOMDIR/SETENV.commands << EOF
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_GATHERV 3
+
 setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
 EOF

geoschemchem_setup

@@ -420,18 +420,28 @@ if ( $SITE == 'NCCS' ) then
    if ("$BUILT_ON_SLES15" == "TRUE") then
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
       echo "   ${C2}mil  (Milan)${CN} (default)"
+      echo "   ${C2}cas  (Cascade Lake)${CN}"
       echo " "
       set MODEL = `echo $<`
       set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
       if ( .$MODEL == .) then
          set MODEL = 'mil'
       endif
 
-      if( $MODEL != 'mil' ) goto ASKPROC
+      if( $MODEL != 'mil' & \
+          $MODEL != 'cas' ) goto ASKPROC
 
       if ($MODEL == 'mil') then
          # We save a couple processes for the kernel
          set NCPUS_PER_NODE = 126
+      else if ($MODEL == 'cas') then
+         # NCCS currently recommends that users do not run with
+         # 48 cores per node on SCU16 due to OS issues and
+         # recommends that CPU-intensive works run with 46 or less
+         # cores. As 45 is a multiple of 3, it's the best value
+         # that doesn't waste too much
+         #set NCPUS_PER_NODE = 48
+         set NCPUS_PER_NODE = 45
       endif
    else
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
@@ -2309,6 +2319,9 @@ if ( $SITE == 'NCCS' ) then
 
 if ("$BUILT_ON_SLES15" == "TRUE") then
 cat >> $HOMDIR/SETENV.commands << EOF
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_GATHERV 3
+
 setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
 EOF

gmichem_setup

@@ -420,18 +420,28 @@ if ( $SITE == 'NCCS' ) then
    if ("$BUILT_ON_SLES15" == "TRUE") then
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
       echo "   ${C2}mil  (Milan)${CN} (default)"
+      echo "   ${C2}cas  (Cascade Lake)${CN}"
       echo " "
       set MODEL = `echo $<`
       set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
       if ( .$MODEL == .) then
          set MODEL = 'mil'
       endif
 
-      if( $MODEL != 'mil' ) goto ASKPROC
+      if( $MODEL != 'mil' & \
+          $MODEL != 'cas' ) goto ASKPROC
 
       if ($MODEL == 'mil') then
          # We save a couple processes for the kernel
          set NCPUS_PER_NODE = 126
+      else if ($MODEL == 'cas') then
+         # NCCS currently recommends that users do not run with
+         # 48 cores per node on SCU16 due to OS issues and
+         # recommends that CPU-intensive works run with 46 or less
+         # cores. As 45 is a multiple of 3, it's the best value
+         # that doesn't waste too much
+         #set NCPUS_PER_NODE = 48
+         set NCPUS_PER_NODE = 45
       endif
    else
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
@@ -2481,6 +2491,9 @@ if ( $SITE == 'NCCS' ) then
 
 if ("$BUILT_ON_SLES15" == "TRUE") then
 cat >> $HOMDIR/SETENV.commands << EOF
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_GATHERV 3
+
 setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
 EOF

scm_setup

@@ -458,6 +458,9 @@ cat $expdir/WSUB_ExtData.tmp | sed '/collection:/ s#WSUB_SWclim.*#/dev/null#' >
 /bin/cp $SOURCEDIR/sedfile.CASE $expdir
 /bin/cp $SOURCEDIR/AGCM.CASE.txt $expdir
 
+# These are the weights files
+/bin/cp $SOURCEDIR/rh_* $expdir
+
 # MAT These sed commands have to be done in the experiment directory.
 #     This should probably be fixed at some point
 

stratchem_setup

@@ -420,18 +420,28 @@ if ( $SITE == 'NCCS' ) then
    if ("$BUILT_ON_SLES15" == "TRUE") then
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
       echo "   ${C2}mil  (Milan)${CN} (default)"
+      echo "   ${C2}cas  (Cascade Lake)${CN}"
       echo " "
       set MODEL = `echo $<`
       set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
       if ( .$MODEL == .) then
          set MODEL = 'mil'
       endif
 
-      if( $MODEL != 'mil' ) goto ASKPROC
+      if( $MODEL != 'mil' & \
+          $MODEL != 'cas' ) goto ASKPROC
 
       if ($MODEL == 'mil') then
          # We save a couple processes for the kernel
          set NCPUS_PER_NODE = 126
+      else if ($MODEL == 'cas') then
+         # NCCS currently recommends that users do not run with
+         # 48 cores per node on SCU16 due to OS issues and
+         # recommends that CPU-intensive works run with 46 or less
+         # cores. As 45 is a multiple of 3, it's the best value
+         # that doesn't waste too much
+         #set NCPUS_PER_NODE = 48
+         set NCPUS_PER_NODE = 45
       endif
    else
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
@@ -2294,6 +2304,9 @@ if ( $SITE == 'NCCS' ) then
 
 if ("$BUILT_ON_SLES15" == "TRUE") then
 cat >> $HOMDIR/SETENV.commands << EOF
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_GATHERV 3
+
 setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
 EOF