Merge pull request #681 from GEOS-ESM/feature/mathomp4/v11-cas-sles

v11: Add Cas for SLES15; Add back IMPI flags on SLES15

Author: sdrabenh

gcm_setup

@@ -420,18 +420,28 @@ if ( $SITE == 'NCCS' ) then
    if ("$BUILT_ON_SLES15" == "TRUE") then
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
       echo "   ${C2}mil  (Milan)${CN} (default)"
+      echo "   ${C2}cas  (Cascade Lake)${CN}"
       echo " "
       set MODEL = `echo $<`
       set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
       if ( .$MODEL == .) then
          set MODEL = 'mil'
       endif
 
-      if( $MODEL != 'mil' ) goto ASKPROC
+      if( $MODEL != 'mil' & \
+          $MODEL != 'cas' ) goto ASKPROC
 
       if ($MODEL == 'mil') then
          # We save a couple processes for the kernel
          set NCPUS_PER_NODE = 126
+      else if ($MODEL == 'cas') then
+         # NCCS currently recommends that users do not run with
+         # 48 cores per node on SCU16 due to OS issues and
+         # recommends that CPU-intensive works run with 46 or less
+         # cores. As 45 is a multiple of 3, it's the best value
+         # that doesn't waste too much
+         #set NCPUS_PER_NODE = 48
+         set NCPUS_PER_NODE = 45
       endif
    else
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
@@ -2279,6 +2289,9 @@ if ( $SITE == 'NCCS' ) then
 
 if ("$BUILT_ON_SLES15" == "TRUE") then
 cat >> $HOMDIR/SETENV.commands << EOF
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_GATHERV 3
+
 setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
 EOF

geoschemchem_setup

@@ -420,18 +420,28 @@ if ( $SITE == 'NCCS' ) then
    if ("$BUILT_ON_SLES15" == "TRUE") then
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
       echo "   ${C2}mil  (Milan)${CN} (default)"
+      echo "   ${C2}cas  (Cascade Lake)${CN}"
       echo " "
       set MODEL = `echo $<`
       set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
       if ( .$MODEL == .) then
          set MODEL = 'mil'
       endif
 
-      if( $MODEL != 'mil' ) goto ASKPROC
+      if( $MODEL != 'mil' & \
+          $MODEL != 'cas' ) goto ASKPROC
 
       if ($MODEL == 'mil') then
          # We save a couple processes for the kernel
          set NCPUS_PER_NODE = 126
+      else if ($MODEL == 'cas') then
+         # NCCS currently recommends that users do not run with
+         # 48 cores per node on SCU16 due to OS issues and
+         # recommends that CPU-intensive works run with 46 or less
+         # cores. As 45 is a multiple of 3, it's the best value
+         # that doesn't waste too much
+         #set NCPUS_PER_NODE = 48
+         set NCPUS_PER_NODE = 45
       endif
    else
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
@@ -2309,6 +2319,9 @@ if ( $SITE == 'NCCS' ) then
 
 if ("$BUILT_ON_SLES15" == "TRUE") then
 cat >> $HOMDIR/SETENV.commands << EOF
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_GATHERV 3
+
 setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
 EOF

gmichem_setup

@@ -420,18 +420,28 @@ if ( $SITE == 'NCCS' ) then
    if ("$BUILT_ON_SLES15" == "TRUE") then
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
       echo "   ${C2}mil  (Milan)${CN} (default)"
+      echo "   ${C2}cas  (Cascade Lake)${CN}"
       echo " "
       set MODEL = `echo $<`
       set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
       if ( .$MODEL == .) then
          set MODEL = 'mil'
       endif
 
-      if( $MODEL != 'mil' ) goto ASKPROC
+      if( $MODEL != 'mil' & \
+          $MODEL != 'cas' ) goto ASKPROC
 
       if ($MODEL == 'mil') then
          # We save a couple processes for the kernel
          set NCPUS_PER_NODE = 126
+      else if ($MODEL == 'cas') then
+         # NCCS currently recommends that users do not run with
+         # 48 cores per node on SCU16 due to OS issues and
+         # recommends that CPU-intensive works run with 46 or less
+         # cores. As 45 is a multiple of 3, it's the best value
+         # that doesn't waste too much
+         #set NCPUS_PER_NODE = 48
+         set NCPUS_PER_NODE = 45
       endif
    else
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
@@ -2481,6 +2491,9 @@ if ( $SITE == 'NCCS' ) then
 
 if ("$BUILT_ON_SLES15" == "TRUE") then
 cat >> $HOMDIR/SETENV.commands << EOF
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_GATHERV 3
+
 setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
 EOF

stratchem_setup

@@ -420,18 +420,28 @@ if ( $SITE == 'NCCS' ) then
    if ("$BUILT_ON_SLES15" == "TRUE") then
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
       echo "   ${C2}mil  (Milan)${CN} (default)"
+      echo "   ${C2}cas  (Cascade Lake)${CN}"
       echo " "
       set MODEL = `echo $<`
       set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
       if ( .$MODEL == .) then
          set MODEL = 'mil'
       endif
 
-      if( $MODEL != 'mil' ) goto ASKPROC
+      if( $MODEL != 'mil' & \
+          $MODEL != 'cas' ) goto ASKPROC
 
       if ($MODEL == 'mil') then
          # We save a couple processes for the kernel
          set NCPUS_PER_NODE = 126
+      else if ($MODEL == 'cas') then
+         # NCCS currently recommends that users do not run with
+         # 48 cores per node on SCU16 due to OS issues and
+         # recommends that CPU-intensive works run with 46 or less
+         # cores. As 45 is a multiple of 3, it's the best value
+         # that doesn't waste too much
+         #set NCPUS_PER_NODE = 48
+         set NCPUS_PER_NODE = 45
       endif
    else
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
@@ -2294,6 +2304,9 @@ if ( $SITE == 'NCCS' ) then
 
 if ("$BUILT_ON_SLES15" == "TRUE") then
 cat >> $HOMDIR/SETENV.commands << EOF
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_GATHERV 3
+
 setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
 EOF