Bring over other gcm changes here

Author: mathomp4

scripts/fv3_setup

@@ -305,10 +305,10 @@ endif
 
 ASKPROC:
 
-set BUILT_ON_SLES15 = @BUILT_ON_SLES15@
-
 if ( $SITE == 'NCCS' ) then
 
+   set BUILT_ON_SLES15 = @BUILT_ON_SLES15@
+
    if ("$BUILT_ON_SLES15" == "TRUE") then
       set DEFAULT_MODEL = 'mil'
       echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
@@ -730,7 +730,7 @@ EOF
 # The below settings seem to be recommended for hybrid
 # systems using MVAPICH2 but could change
 
-else if( $MPI == mvapich2 ) then
+else if( $MPI == mvapich ) then
 
 cat > $EXPDIR/SETENV.commands << EOF
    setenv MV2_ENABLE_AFFINITY     0
@@ -800,6 +800,17 @@ EOF
 
 endif # if NOT Singularity
 
+# Testing on SLES15 showed that the mlx provider did not seem
+# to work at scale. So we move to use the verbs provider. Note:
+# still seems to have issues at c720
+if ("$BUILT_ON_SLES15" == "TRUE") then
+cat >> $EXPDIR/SETENV.commands << EOF
+setenv I_MPI_OFI_PROVIDER verbs
+setenv I_MPI_COLL_EXTERNAL 0
+EOF
+
+endif # if SLES15
+
 endif # if NCCS
 
 endif # if mpi