diff --git a/scripts/fv3_setup b/scripts/fv3_setup index 5cb8592..d326044 100755 --- a/scripts/fv3_setup +++ b/scripts/fv3_setup @@ -290,49 +290,30 @@ ASKPROC: if ( $SITE == 'NCCS' ) then - set BUILT_ON_SLES15 = @BUILT_ON_SLES15@ - - if ("$BUILT_ON_SLES15" == "TRUE") then - echo "Enter the ${C1}Processor Type${CN} you wish to run on:" - echo " ${C2}mil (Milan)${CN} (default)" - echo " " - set MODEL = `echo $<` - set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` - if ( .$MODEL == .) then - set MODEL = 'mil' - endif - - if( $MODEL != 'mil' ) goto ASKPROC - - if ($MODEL == 'mil') then - # We save a couple processes for the kernel - set NCPUS_PER_NODE = 126 - endif - else - echo "Enter the ${C1}Processor Type${CN} you wish to run on:" - echo " ${C2}sky (Skylake)${CN}" - echo " ${C2}cas (Cascade Lake) (default)${CN}" - echo " " - set MODEL = `echo $<` - set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` - if ( .$MODEL == .) then - set MODEL = 'cas' - endif + echo "Enter the ${C1}Processor Type${CN} you wish to run on:" + echo " ${C2}mil (Milan)${CN} (default)" + echo " ${C2}cas (Cascade Lake)${CN}" + echo " " + set MODEL = `echo $<` + set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` + if ( .$MODEL == .) then + set MODEL = 'mil' + endif - if( $MODEL != 'sky' & \ - $MODEL != 'cas' ) goto ASKPROC - - if ($MODEL == 'sky') then - set NCPUS_PER_NODE = 40 - else if ($MODEL == 'cas') then - # NCCS currently recommends that users do not run with - # 48 cores per node on SCU16 due to OS issues and - # recommends that CPU-intensive works run with 46 or less - # cores. As 45 is a multiple of 3, it's the best value - # that doesn't waste too much - #set NCPUS_PER_NODE = 48 - set NCPUS_PER_NODE = 45 - endif + if( $MODEL != 'mil' & \ + $MODEL != 'cas' ) goto ASKPROC + + if ($MODEL == 'mil') then + # We save a couple processes for the kernel + set NCPUS_PER_NODE = 126 + else if ($MODEL == 'cas') then + # NCCS currently recommends that users do not run with + # 48 cores per node on SCU16 due to OS issues and + # recommends that CPU-intensive works run with 46 or less + # cores. As 45 is a multiple of 3, it's the best value + # that doesn't waste too much + #set NCPUS_PER_NODE = 48 + set NCPUS_PER_NODE = 45 endif else if ( $SITE == 'NAS' ) then @@ -811,51 +792,13 @@ EOF if ( $SITE == 'NCCS' ) then cat >> $EXPDIR/SETENV.commands << EOF -setenv I_MPI_SHM_HEAP_VSIZE 512 -setenv PSM2_MEMORY large -EOF - -# Testing at NCCS showed these caused a crash at higher resolutions at -# restart read -if ( $USING_SINGULARITY == FALSE ) then - -cat >> $EXPDIR/SETENV.commands << EOF -setenv I_MPI_EXTRA_FILESYSTEM 1 -setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs -EOF +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_GATHERV 3 -endif # if NOT Singularity - -# Testing by Bill Putman found these to be -# useful flags with Intel MPI on SLES15 on the -# Milan nodes. -# Note 1: Testing by NCCS shows the PSM3 provider -# runs on the Infiniband fabric. Tests show it runs -# up to C720. -# Note 2: When the Cascade Lakes are moved to -# SLES15, these will need to be Milan-only flags -# as Intel MPI will probably work just fine with -# Intel chips. -if ("$BUILT_ON_SLES15" == "TRUE") then -cat >> $EXPDIR/SETENV.commands << EOF -setenv I_MPI_FALLBACK 0 -setenv I_MPI_FABRICS ofi +setenv I_MPI_FABRICS shm:ofi setenv I_MPI_OFI_PROVIDER psm3 -setenv I_MPI_ADJUST_SCATTER 2 -setenv I_MPI_ADJUST_SCATTERV 2 -setenv I_MPI_ADJUST_GATHER 2 -setenv I_MPI_ADJUST_GATHERV 3 -setenv I_MPI_ADJUST_ALLGATHER 3 -setenv I_MPI_ADJUST_ALLGATHERV 3 -setenv I_MPI_ADJUST_ALLREDUCE 12 -setenv I_MPI_ADJUST_REDUCE 10 -setenv I_MPI_ADJUST_BCAST 11 -setenv I_MPI_ADJUST_REDUCE_SCATTER 4 -setenv I_MPI_ADJUST_BARRIER 9 EOF -endif # if SLES15 - endif # if NCCS endif # if mpi