Merge pull request #303 from GEOS-ESM/feature/mathomp4/v12-benchmark-fv3

Author: mathomp4

CMakeLists.txt

@@ -160,6 +160,12 @@ ecbuild_add_executable (
   SOURCES StandAlone_FV3_Dycore.F90
   LIBS ${this} OpenMP::OpenMP_Fortran)
 
+find_package (JeMalloc)
+if(JeMalloc_FOUND)
+  message(STATUS "JeMalloc found, using for StandAlone_FV3_Dycore.x")
+  target_link_libraries (StandAlone_FV3_Dycore.x JeMalloc::JeMalloc)
+endif()
+
 ecbuild_add_executable (
   TARGET rs_scale.x
   SOURCES rs_scale.F90

scripts/fv3.j

@@ -20,7 +20,6 @@ limit coredumpsize 0
 #                     Set Model Run Parameters
 #######################################################################
 
-set NH    = @USE_NONHYDRO
 set FV_NX = @FV_NX
 set FV_NY = @FV_NY
 set NX = $FV_NX
@@ -124,8 +123,8 @@ endif
 #######################################################################
 
 setenv GEOSDIR @GEOSDIR
-setenv GEOSBIN @GEOSBIN
-setenv GEOSETC @GEOSETC
+setenv GEOSBIN ${GEOSDIR}/bin
+setenv GEOSETC ${GEOSDIR}/etc
 
 set TAG = `cat $GEOSETC/.FV3_VERSION`
 set RUN_CMD = "$GEOSBIN/esma_mpirun -np "
@@ -151,7 +150,6 @@ module list
          setenv EXPDSC  "c${AGCM_IM}_L${AGCM_LM}_T${N_TRACERS}_${NX}x${NY}_${N_OMP}threads"
          setenv EXPDIR  @EXPDIR
          setenv SCRDIR  $EXPDIR/scratch_${EXPID}_${EXETAG}-${FV3PRC}
-if ($NH) setenv SCRDIR  ${SCRDIR}_NH.$$
 
 #######################################################################
 #                 Create Experiment Scratch-Directory
@@ -314,30 +312,25 @@ PC@HIST_IMx@HIST_JM-DC.LM: @AGCM_LM
                             ::
 EOF
 
-         set hydrostatic='.true.'
-if ($NH) set hydrostatic='.false.'
-
 set GRID_INPUT = "'INLINE'"
 
 /bin/rm -f input.nml
 cat >      input.nml << EOF
 &fv_core_nml
-       npx = ${FV3_NPX}
-       npy = ${FV3_NPX}
-       npz = ${FV3_NPZ}
-       adiabatic = .true.
-       hydrostatic = ${hydrostatic}
-       make_nh = .T.
-       fv_debug = .F.
-       fv_sg_adj = -1
-       n_sponge = -1
-       n_zfilter = 0
-       compute_coords_locally = .false.
+  npx = ${FV3_NPX}
+  npy = ${FV3_NPX}
+  npz = ${FV3_NPZ}
+  adiabatic = .true.
+  fv_debug = .F.
+  range_warn = .F.
+  fv_sg_adj = -1
+  n_sponge = -1
+  n_zfilter = 0
+  compute_coords_locally = .false.
 /
 
 &fv_grid_nml
 /
-#       grid_file = $GRID_INPUT
 
 &main_nml
 /
@@ -350,8 +343,10 @@ cat >      input.nml << EOF
 /
 
 &fms_nml
-        print_memory_usage=.false.
-        domains_stack_size = 24000000
+  print_memory_usage=.true.
+  domains_stack_size = 12000000
+  clock_grain='MODULE',
+  clock_flags='DETAILED',
 /
 EOF
 

scripts/fv3_setup

@@ -198,47 +198,10 @@ set CUBE_AGCM = ""
 set AGCM_NF = 6
 set GRID_TYPE = "Cubed-Sphere"
 
-echo "Enter the ${C1}Vertical Resolution${CN}: ${C2}LM${CN} (Default: 72)"
+echo "Enter the ${C1}Vertical Resolution${CN}: ${C2}LM${CN} (Default: 181)"
 set   AGCM_LM = $<
 if( .$AGCM_LM == . ) then
-  set AGCM_LM = 72
-endif
-
-
-#######################################################################
-#         Test to see if you want to use non-hydrostatic
-#######################################################################
-
-ASKHYDRO:
-
-set DEFAULT_NON_HYDROSTATIC = FALSE
-echo "Do you wish to run ${C1}NON-HYDROSTATIC${CN} dynamics? (Default: ${C2}${DEFAULT_NON_HYDROSTATIC}${CN})"
-
-set   DO_NONHYDRO  = $<
-if( .$DO_NONHYDRO == . ) then
-   set   DO_NONHYDRO  = $DEFAULT_NON_HYDROSTATIC
-else
-   set   DO_NONHYDRO  = `echo   $DO_NONHYDRO | tr "[:lower:]" "[:upper:]"`
-   if(  $DO_NONHYDRO == "Y"     | \
-        $DO_NONHYDRO == "YES"   | \
-        $DO_NONHYDRO == "T"     | \
-        $DO_NONHYDRO == "TRUE"  ) set DO_NONHYDRO = TRUE
-   if(  $DO_NONHYDRO == "N"     | \
-        $DO_NONHYDRO == "NO"    | \
-        $DO_NONHYDRO == "F"     | \
-        $DO_NONHYDRO == "FALSE" ) set DO_NONHYDRO = FALSE
-
-   if( $DO_NONHYDRO != "TRUE" & $DO_NONHYDRO != "FALSE" ) then
-      echo
-      echo "${C1}NON-HYDROSTATIC${CN} must be set equal to ${C2}TRUE/YES${CN} or ${C2}FALSE/NO${CN}!"
-      goto ASKHYDRO
-   endif
-endif
-
-if ( $DO_NONHYDRO == "TRUE" ) then
-   set USE_NONHYDRO = 1
-else
-   set USE_NONHYDRO = 0
+  set AGCM_LM = 181
 endif
 
 #######################################################################
@@ -358,15 +321,16 @@ else if ( $SITE == 'NAS' ) then
    if ($MODEL == 'has') then
       set NCPUS_PER_NODE = 24
    else if ($MODEL == 'bro') then
-      set NCPUS_PER_NODE = 28
+      # We use 24 of 28 cores per node for even division
+      set NCPUS_PER_NODE = 24
    else if ($MODEL == 'sky_ele') then
       set NCPUS_PER_NODE = 40
    else if ($MODEL == 'cas_ait') then
       set NCPUS_PER_NODE = 40
    else if ($MODEL == 'rom_ait') then
-      set NCPUS_PER_NODE = 128
+      set NCPUS_PER_NODE = 120
    else if ($MODEL == 'mil_ait') then
-      set NCPUS_PER_NODE = 128
+      set NCPUS_PER_NODE = 120
    endif
 
 else if( $SITE == 'AWS' | $SITE == 'Azure' ) then
@@ -428,62 +392,68 @@ set HIST_IM  = `expr $AGCM_IM \* 4`
 set HIST_JM  = `expr $AGCM_IM \* 2 + 1`
 
 if( $AGCM_IM <=  12 ) then
-   set DT = 900
+   set DT = 3600
    set FV_NX = 2
 else if( $AGCM_IM <=  24 ) then
-   set DT = 900
+   set DT = 1800
    set FV_NX = 4
 else if( $AGCM_IM <=  48 ) then
-   set DT = 450
-   set FV_NX = 4
+   set DT = 1200
+   set FV_NX = 6
 else if( $AGCM_IM <=  90 ) then
-   set DT = 450
-   set FV_NX = 4
+   set DT = 900
+   set FV_NX = 10
 else if( $AGCM_IM <=  180 ) then
-   set DT = 450
-   set FV_NX = 6
+   set DT = 600
+   set FV_NX = 20
    set NUM_READERS = 2
    set DEF_IOS_NDS = 2
 else if( $AGCM_IM <= 360 ) then
    set DT = 450
-   set FV_NX = 12
+   set FV_NX = 30
    set NUM_READERS = 4
    set DEF_IOS_NDS = 2
 else if( $AGCM_IM <= 500 ) then
    set DT = 450
-   set FV_NX = 12
+   set FV_NX = 30
    set NUM_READERS = 4
    set USE_SHMEM = 1
    set DEF_IOS_NDS = 2
 else if( $AGCM_IM <= 720 ) then
-   set DT = 450
-   set FV_NX = 16
+   set DT = 300
+   set FV_NX = 40
    set NUM_READERS = 6
    set USE_SHMEM = 1
    set DEF_IOS_NDS = 3
+else if( $AGCM_IM <= 1120 ) then
+   set DT = 300
+   set FV_NX = 60
+   set NUM_READERS = 6
+   set USE_SHMEM = 1
+   set DEF_IOS_NDS = 4
 else if( $AGCM_IM <= 1440 ) then
-   set DT = 450
-   set FV_NX = 30
+   set DT = 225
+   set FV_NX = 80
    set NUM_READERS = 6
    set USE_SHMEM = 1
    set DEF_IOS_NDS = 4
-else if( $AGCM_IM <= 1536 ) then
-   set DT = 90
-   set FV_NX = 64
+else if( $AGCM_IM <= 2880 ) then
+   set DT = 150
+   set FV_NX = 80
    set NUM_READERS = 6
    set USE_SHMEM = 1
    set DEF_IOS_NDS = 4
 else
-   set DT = 45
-   set FV_NX = 64
+   set DT = 75
+   set FV_NX = 80
    set NUM_READERS = 6
    set USE_SHMEM = 1
    set DEF_IOS_NDS = 5
 endif
 
-# On desktop, we default to 6 processes
+# On SITE == GMAO.desktop, we default to 6 processes
 # Must be set here due to MODEL_NPES calc below
-if( $SITE != 'NAS' && $SITE != 'NCCS' && $SITE != 'AWS' && $SITE != 'Azure' ) then
+if( $SITE == 'GMAO.desktop' ) then
    set FV_NX = 1
 endif
 
@@ -643,9 +613,6 @@ set     GEOSDIR  = $GEOSDEF
 #
 setenv GEOSSRC  ${GEOSDIR}
 setenv GEOSBIN  ${GEOSDIR}/bin
-setenv GEOSETC  ${GEOSDIR}/etc
-
-setenv GEOSUTIL ${GEOSSRC}
 
 #######################################################################
 #                    Check for Group ID Sponsor Code
@@ -831,10 +798,6 @@ s?@BATCH_OUTPUTNAME?$BATCH_OUTPUTNAME?g
 s?@BATCH_JOINOUTERR?$BATCH_JOINOUTERR?g
 s?@SITE?$SITE?g
 s?@GEOSDIR?$GEOSDIR?g
-s?@GEOSSRC?$GEOSSRC?g
-s?@GEOSBIN?$GEOSBIN?g
-s?@GEOSETC?$GEOSETC?g
-s?@GEOSUTIL?$GEOSUTIL?g
 s?@SINGULARITY_BUILD?$SINGULARITY_BUILD?g
 s?@NATIVE_BUILD?$NATIVE_BUILD?g
 s?@SINGULARITY_SANDBOX?$SINGULARITY_SANDBOX?g
@@ -850,7 +813,6 @@ s/@FV_NY/$FV_NY/g
 s/@FV_PRECISION/$FV_PRECISION/g
 s/@USE_SHMEM/$USE_SHMEM/g
 s/@USE_IOSERVER/$USE_IOSERVER/g
-s/@USE_NONHYDRO/$USE_NONHYDRO/g
 s/@IOS_NDS/$IOS_NDS/g
 s/@NCPUS_PER_NODE/$NCPUS_PER_NODE/g
 s/@NUM_READERS/$NUM_READERS/g