Merge branch 'feature/sdrabenh/gcm_v12' into feature/mathomp4/v12-benchmark-fv3

# Conflicts:
#	FV_StateMod.F90
#	scripts/fv3_setup

Author: mathomp4

AdvCore_GridCompMod.F90

@@ -75,6 +75,7 @@ module AdvCore_GridCompMod
       implicit none
       private
 
+      integer     :: QSPLIT
       integer     :: nx, ny
       integer     :: npes_x, npes_y
       real(FVPRC) :: dt
@@ -330,6 +331,10 @@ subroutine SetServices(GC, rc)
       call MAPL_GetResource( MAPL, rpt_mass, 'ADV_CORE_REPORT_TRACER_MASS:', default=rpt_mass, RC=STATUS )
       VERIFY_(STATUS)
 
+      call MAPL_GetResource( MAPL, QSPLIT, 'ADV_QSPLIT:', default=0, RC=STATUS )
+      VERIFY_(STATUS)
+
+
       ! Start up FV if AdvCore is running without FV3_DynCoreIsRunning
       !--------------------------------------------------
       if (.NOT. FV3_DynCoreIsRunning) then
@@ -758,7 +763,7 @@ subroutine Run(GC, IMPORT, EXPORT, CLOCK, RC)
          call offline_tracer_advection(TRACERS, PLE0, PLE1, MFX, MFY, CX, CY, &
                                        FV_Atm(1)%gridstruct, FV_Atm(1)%flagstruct, FV_Atm(1)%bd, &
                                        FV_Atm(1)%domain, FV_Atm(1)%npx, FV_Atm(1)%npy, FV_Atm(1)%npz,   &
-                                       NQ, dt)
+                                       NQ, dt, QSPLIT)
 
          ! Get Tracer Mass after advection
          !--------------------------------

DynCore_GridCompMod.F90

@@ -6617,7 +6617,10 @@ subroutine RunAddIncs(gc, import, export, clock, rc)
 
 ! Add Diabatic Forcing to State Variables
 ! ---------------------------------------
+    call MAPL_TimerOn (GENSTATE,"PHYS_ADD_INCS")
     call ADD_INCS ( STATE,IMPORT,DT )
+    call MAPL_TimerOff(GENSTATE,"PHYS_ADD_INCS")
+
 
     if (DYN_DEBUG) call DEBUG_FV_STATE('PHYSICS ADD_INCS',STATE)
 

FV_StateMod.F90

@@ -74,6 +74,7 @@ module FV_StateMod
   logical :: fix_mass = .true.
   integer :: CASE_ID = 11
   integer :: AdvCore_Advection = 0
+  integer :: FV3_QSPLIT = 0
   character(LEN=ESMF_MAXSTR) :: FV3_CONFIG
 
   public FV_Atm
@@ -437,6 +438,8 @@ subroutine FV_Setup(GC,LAYOUT_FILE, RC)
   call MAPL_GetResource( MAPL, AdvCore_Advection, label='AdvCore_Advection:', default=AdvCore_Advection, rc=status )
   VERIFY_(STATUS)
 
+  call MAPL_GetResource( MAPL, FV3_QSPLIT, label='FV3_QSPLIT:', default=FV3_QSPLIT, rc=status )
+  VERIFY_(STATUS)
   call MAPL_GetResource( MAPL, ADJUST_DT,       label='ADJUST_DT:'   , default=ADJUST_DT, rc=status )
   VERIFY_(STATUS)
   call MAPL_GetResource( MAPL, INT_fix_mass,    label='fix_mass:'    , default=INT_fix_mass, rc=status )
@@ -642,9 +645,10 @@ subroutine FV_Setup(GC,LAYOUT_FILE, RC)
       FV_Atm(1)%flagstruct%rf_cutoff = 0.35e2
      ! 6th order divergence default damping options
       FV_Atm(1)%flagstruct%nord = 2
-      FV_Atm(1)%flagstruct%dddmp = 0.2  ! Smagorinsky damping
-      FV_Atm(1)%flagstruct%d4_bg = 0.12
-      FV_Atm(1)%flagstruct%d2_bg = 0.0
+      FV_Atm(1)%flagstruct%dddmp = 0.2  ! Smagorinsky damping coef
+      FV_Atm(1)%flagstruct%d4_bg_top = 0.12 ! High-order Divg Damping coef
+      FV_Atm(1)%flagstruct%d4_bg_bot = 0.12 ! High-order Divg Damping coef
+      FV_Atm(1)%flagstruct%d2_bg = 0.0  ! 2nd order Divg Damping coef
       FV_Atm(1)%flagstruct%d_ext = 0.0  ! External damping
      ! Local Richardson-number turbulent mixing 
       FV_Atm(1)%flagstruct%fv_sg_adj = DT*4
@@ -720,7 +724,8 @@ subroutine FV_Setup(GC,LAYOUT_FILE, RC)
        ! 2nd order damping
         FV_Atm(1)%flagstruct%nord = 0
         FV_Atm(1)%flagstruct%dddmp = 0.2
-        FV_Atm(1)%flagstruct%d4_bg = 0.0
+        FV_Atm(1)%flagstruct%d4_bg_top = 0.0
+        FV_Atm(1)%flagstruct%d4_bg_bot = 0.0
         FV_Atm(1)%flagstruct%d2_bg = 0.0075
         FV_Atm(1)%flagstruct%d_ext = 0.02
        ! disable vorticity damping
@@ -1942,13 +1947,13 @@ subroutine FV_Run (STATE, EXPORT, CLOCK, GC, PLE0, RC)
     if (run_gtfv3 == 0) then
        call cpu_time(start)
 #endif
-
+       if (FV3_QSPLIT == 0) FV3_QSPLIT = FV_Atm(1)%flagstruct%q_split
        call fv_dynamics( &
             FV_Atm(1)%npx, FV_Atm(1)%npy, FV_Atm(1)%npz, FV_Atm(1)%ncnst, FV_Atm(1)%ng, myDT, &
             FV_Atm(1)%flagstruct%consv_te, FV_Atm(1)%flagstruct%fill, &
             kappa, cp, zvir, &
             FV_Atm(1)%ptop, FV_Atm(1)%ks, FV_Atm(1)%flagstruct%ncnst, &
-            state%ksplit, state%nsplit, FV_Atm(1)%flagstruct%q_split, &
+            state%ksplit, state%nsplit, FV3_QSPLIT, &
             FV_Atm(1)%u, FV_Atm(1)%v, FV_Atm(1)%w, FV_Atm(1)%delz, &
             FV_Atm(1)%flagstruct%hydrostatic, &
             FV_Atm(1)%pt, FV_Atm(1)%delp, FV_Atm(1)%q, &
@@ -2335,24 +2340,14 @@ subroutine FV_Run (STATE, EXPORT, CLOCK, GC, PLE0, RC)
     if (mpp_pe()==0) print*,''
     if (mpp_pe()==0) print*,'-------------- FV3 Tracer Debug After DYN --------------'
     allocate( DEBUG_ARRAY(isc:iec,jsc:jec,npz) )
-  endif     
-  do n=1,STATE%GRID%NQ
-     if (state%vars%tracer(n)%is_r4) then
-        where (state%vars%tracer(n)%content_r4 < tiny(0.0))
-               state%vars%tracer(n)%content_r4 = 0.0
-        end where
-        if (DEBUG_ADV) DEBUG_ARRAY(:,:,1:npz) = state%vars%tracer(n)%content_r4
-     else
-        where (state%vars%tracer(n)%content < tiny(0.0))
-               state%vars%tracer(n)%content = 0.0
-        end where
-        if (DEBUG_ADV) DEBUG_ARRAY(:,:,1:npz) = state%vars%tracer(n)%content
-     endif
-     if (DEBUG_ADV) then
-        call prt_maxmin(TRIM(state%vars%tracer(n)%tname), DEBUG_ARRAY, isc, iec, jsc, jec, 0, npz, fac1)
-     endif
-  enddo
-  if (DEBUG_ADV) then
+    do n=1,STATE%GRID%NQ
+       if (state%vars%tracer(n)%is_r4) then
+          DEBUG_ARRAY(:,:,1:npz) = state%vars%tracer(n)%content_r4
+       else
+          DEBUG_ARRAY(:,:,1:npz) = state%vars%tracer(n)%content
+       endif
+       call prt_maxmin(TRIM(state%vars%tracer(n)%tname), DEBUG_ARRAY, isc, iec, jsc, jec, 0, npz, fac1)
+    enddo
     deallocate ( DEBUG_ARRAY )
     if (mpp_pe()==0) print*,'-------------- FV3 Tracer Debug After DYN --------------'
     if (mpp_pe()==0) print*,''
@@ -4832,7 +4827,8 @@ subroutine echo_fv3_setup()
    call WRITE_PARALLEL ( FV_Atm(1)%flagstruct%nord ,format='("FV3 nord: ",(I3))' )
    call WRITE_PARALLEL ( FV_Atm(1)%flagstruct%dddmp ,format='("FV3 dddmp: ",(F7.5))' )
    call WRITE_PARALLEL ( FV_Atm(1)%flagstruct%d2_bg ,format='("FV3 d2_bg: ",(F7.5))' )
-   call WRITE_PARALLEL ( FV_Atm(1)%flagstruct%d4_bg ,format='("FV3 d4_bg: ",(F7.5))' )
+   call WRITE_PARALLEL ( FV_Atm(1)%flagstruct%d4_bg_top ,format='("FV3 d4_bg_top: ",(F7.5))' )
+   call WRITE_PARALLEL ( FV_Atm(1)%flagstruct%d4_bg_bot ,format='("FV3 d4_bg_bot: ",(F7.5))' )
    call WRITE_PARALLEL ( FV_Atm(1)%flagstruct%vtdm4 ,format='("FV3 vtdm4: ",(F7.5))' )
    call WRITE_PARALLEL ( FV_Atm(1)%flagstruct%d2_bg_k1 ,format='("FV3 d2_bg_k1: ",(F7.5))' )
    call WRITE_PARALLEL ( FV_Atm(1)%flagstruct%d2_bg_k2 ,format='("FV3 d2_bg_k2: ",(F7.5))' )
@@ -4981,19 +4977,21 @@ subroutine adiabatic_init(myDT,DEBUG_ARRAY,fac1)
    real(FVPRC), allocatable :: w_dt(:,:,:)
 
    integer :: nord
-   real(FVPRC):: dddmp, d4_bg, d2_bg, d_ext
+   real(FVPRC):: dddmp, d4_bg_top, d4_bg_bot, d2_bg, d_ext
 
 ! Save input damping state
    nord  = FV_Atm(1)%flagstruct%nord
    dddmp = FV_Atm(1)%flagstruct%dddmp
-   d4_bg = FV_Atm(1)%flagstruct%d4_bg
+   d4_bg_top = FV_Atm(1)%flagstruct%d4_bg_top
+   d4_bg_bot = FV_Atm(1)%flagstruct%d4_bg_bot
    d2_bg = FV_Atm(1)%flagstruct%d2_bg
    d_ext = FV_Atm(1)%flagstruct%d_ext
 
 ! Use 2nd order damping for spinup
    FV_Atm(1)%flagstruct%nord = 0
    FV_Atm(1)%flagstruct%dddmp = 0.2
-   FV_Atm(1)%flagstruct%d4_bg = 0.0
+   FV_Atm(1)%flagstruct%d4_bg_top = 0.0
+   FV_Atm(1)%flagstruct%d4_bg_bot = 0.0
    FV_Atm(1)%flagstruct%d2_bg = 0.0075
    FV_Atm(1)%flagstruct%d_ext = 0.02
 
@@ -5188,7 +5186,8 @@ subroutine adiabatic_init(myDT,DEBUG_ARRAY,fac1)
 ! Reset input damping parameters
      FV_Atm(1)%flagstruct%nord  = nord
      FV_Atm(1)%flagstruct%dddmp = dddmp
-     FV_Atm(1)%flagstruct%d4_bg = d4_bg
+     FV_Atm(1)%flagstruct%d4_bg_top = d4_bg_top
+     FV_Atm(1)%flagstruct%d4_bg_bot = d4_bg_bot
      FV_Atm(1)%flagstruct%d2_bg = d2_bg
      FV_Atm(1)%flagstruct%d_ext = d_ext
 

scripts/fv3_setup

@@ -253,45 +253,30 @@ ASKPROC:
 
 if ( $SITE == 'NCCS' ) then
 
-   set BUILT_ON_SLES15 = @BUILT_ON_SLES15@
-
-   if ("$BUILT_ON_SLES15" == "TRUE") then
-      echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
-      echo "   ${C2}cas  (Cascade Lake)${CN}"
-      echo "   ${C2}mil  (Milan)${CN} (default)"
-      echo " "
-      set MODEL = `echo $<`
-      set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
-      if ( .$MODEL == .) then
-         set MODEL = 'mil'
-      endif
-
-      if( $MODEL != 'mil' & \
-          $MODEL != 'cas' ) goto ASKPROC
-
-      if ($MODEL == 'mil') then
-         # we use 120 of 128 cores per node for even division
-         set NCPUS_PER_NODE = 120
-      else if ($MODEL == 'cas') then
-         # we use 40 of 46 cores per node for even division
-         set NCPUS_PER_NODE = 40
-      endif
-   else
-      echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
-      echo "   ${C2}cas  (Cascade Lake) (default)${CN}"
-      echo " "
-      set MODEL = `echo $<`
-      set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
-      if ( .$MODEL == .) then
-         set MODEL = 'cas'
-      endif
-
-      if( $MODEL != 'cas' ) goto ASKPROC
+   echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
+   echo "   ${C2}mil  (Milan)${CN} (default)"
+   echo "   ${C2}cas  (Cascade Lake)${CN}"
+   echo " "
+   set MODEL = `echo $<`
+   set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
+   if ( .$MODEL == .) then
+      set MODEL = 'mil'
+   endif
 
-      if ($MODEL == 'cas') then
-         # We use 40 of 46 cores per node for even division
-         set NCPUS_PER_NODE = 40
-      endif
+   if( $MODEL != 'mil' & \
+       $MODEL != 'cas' ) goto ASKPROC
+
+   if ($MODEL == 'mil') then
+      # We save a couple processes for the kernel
+      set NCPUS_PER_NODE = 126
+   else if ($MODEL == 'cas') then
+      # NCCS currently recommends that users do not run with
+      # 48 cores per node on SCU16 due to OS issues and
+      # recommends that CPU-intensive works run with 46 or less
+      # cores. As 45 is a multiple of 3, it's the best value
+      # that doesn't waste too much
+      #set NCPUS_PER_NODE = 48
+      set NCPUS_PER_NODE = 45
    endif
 
 else if ( $SITE == 'NAS' ) then
@@ -774,41 +759,13 @@ EOF
 if ( $SITE == 'NCCS' ) then
 
 cat >> $EXPDIR/SETENV.commands << EOF
-setenv I_MPI_SHM_HEAP_VSIZE 512
-setenv PSM2_MEMORY large
-EOF
-
-# Testing at NCCS showed these caused a crash at higher resolutions at
-# restart read
-if ( $USING_SINGULARITY == FALSE ) then
-
-cat >> $EXPDIR/SETENV.commands << EOF
-setenv I_MPI_EXTRA_FILESYSTEM 1
-setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs
-EOF
+setenv I_MPI_ADJUST_ALLREDUCE 12
+setenv I_MPI_ADJUST_GATHERV 3
 
-endif # if NOT Singularity
-
-# Testing by Bill Putman found these to be
-# useful flags with Intel MPI on SLES15 on the
-# Milan nodes.
-# Note 1: Testing by NCCS shows the PSM3 provider
-# runs on the Infiniband fabric. Tests show it runs
-# up to C720.
-# Note 2: When the Cascade Lakes are moved to
-# SLES15, these will need to be Milan-only flags
-# as Intel MPI will probably work just fine with
-# Intel chips.
-if ("$BUILT_ON_SLES15" == "TRUE") then
-cat >> $EXPDIR/SETENV.commands << EOF
 setenv I_MPI_FABRICS shm:ofi
 setenv I_MPI_OFI_PROVIDER psm3
-setenv I_MPI_ADJUST_GATHERV 3
-setenv I_MPI_ADJUST_ALLREDUCE 12
 EOF
 
-endif # if SLES15
-
 endif # if NCCS
 
 endif # if mpi