Merge branch 'origin/oksanaguba/homme/xx-ttype9' into master (PR #4682)

Adding ttype9 and dcmip2012 tests 2.1, 2.2, and 3 to the bfb suite.
Converting some of the tests to ttype9.

Not yet added is a limiter call between ttype9 stages.

There will be another PR to move limiter from CaarFunctorImpl
and to verify it is bfb with F code.

[nonbfb] for homme bfb suite for theta-fhs3 (now with ttype9)
and new baselines for theta-fdc tests.

Author: oksanaguba

components/homme/src/preqx_kokkos/cxx/HyperviscosityFunctorImpl.hpp

@@ -11,6 +11,7 @@
 #include "SimulationParams.hpp"
 #include "KernelVariables.hpp"
 #include "SphereOperators.hpp"
+#include "PhysicalConstants.hpp"
 
 #include <memory>
 

components/homme/src/preqx_kokkos/cxx/cxx_f90_interface_preqx.cpp

@@ -39,7 +39,8 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option,
                                const int& hypervis_order, const int& hypervis_subcycle, const double& hypervis_scaling,
                                const int& ftype, const bool& prescribed_wind, const bool& moisture, const bool& disable_diagnostics,
                                const bool& use_cpstar, const int& transport_alg,
-                               const int& dt_remap_factor, const int& dt_tracer_factor)
+                               const int& dt_remap_factor, const int& dt_tracer_factor,
+                               const double& rearth)
 {
   // Check that the simulation options are supported. This helps us in the future, since we
   // are currently 'assuming' some option have/not have certain values. As we support for more
@@ -68,7 +69,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option,
     params.remap_alg = RemapAlg::PPM_FIXED_MEANS;
   }
   if (time_step_type==5) {
-    params.time_step_type = TimeStepType::ULLRICH_RK35;;
+    params.time_step_type = TimeStepType::ttype5;
   }
 
   params.limiter_option                = limiter_option;
@@ -92,6 +93,7 @@ void init_simulation_params_c (const int& remap_alg, const int& limiter_option,
   params.moisture                      = (moisture ? MoistDry::MOIST : MoistDry::DRY);
   params.use_cpstar                    = use_cpstar;
   params.transport_alg                 = transport_alg;
+  params.rearth                        = rearth;
 
   //set nu_ratios values
   if (params.nu != params.nu_div) {
@@ -236,7 +238,7 @@ void init_elements_c (const int& num_elems)
   const SimulationParams& params = c.get<SimulationParams>();
 
   const bool consthv = (params.hypervis_scaling==0.0);
-  e.init (num_elems, consthv, /* alloc_gradphis = */ false);
+  e.init (num_elems, consthv, /* alloc_gradphis = */ false, params.rearth);
 
   // Init also the tracers structure
   Tracers& t = c.create<Tracers> ();

components/homme/src/preqx_kokkos/cxx/prim_advance_exp.cpp

@@ -51,7 +51,7 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost
 
   // Perform time-advance
   switch (params.time_step_type) {
-    case TimeStepType::ULLRICH_RK35:
+    case TimeStepType::ttype5:
       // Perform RK stages
       u3_5stage_timestep(tl, dt, eta_ave_w);
       break;
@@ -71,13 +71,11 @@ void prim_advance_exp (TimeLevel& tl, const Real dt, const bool compute_diagnost
   }
 #endif
 
-  if (!is_implicit(params.time_step_type)) {
-    // Get and run the HVF
-    HyperviscosityFunctor& functor = Context::singleton().get<HyperviscosityFunctor>();
-    GPTLstart("tl-ae advance_hypervis_dp");
-    functor.run(tl.np1,dt,eta_ave_w);
-    GPTLstop("tl-ae advance_hypervis_dp");
-  }
+  // Get and run the HVF
+  HyperviscosityFunctor& functor = Context::singleton().get<HyperviscosityFunctor>();
+  GPTLstart("tl-ae advance_hypervis_dp");
+  functor.run(tl.np1,dt,eta_ave_w);
+  GPTLstop("tl-ae advance_hypervis_dp");
 
 #ifdef ENERGY_DIAGNOSTICS
   if (compute_diagnostics) {

components/homme/src/preqx_kokkos/preqx_f2c_mod.F90

@@ -12,14 +12,14 @@ subroutine init_simulation_params_c (remap_alg, limiter_option, rsplit, qsplit,
                                        hypervis_order, hypervis_subcycle, hypervis_scaling,          &
                                        ftype, prescribed_wind, moisture,                             &
                                        disable_diagnostics, use_cpstar, transport_alg, &
-                                       dt_remap_factor, dt_tracer_factor) bind(c)
+                                       dt_remap_factor, dt_tracer_factor, rearth) bind(c)
     use iso_c_binding, only: c_int, c_bool, c_double, c_ptr
     !
     ! Inputs
     !
     integer(kind=c_int),  intent(in) :: remap_alg, limiter_option, rsplit, qsplit, time_step_type
     integer(kind=c_int),  intent(in) :: state_frequency, qsize, dt_remap_factor, dt_tracer_factor
-    real(kind=c_double),  intent(in) :: nu, nu_p, nu_q, nu_s, nu_div, nu_top, hypervis_scaling
+    real(kind=c_double),  intent(in) :: nu, nu_p, nu_q, nu_s, nu_div, nu_top, hypervis_scaling, rearth
     integer(kind=c_int),  intent(in) :: hypervis_order, hypervis_subcycle, ftype, transport_alg
     logical(kind=c_bool), intent(in) :: prescribed_wind, moisture, disable_diagnostics, use_cpstar
   end subroutine init_simulation_params_c

components/homme/src/preqx_kokkos/prim_driver_mod.F90

@@ -8,6 +8,7 @@ module prim_driver_mod
   use element_mod,          only : element_t
   use prim_driver_base,     only : deriv1, smooth_topo_datasets
   use prim_cxx_driver_base, only : prim_init1, prim_finalize
+  use physical_constants, only : rearth
 
   implicit none
 
@@ -88,7 +89,7 @@ subroutine prim_create_c_data_structures (tl, hvcoord, mp)
                                    LOGICAL(disable_diagnostics,c_bool),                           &
                                    LOGICAL(use_cpstar==1,c_bool),                                 &
                                    transport_alg,                                                 &
-                                   dt_remap_factor, dt_tracer_factor)
+                                   dt_remap_factor, dt_tracer_factor, rearth)
 
     ! Initialize time level structure in C++
     call init_time_level_c(tl%nm1, tl%n0, tl%np1, tl%nstep, tl%nstep0)

components/homme/src/share/cxx/ComposeTransportImplTrajectory.cpp

@@ -435,7 +435,7 @@ void ComposeTransportImpl::calc_trajectory (const int np1, const Real dt) {
     const int num_phys_lev = this->num_phys_lev;
     const auto m_sphere_cart = geo.m_sphere_cart;
     //todo get scale_factor into PhysicalConstants
-    const auto scale_factor = m_data.geometry_type == 1 ? 1 : PhysicalConstants::rearth;
+    const auto scale_factor = m_data.geometry_type == 1 ? 1 : geo.m_rearth;
     const auto m_dep_pts = m_data.dep_pts;
     const auto calc_departure_point = KOKKOS_LAMBDA (const MT& team) {
       KernelVariables kv(team, tu_ne);

components/homme/src/share/cxx/Elements.cpp

@@ -13,13 +13,14 @@
 namespace Homme {
 
 void Elements::init(const int num_elems, const bool consthv, const bool alloc_gradphis,
+                    const Real rearth,
                     const bool alloc_sphere_coords) {
   // Sanity check
   assert (num_elems>0);
 
   m_num_elems = num_elems;
 
-  m_geometry.init(num_elems,consthv,alloc_gradphis,alloc_sphere_coords);
+  m_geometry.init(num_elems,consthv,alloc_gradphis,rearth,alloc_sphere_coords);
   m_state.init(num_elems);
   m_derived.init(num_elems);
   m_forcing.init(num_elems);

components/homme/src/share/cxx/Elements.hpp

@@ -42,6 +42,7 @@ class Elements {
   int num_elems () const { return m_num_elems; }
 
   void init (const int num_elems, const bool consthv, const bool alloc_gradphis,
+             const Real rearth,
              const bool alloc_sphere_coords=false);
   void randomize (const int seed, const Real max_pressure = 1.0);
   void randomize (const int seed, const Real max_pressure, const Real ps0, const Real hyai0);

components/homme/src/share/cxx/ElementsGeometry.cpp

@@ -17,13 +17,17 @@
 namespace Homme {
 
 void ElementsGeometry::init(const int num_elems, const bool consthv, const bool alloc_gradphis,
+                            const Real rearth,
                             const bool alloc_sphere_coords) {
   // Sanity check
   assert (num_elems>0);
 
   m_num_elems = num_elems;
   m_consthv   = consthv;
 
+  assert( rearth > 0);
+  m_rearth     = rearth;
+  
   // Coriolis force
   m_fcor = ExecViewManaged<Real * [NP][NP]>("FCOR", m_num_elems);
 

components/homme/src/share/cxx/ElementsGeometry.hpp

@@ -48,7 +48,10 @@ class ElementsGeometry {
 
   ElementsGeometry() : m_num_elems(0) {}
 
+  Real m_rearth;
+
   void init (const int num_elems, const bool consthv, const bool alloc_gradphis,
+             const Real rearth,
              const bool alloc_sphere_coords=false);
 
   void randomize (const int seed);
@@ -69,7 +72,7 @@ class ElementsGeometry {
 
 private:
   bool m_consthv;
-  int m_num_elems;
+  int  m_num_elems;
 };
 
 } // Homme