Remove unecessary solver communication (#1344)

* Add solver timing blocks

* fixes to example

* Add boundary communication changes

* cpp-py-formatter

* changelog

* increase timeout

Author: lroberts36

.github/workflows/ci-extended.yml

@@ -90,7 +90,7 @@ jobs:
           export ASAN_OPTIONS=abort_on_error=1:fast_unwind_on_malloc=1
           export UBSAN_OPTIONS=print_stacktrace=0
           export LSAN_OPTIONS=detect_leaks=0
-          ctest -L regression -L ${{ matrix.parallel }} -LE perf-reg --timeout 3600
+          ctest -L regression -L ${{ matrix.parallel }} -LE perf-reg --timeout 7200
 
       # Test Ascent integration (only most complex setup with MPI and on device)
       - name: Ascent tests

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Current develop
 
 ### Added (new features/APIs/variables/...)
+- [[PR 1344]](https://github.com/parthenon-hpc-lab/parthenon/pull/1344) Add option to communicate single layer of ghosts, only communicate required two-level composite boundaries
 
 
 ### Changed (changing behavior/API/variables/...)

example/poisson_gmg/poisson_package.cpp

@@ -123,9 +123,10 @@ std::shared_ptr<StateDescriptor> Initialize(ParameterInput *pin) {
   // for the standard Poisson equation.
   pkg->AddField(D::name(), mD);
 
-  std::vector<MetadataFlag> flags{Metadata::Cell,        Metadata::Independent,
-                                  Metadata::FillGhost,   Metadata::WithFluxes,
-                                  Metadata::GMGRestrict, Metadata::GMGProlongate};
+  std::vector<MetadataFlag> flags{Metadata::Cell,          Metadata::Independent,
+                                  Metadata::FillGhost,     Metadata::WithFluxes,
+                                  Metadata::GMGRestrict,   Metadata::GMGProlongate,
+                                  Metadata::CommunicateOne};
   auto mflux_comm = Metadata(flags);
   if (prolong == "Linear") {
     mflux_comm.RegisterRefinementOps<ProlongateSharedLinear, RestrictAverage>();

src/bvals/comms/bnd_info.cpp

@@ -150,10 +150,14 @@ CalcIndices(const NeighborBlock &nb, MeshBlock *pmb,
                                 TopologicalOffsetK(el)};
   std::array<int, 3> block_offset = nb.offsets;
 
+  int communicated_ghosts = Globals::nghost;
+  if (!prores && loc.level() == nb.origin_loc.level() &&
+      v->IsSet(Metadata::CommunicateOne))
+    communicated_ghosts = 1;
   int interior_offset =
-      ir_type == IndexRangeType::BoundaryInteriorSend ? Globals::nghost : 0;
+      ir_type == IndexRangeType::BoundaryInteriorSend ? communicated_ghosts : 0;
   int exterior_offset =
-      ir_type == IndexRangeType::BoundaryExteriorRecv ? Globals::nghost : 0;
+      ir_type == IndexRangeType::BoundaryExteriorRecv ? communicated_ghosts : 0;
   if (prores) {
     // The coarse ghosts cover twice as much volume as the fine ghosts, so when working in
     // the exterior (i.e. ghosts) we must only go over the coarse ghosts that have

src/interface/metadata.hpp

@@ -128,6 +128,8 @@
   PARTHENON_INTERNAL_FOR_FLAG(CellMemAligned)                                            \
   /** Particles in a Swarm will not contain a persistent, unique id field **/            \
   PARTHENON_INTERNAL_FOR_FLAG(NoPersistentParticleIds)                                   \
+  /** Only communicate one layer of ghosts at same-to-same boundaries **/                \
+  PARTHENON_INTERNAL_FOR_FLAG(CommunicateOne)                                            \
   /************************************************/                                     \
   /** Vars specifying coordinates for visualization purposes **/                         \
   /** You can specify a single 3D var **/                                                \

src/utils/loop_utils.hpp

@@ -77,6 +77,7 @@ inline auto func_caller(F func, Args &&...args) -> typename std::enable_if<
 // need to be a template parameter, it could just be a function argument]
 template <BoundaryType bound = BoundaryType::any, class F>
 inline void ForEachBoundary(std::shared_ptr<MeshData<Real>> &md, F func) {
+  int fine_level = md->grid.logical_level;
   for (int block = 0; block < md->NumBlocks(); ++block) {
     auto &rc = md->GetBlockData(block);
     auto pmb = rc->GetBlockPointer();
@@ -125,8 +126,10 @@ inline void ForEachBoundary(std::shared_ptr<MeshData<Real>> &md, F func) {
       } else if constexpr (bound == BoundaryType::gmg_same) {
         if (v->IsSet(Metadata::FillGhost)) {
           for (auto &nb : *gmg_same) {
-            if (func_caller(func, pmb, rc, nb, v) == LoopControl::break_out) {
-              return;
+            if (pmb->loc.level() == fine_level || nb.loc.level() == fine_level) {
+              if (func_caller(func, pmb, rc, nb, v) == LoopControl::break_out) {
+                return;
+              }
             }
           }
         }