Work on yurlungur comments

Author: lroberts36

src/bvals/comms/bnd_id.cpp

@@ -43,7 +43,7 @@ namespace parthenon {
 BndId BndId::GetSend(MeshBlock *pmb, const NeighborBlock &nb,
                      std::shared_ptr<Variable<Real>> v, BoundaryType b_type,
                      int partition, int start_idx) {
-  // TODO: This needs to be fixed for unique buffer ids
+  // TODO(LFR): This needs to be fixed for unique buffer ids
   auto [send_gid, recv_gid, vlabel, loc, extra_id] = SendKey(pmb, nb, v, b_type, 0);
   BndId out;
   out.send_gid() = send_gid;

src/bvals/comms/build_boundary_buffers.cpp

@@ -117,7 +117,8 @@ void BuildBoundaryBufferSubset(std::shared_ptr<MeshData<Real>> &md,
 
     // Build send buffer (unless this is a receiving flux boundary)
     if constexpr (IsSender(BTYPE)) {
-      for (int id = 0; id <= (BTYPE == BoundaryType::gmg_restrict_send); ++id) {
+      pmesh->LockCommChannelNumbers(BTYPE);
+      for (int id = 0; id < pmesh->GetNumberOfCommChannels(BTYPE); ++id) {
         auto s_key = SendKey(pmb, nb, v, BTYPE, id);
         const int tag = pmesh->tag_map.GetTag(pmb, nb, id);
         if (buf_map.count(s_key) == 0)
@@ -130,7 +131,8 @@ void BuildBoundaryBufferSubset(std::shared_ptr<MeshData<Real>> &md,
     // Also build the non-local receive buffers here
     if constexpr (IsReceiver(BTYPE)) {
       if (sender_rank != receiver_rank) {
-        for (int id = 0; id <= (BTYPE == BoundaryType::gmg_restrict_recv); ++id) {
+        pmesh->LockCommChannelNumbers(BTYPE);
+        for (int id = 0; id < pmesh->GetNumberOfCommChannels(BTYPE); ++id) {
           auto r_key = ReceiveKey(pmb, nb, v, BTYPE, id);
           const int tag = pmesh->tag_map.GetTag(pmb, nb, id);
           if (buf_map.count(r_key) == 0)

src/bvals/comms/bvals_utils.hpp

@@ -54,11 +54,13 @@ inline Mesh::channel_key_t SendKey(const MeshBlock *pmb, const NeighborBlock &nb
   const int sender_id = pmb->gid;
   const int receiver_id = nb.gid;
   const int location_idx = nb.offsets.GetIdx();
-  int other = (nb.gid == pmb->gid && (btype == BoundaryType::gmg_restrict_recv ||
-                                      btype == BoundaryType::gmg_restrict_send))
-                  ? 1
-                  : 0;
-  other += 2 * id;
+
+  int gmg_self_comm = (nb.gid == pmb->gid && (btype == BoundaryType::gmg_restrict_recv ||
+                                              btype == BoundaryType::gmg_restrict_send))
+                          ? 1
+                          : 0;
+  int other =
+      gmg_self_comm + 2 * id; // Combine the id and gmg_self_comm into a single tag
   return {sender_id, receiver_id, pcv->label(), location_idx, other};
 }
 
@@ -68,11 +70,12 @@ inline Mesh::channel_key_t ReceiveKey(const MeshBlock *pmb, const NeighborBlock
   const int receiver_id = pmb->gid;
   const int sender_id = nb.gid;
   const int location_idx = nb.lcoord_trans.Transform(nb.offsets).GetReverseIdx();
-  int other = (nb.gid == pmb->gid && (btype == BoundaryType::gmg_restrict_recv ||
-                                      btype == BoundaryType::gmg_restrict_send))
-                  ? 1
-                  : 0;
-  other += 2 * id;
+  int gmg_self_comm = (nb.gid == pmb->gid && (btype == BoundaryType::gmg_restrict_recv ||
+                                              btype == BoundaryType::gmg_restrict_send))
+                          ? 1
+                          : 0;
+  int other =
+      gmg_self_comm + 2 * id; // Combine the id and gmg_self_comm into a single tag
   return {sender_id, receiver_id, pcv->label(), location_idx, other};
 }
 

src/bvals/comms/tag_map.cpp

@@ -16,6 +16,7 @@
 //========================================================================================
 
 #include <memory>
+#include <utility>
 
 #include "bnd_info.hpp"
 #include "bvals_utils.hpp"

src/interface/mesh_data.cpp

@@ -44,6 +44,15 @@ void MeshData<T>::SetMeshProperties(Mesh *pmesh) {
   ndim_ = pmesh == nullptr ? 0 : pmesh->ndim;
 }
 
+template <typename T>
+void MeshData<T>::SetBoundBufferId(BoundaryType btype, int id) {
+  PARTHENON_REQUIRE(id < pmy_mesh_->GetNumberOfCommChannels(btype),
+                    "Trying to set MeshData to communicate on a non-existent channel.");
+  // We do not enforce symmetry here between associated senders and
+  // receivers for maximum flexibility.
+  bound_buffer_ids_[btype] = id;
+}
+
 template class MeshData<Real>;
 
 } // namespace parthenon

src/interface/mesh_data.hpp

@@ -199,7 +199,7 @@ class MeshData {
   GridIdentifier grid;
   int partition{-1};
 
-  void SetBoundBufferId(BoundaryType btype, int id) { bound_buffer_ids_[btype] = id; }
+  void SetBoundBufferId(BoundaryType btype, int id);
 
   int GetBoundBufferId(BoundaryType btype) const {
     if (bound_buffer_ids_.count(btype)) return bound_buffer_ids_.at(btype);

src/mesh/mesh-gmg.cpp

@@ -22,6 +22,7 @@
 #include <iostream>
 #include <limits>
 #include <map>
+#include <memory>
 #include <numeric>
 #include <sstream>
 #include <string>

src/mesh/mesh.cpp

@@ -188,6 +188,8 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages,
     max_level = max_level_ref_;
   }
 
+  if (multigrid) SetNumberOfCommChannels(BoundaryType::gmg_restrict_send, 2);
+
   SetupMPIComms();
 
   RegisterLoadBalancing_(pin);

src/mesh/mesh.hpp

@@ -26,6 +26,7 @@
 #include <functional>
 #include <map>
 #include <memory>
+#include <set>
 #include <string>
 #include <tuple>
 #include <type_traits>
@@ -306,6 +307,37 @@ class Mesh {
   comm_buf_map_t boundary_comm_map;
   TagMap tag_map;
 
+  // Sets the number of communication buffers that can be in-flight concurrently
+  // for a given boundary type. This *must* be called before build boundary buffers
+  // is called internally, so use beyond the defaults with care
+  void SetNumberOfCommChannels(BoundaryType bound, std::size_t n_channels) {
+    if (locked_comm_channel_numbers_.count(bound))
+      PARTHENON_FAIL("Trying to reset the number of comm channels after boundary buffers "
+                     "have been set up.");
+    if (number_of_comm_channels_.count(bound) &&
+        number_of_comm_channels_[bound] > n_channels)
+      PARTHENON_WARN(
+          "You are reducing the number of comm channels from a previously set value.");
+    number_of_comm_channels_[bound] = n_channels;
+
+    // Need to set the complementary channels to the same value
+    if (!IsSender(bound))
+      number_of_comm_channels_[GetAssociatedSender(bound)] = n_channels;
+    if (!IsReceiver(bound))
+      number_of_comm_channels_[GetAssociatedReceiver(bound)] = n_channels;
+  }
+
+  void LockCommChannelNumbers(BoundaryType bound) {
+    locked_comm_channel_numbers_.insert(GetAssociatedSender(bound));
+    locked_comm_channel_numbers_.insert(GetAssociatedReceiver(bound));
+  }
+
+  std::size_t GetNumberOfCommChannels(BoundaryType bound) const {
+    if (number_of_comm_channels_.count(bound)) return number_of_comm_channels_.at(bound);
+    // We default to only having a single communication channel
+    return 1;
+  }
+
   std::shared_ptr<CoalescedComms> pcoalesced_comms;
 
   bool TryReallocCommBufferPools();
@@ -365,6 +397,8 @@ class Mesh {
   // the last 4x should be std::size_t, but are limited to int by MPI
   // Refinement tags used by MeshData checks
   ParArray1D<AmrTag> amr_tags;
+  std::map<BoundaryType, std::size_t> number_of_comm_channels_;
+  std::set<BoundaryType> locked_comm_channel_numbers_;
 
   std::vector<LogicalLocation> loclist;
 

src/mesh/meshblock.hpp

@@ -453,6 +453,7 @@ class MeshBlock : public std::enable_shared_from_this<MeshBlock> {
                                  std::forward<Args>(args)...);
   }
 
+  // Checks if the LogicalLocation of this block is a leaf logical location
   bool IsLeafLL() const { return is_leaf_ll_; }
 
  private: