Merge pull request #1259 from parthenon-hpc-lab/jmm/packs-per-rank

Add default num partitions per rank

Author: Yurlungur

doc/sphinx/src/tasks.rst

@@ -138,6 +138,68 @@ NOTE: Work remains to make the rest of
 Parthenon thread-safe, so it is currently required to use a ``ThreadPool``
 with one thread.
 
+Mesh data and Packing over collections of meshblocks
+-----------------------------------------------------
+
+The most common way to interact with collections of mesh blocks is the
+`MeshData` type, which collects ``MeshBlockData`` across a collection
+of blocks. It is through ``MeshData`` that sparse packs can be built,
+and multiple integrator stages, e.g., for a Runge-Kutte integrator,
+may be exposed.
+
+When building a set of task lists over a collection of mesh blocks,
+you may choose the number of blocks per task list by hand, but
+defaults can be set at runtime in one of two ways. The run time
+parameter
+
+.. code::
+
+  <parthenon/mesh>
+  pack_size = 1
+
+specifies the number of blocks per pack. For example, a value of 1
+indicates that each task list, even in synchronous regions, is over
+only 1 block. A value of -1 indicates it is all blocks per rank. You
+may choose -1 or any positive finite value.
+
+Alternatively, you may set
+
+.. code::
+
+  <parthenon/mesh>
+  packs_per_rank = 1
+
+which specifies the number of packs on a given rank. A value of 1
+indicates each task list contains all blocks on a given rank. A value
+of 2 indicates each list contains half, etc.
+
+If both of these parameters are set, then ``pack_size`` takes
+precedent. If neither are set, the default is ``pack_size=-1``, which
+is equivalent to ``packs_per_rank=1``.
+
+In a task list, you may access this information as, for example,
+
+.. code::
+
+  const int num_partitions = pmesh->DefaultNumPartitions();
+
+which will be equal to the pack size. Parthenon can automatically
+partition your list of meshblocks into smaller block lists and wrap it
+in a ``MeshData`` object with the ``GetOrAdd`` method. For example:
+
+.. code::
+
+  TaskRegion &tr = tc.AddRegion(num_partitions);
+  for (int i = 0; i < num_partitions; ++i) {
+    auto &tl = tr[i];
+    // mbase now points to the ith partition of the full block list
+    // and is the equivalent of the ``MeshData`` object named "base"
+    auto &mbase = pmesh->mesh_data.GetOrAdd("base", i);
+  }
+
+and ``GetOrAdd`` may be used on any "complete" ``MeshData`` object
+you've created on the entire block list.
+
 TaskQualifier
 -------------
 

src/mesh/mesh.cpp

@@ -79,14 +79,41 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages,
                     : false),
       nbnew(), nbdel(), step_since_lb(), gflag(), packages(packages),
       resolved_packages(ResolvePackages(packages)),
-      default_pack_size_(pin->GetOrAddInteger("parthenon/mesh", "pack_size", -1)),
       // private members:
       num_mesh_threads_(pin->GetOrAddInteger("parthenon/mesh", "num_threads", 1)),
       use_uniform_meshgen_fn_{true, true, true, true}, lb_flag_(true), lb_automatic_(),
       lb_manual_(), nslist(Globals::nranks), nblist(Globals::nranks),
       nref(Globals::nranks), nderef(Globals::nranks), rdisp(Globals::nranks),
       ddisp(Globals::nranks), bnref(Globals::nranks), bnderef(Globals::nranks),
       brdisp(Globals::nranks), bddisp(Globals::nranks) {
+  // pack size
+  bool pack_size_exists = pin->DoesParameterExist("parthenon/mesh", "pack_size");
+  bool num_partitions_exists =
+      pin->DoesParameterExist("parthenon/mesh", "packs_per_rank");
+  // If both exists, the assumption is that packs_per_rank was added later on purpose (as
+  // pack_size existed first) so the new value should take precedent.
+  if (pack_size_exists && num_partitions_exists) {
+    use_pack_size_ = false;
+    default_num_packs_ = pin->GetInteger("parthenon/mesh", "packs_per_rank");
+    auto pack_size = pin->GetInteger("parthenon/mesh", "pack_size");
+    bool are_both_default = (default_num_packs_ == 1) && (pack_size == -1);
+    if (!are_both_default && (Globals::my_rank == 0)) {
+      PARTHENON_WARN("Both pack_size and packs_per_rank set to non default values! "
+                     "New packs_per_rank takes precedent.");
+    }
+    // Only one or none is set
+  } else {
+    if (pack_size_exists) {
+      use_pack_size_ = true;
+      default_pack_size_ = pin->GetInteger("parthenon/mesh", "pack_size");
+      // use packs_per_rank (and set default value if not set)
+    } else {
+      use_pack_size_ = false;
+      default_num_packs_ =
+          std::max(1, pin->GetOrAddInteger("parthenon/mesh", "packs_per_rank", 1));
+    }
+  }
+
   // Allow for user overrides to default Parthenon functions
   if (app_in->InitUserMeshData != nullptr) {
     InitUserMeshData = app_in->InitUserMeshData;

src/mesh/mesh.hpp

@@ -160,11 +160,19 @@ class Mesh {
   void LoadBalancingAndAdaptiveMeshRefinement(ParameterInput *pin,
                                               ApplicationInput *app_in);
   int DefaultPackSize() {
-    return default_pack_size_ < 1 ? std::max(static_cast<int>(block_list.size()), 1)
-                                  : default_pack_size_;
+    if (use_pack_size_) {
+      return default_pack_size_ < 1 ? std::max(static_cast<int>(block_list.size()), 1)
+                                    : default_pack_size_;
+    } else {
+      return partition::partition_impl::IntCeil(block_list.size(), default_num_packs_);
+    }
   }
   int DefaultNumPartitions() {
-    return partition::partition_impl::IntCeil(block_list.size(), DefaultPackSize());
+    if (use_pack_size_) {
+      return partition::partition_impl::IntCeil(block_list.size(), DefaultPackSize());
+    } else {
+      return std::min(default_num_packs_, block_list.size());
+    }
   }
 
   const std::vector<std::shared_ptr<BlockListPartition>> &
@@ -314,7 +322,9 @@ class Mesh {
   int lb_interval_;
 
   // size of default MeshBlockPacks
+  bool use_pack_size_;
   int default_pack_size_;
+  std::size_t default_num_packs_;
 
   int gmg_min_logical_level_ = 0;