Fix balancing empty parts with metis

metis/apfMETISbalancer.cc

@@ -33,9 +33,9 @@ static void gatherGraph(
   std::vector<idx_t>& xadj, std::vector<idx_t>& adjncy,
   std::vector<int>& vtx_cts
 ) {
-  PCU_ALWAYS_ASSERT(PCU.Peers() > 1);
-  PCU_ALWAYS_ASSERT(owned_xadj.size() > 1);
-  PCU_ALWAYS_ASSERT(owned_adjncy.size() == size_t(owned_xadj.back()));
+  PCU_DEBUG_ASSERT(PCU.Peers() > 1);
+  PCU_DEBUG_ASSERT(!owned_xadj.empty());
+  PCU_DEBUG_ASSERT(owned_adjncy.size() == size_t(owned_xadj.back()));
   auto t0 = pcu::Time();
   int owned_vtx_ct = owned_xadj.size() - 1;
   int xadj_size = PCU.Add(owned_vtx_ct) + 1;
@@ -92,12 +92,11 @@ static void scatterPart(
   const std::vector<int>& vtx_cts,
   std::vector<idx_t>& owned_part, int n_owned
 ) {
-  PCU_ALWAYS_ASSERT(PCU.Peers() > 1);
-  PCU_ALWAYS_ASSERT(PCU.Self() != 0 || vtx_cts.size() == size_t(PCU.Peers()));
-  PCU_ALWAYS_ASSERT(
+  PCU_DEBUG_ASSERT(PCU.Peers() > 1);
+  PCU_DEBUG_ASSERT(PCU.Self() != 0 || vtx_cts.size() == size_t(PCU.Peers()));
+  PCU_DEBUG_ASSERT(
     std::accumulate(vtx_cts.begin(), vtx_cts.end(), 0UL) == part.size()
   );
-  PCU_ALWAYS_ASSERT(n_owned > 0);
   auto t0 = pcu::Time();
   owned_part.resize(n_owned);
   PCU.Begin();
@@ -171,7 +170,8 @@ static void remapPart(int nparts, std::vector<idx_t>& part, const std::vector<in
 
 void MetisBalancer::balance(MeshTag* weights, double tolerance) {
   PCU_ALWAYS_ASSERT(tolerance > 1.0);
-  if (mesh_->getPCU()->Peers() > APF_METIS_MAXRANKS) {
+  if (mesh_->getPCU()->Peers() == 1) return; // no work to be done.
+  else if (mesh_->getPCU()->Peers() > APF_METIS_MAXRANKS) {
     fail(
       "METIS called with > " STRINGIFY(APF_METIS_MAXRANKS)
       " procs, which is unsupported due to memory requirements\n"

test/CMakeLists.txt

@@ -112,6 +112,7 @@ if(ENABLE_ZOLTAN)
 endif()
 if(ENABLE_METIS)
   util_exe_func(mbalance mbalance.cc)
+  test_exe_func(mbalanceEmpty mbalanceEmpty.cc)
 endif()
 
 # Mesh improvement utilities

test/mbalanceEmpty.cc

@@ -0,0 +1,86 @@
+#include <iostream>
+#include <exception>
+#include <memory>
+#include <string>
+#include <apf.h>
+#include <apfMDS.h>
+#include <apfMesh2.h>
+#include <apfPartition.h>
+#include <gmi_mesh.h>
+#include <apfMETIS.h>
+#include <lionPrint.h>
+#include <pcu_util.h>
+
+namespace {
+void print_exception(const std::exception& e, int level = 0);
+}
+
+int main(int argc, char* argv[]) {
+  int retval = 0;
+  pcu::Init(&argc, &argv);
+  {
+  pcu::PCU PCU;
+  try {
+    if (argc != 5) {
+      if (PCU.Self() == 0)
+        std::cerr << "USAGE: <model.dmg> <mesh.smb> <inParts> <outMesh.smb>\n"
+          "\nwhere inParts < PCU.Peers()"
+          << std::endl;
+      throw std::runtime_error("invalid arguments");
+    }
+    lion_set_verbosity(1);
+    gmi_register_mesh();
+    // load model and mesh
+    int inParts = std::stoi(argv[3]);
+    if (inParts >= PCU.Peers()) {
+      throw std::runtime_error("inParts >= PCU.Peers()");
+    }
+    int group = PCU.Self() / inParts;
+    auto loadPCU = PCU.Split(group, 0);
+    gmi_model* model = gmi_load(argv[1]);
+    apf::Mesh2* m = nullptr;
+    if (group == 0) {
+      m = apf::loadMdsMesh(model, argv[2], loadPCU.get());
+      m->switchPCU(&PCU);
+    }
+    m = apf::expandMdsMesh(m, model, inParts, &PCU);
+    try {
+      std::unique_ptr<apf::Balancer> balancer(apf::makeMETISbalancer(m));
+      double imbalance = 1.1;
+      balancer->balance(nullptr, imbalance);
+    } catch (...) {
+      std::throw_with_nested(std::runtime_error("balancing failed"));
+    }
+    m->verify();
+    m->writeNative(argv[4]);
+    // destroy mds
+    m->destroyNative();
+    apf::destroyMesh(m);
+  } catch (const std::exception& e) {
+    if (PCU.Self() == 0) {
+      std::cerr << "ERROR: ";
+      print_exception(e);
+    }
+    retval = 1;
+  } catch (...) {
+    if (PCU.Self() == 0)
+      std::cerr << "Unknown exception occurred." << std::endl;
+    retval = 1;
+  }
+  } // PCU object scope
+  pcu::Finalize();
+  return retval;
+}
+
+namespace {
+
+void print_exception(const std::exception& e, int level) {
+  std::cerr << std::string(level * 2, ' ') << e.what() << '\n';
+  try {
+    std::rethrow_if_nested(e);
+  } catch (const std::exception& nestedE) {
+    print_exception(nestedE, level + 1);
+  } catch (...) {}
+}
+
+} // namespace

test/testing.cmake

@@ -442,6 +442,17 @@ if(ENABLE_METIS)
     3
   )
   set_test_depends(TESTS msplit_6 DEPENDS msplit_2)
+  mpi_test(mbalanceEmpty 4
+    ./mbalanceEmpty
+    "${MDIR}/pipe.dmg" "pipe.smb" 1
+    "pipe_mbe_.smb"
+  )
+  if(ENABLE_SIMMETRIX)
+    set_test_depends(
+      TESTS msplit_2 msplit_3 msplit_6 mBalanceEmpty
+      DEPENDS convert
+    )
+  endif()
 endif()
 if(ENABLE_ZOLTAN)
   mpi_test(refineX 2