Add an implementation of ParticleCopyPlan::doHandShake that uses one-sided communication from MPI-3

Author: atmyers

Src/Particle/AMReX_ParticleCommunication.H

@@ -10,6 +10,7 @@
 #include <AMReX_Scan.H>
 #include <AMReX_TypeTraits.H>
 #include <AMReX_MakeParticle.H>
+#include <AMReX_ParmParse.H>
 
 #include <map>
 
@@ -137,6 +138,8 @@ struct ParticleCopyPlan
         BL_PROFILE("ParticleCopyPlan::build");
 
         m_local = local;
+        ParmParse pp("particles");
+        pp.query("do_one_sided_comms", m_do_one_sided_comms);
 
         const int ngrow = 1;  // note - fix
 
@@ -294,14 +297,21 @@ private:
     // In the global version, we don't know who we'll receive from, so we
     // need to do some collective communication first.
     //
-    static void doHandShakeGlobal (const Vector<Long>& Snds, Vector<Long>& Rcvs);
+    static void doHandShakeReduceScatter (const Vector<Long>& Snds, Vector<Long>& Rcvs);
+
+    //
+    // Another version of the global handshake implemented with MPI-3
+    // one-sided communication.
+    //
+    static void doHandShakeOneSided (const Vector<Long>& Snds, Vector<Long>& Rcvs);
 
     //
     // Another version of the above that is implemented using MPI All-to-All
     //
     static void doHandShakeAllToAll (const Vector<Long>& Snds, Vector<Long>& Rcvs);
 
-    bool m_local;
+    bool m_local = false;
+    int m_do_one_sided_comms = 0;
 };
 
 struct GetSendBufferOffset

Src/Particle/AMReX_ParticleCommunication.cpp

@@ -263,7 +263,14 @@ void ParticleCopyPlan::doHandShake (const Vector<Long>& Snds, Vector<Long>& Rcvs
 {
     BL_PROFILE("ParticleCopyPlan::doHandShake");
     if (m_local) { doHandShakeLocal(Snds, Rcvs); }
-    else         { doHandShakeGlobal(Snds, Rcvs); }
+    else if (m_do_one_sided_comms) {
+#if defined(BL_USE_MPI3)
+        doHandShakeOneSided(Snds, Rcvs);
+#else
+        amrex::Abort("ParticleCopyPlan::doHandShake: particles.do_one_sided_comms=1 requires MPI-3");
+#endif
+    }
+    else         { doHandShakeReduceScatter(Snds, Rcvs); }
 }
 
 void ParticleCopyPlan::doHandShakeLocal (const Vector<Long>& Snds, Vector<Long>& Rcvs) const // NOLINT(readability-convert-member-functions-to-static)
@@ -333,7 +340,7 @@ void ParticleCopyPlan::doHandShakeAllToAll (const Vector<Long>& Snds, Vector<Lon
 #endif
 }
 
-void ParticleCopyPlan::doHandShakeGlobal (const Vector<Long>& Snds, Vector<Long>& Rcvs)
+void ParticleCopyPlan::doHandShakeReduceScatter (const Vector<Long>& Snds, Vector<Long>& Rcvs)
 {
 #ifdef AMREX_USE_MPI
     const int SeqNum = ParallelDescriptor::SeqNum();
@@ -381,6 +388,50 @@ void ParticleCopyPlan::doHandShakeGlobal (const Vector<Long>& Snds, Vector<Long>
 #endif
 }
 
+void ParticleCopyPlan::doHandShakeOneSided (const Vector<Long>& Snds, Vector<Long>& Rcvs)
+{
+#if defined(AMREX_USE_MPI) && defined(BL_USE_MPI3)
+    const int MyProc = ParallelContext::MyProcSub();
+    const int NProcs = ParallelContext::NProcsSub();
+
+    AMREX_ALWAYS_ASSERT(static_cast<int>(Snds.size()) == NProcs);
+    AMREX_ALWAYS_ASSERT(static_cast<int>(Rcvs.size()) == NProcs);
+
+    std::fill(Rcvs.begin(), Rcvs.end(), 0);
+
+    MPI_Win win;
+    BL_MPI_REQUIRE(MPI_Win_create(Rcvs.dataPtr(),
+                                  static_cast<MPI_Aint>(NProcs*sizeof(Long)),
+                                  sizeof(Long),
+                                  MPI_INFO_NULL,
+                                  ParallelContext::CommunicatorSub(),
+                                  &win));
+
+    BL_MPI_REQUIRE(MPI_Win_fence(0, win));
+
+    for (int i = 0; i < NProcs; ++i)
+    {
+        if (i == MyProc || Snds[i] == 0) { continue; }
+
+        BL_MPI_REQUIRE(MPI_Put(&Snds[i],
+                               1,
+                               ParallelDescriptor::Mpi_typemap<Long>::type(),
+                               i,
+                               MyProc,
+                               1,
+                               ParallelDescriptor::Mpi_typemap<Long>::type(),
+                               win));
+    }
+
+    BL_MPI_REQUIRE(MPI_Win_fence(0, win));
+    BL_MPI_REQUIRE(MPI_Win_free(&win));
+
+    AMREX_ASSERT(Rcvs[MyProc] == 0);
+#else
+    amrex::ignore_unused(Snds,Rcvs);
+#endif
+}
+
 void communicateParticlesFinish (const ParticleCopyPlan& plan)
 {
     BL_PROFILE("amrex::communicateParticlesFinish");