Merge PR #200 into development

* upstream/pr/200/head:
  encap: Consolidate MPIVectorEncaps into VectorEncaps.

Signed-off-by: Torbjörn Klatt <[email protected]>

Author: torbjoernk

examples/advection_diffusion/mpi_pfasst.cpp

@@ -20,7 +20,7 @@ using namespace std;
 #include <pfasst/controller/pfasst.hpp>
 #include <pfasst/mpi_communicator.hpp>
 #include <pfasst/encap/automagic.hpp>
-#include <pfasst/encap/mpi_vector.hpp>
+#include <pfasst/encap/vector.hpp>
 
 #include "advection_diffusion_sweeper.hpp"
 #include "spectral_transfer_1d.hpp"
@@ -61,7 +61,7 @@ namespace pfasst
         vector<size_t> ndofs = { ndofs_c, ndofs_f };
 
         auto build_level = [ndofs](size_t level) {
-          auto factory  = make_shared<MPIVectorFactory<double>>(ndofs[level]);
+          auto factory  = make_shared<VectorFactory<double>>(ndofs[level]);
           auto sweeper  = make_shared<AdvectionDiffusionSweeper<>>(ndofs[level]);
           auto transfer = make_shared<SpectralTransfer1D<>>();
 

include/pfasst/encap/mpi_vector.hpp

@@ -5,6 +5,11 @@
 #include <vector>
 using namespace std;
 
+#ifdef WITH_MPI
+#include "pfasst/mpi_communicator.hpp"
+using namespace pfasst::mpi;
+#endif
+
 #include "pfasst/encap/encapsulation.hpp"
 
 
@@ -24,6 +29,7 @@ namespace pfasst
         public Encapsulation<time>
     {
       public:
+
         //! @{
         VectorEncapsulation(const size_t size);
 
@@ -89,6 +95,29 @@ namespace pfasst
         virtual time norm0() const override;
         //! @}
 
+#ifdef WITH_MPI
+        //! @{
+        MPI_Request recv_request = MPI_REQUEST_NULL;
+        MPI_Request send_request = MPI_REQUEST_NULL;
+        //! @}
+
+        //! @{
+        inline MPICommunicator& as_mpi(ICommunicator* comm)
+        {
+          auto mpi = dynamic_cast<MPICommunicator*>(comm);
+          assert(mpi);
+          return *mpi;
+        }
+        //! @}
+
+        //! @{
+        virtual void post(ICommunicator* comm, int tag) override;
+        virtual void recv(ICommunicator* comm, int tag, bool blocking) override;
+        virtual void send(ICommunicator* comm, int tag, bool blocking) override;
+        virtual void broadcast(ICommunicator* comm) override;
+        //! @}
+#endif
+
     };
 
     /**

include/pfasst/encap/vector.hpp

@@ -1,10 +1,7 @@
-#include "pfasst/encap/vector.hpp"
-
 #include <algorithm>
 #include <cassert>
-#include <vector>
-using namespace std;
 
+#include "pfasst/encap/vector.hpp"
 
 namespace pfasst
 {
@@ -167,5 +164,82 @@ namespace pfasst
       return *y.get();
     }
 
+#ifdef WITH_MPI
+        template<typename scalar, typename time>
+    void VectorEncapsulation<scalar, time>::post(ICommunicator* comm, int tag)
+    {
+      auto& mpi = as_mpi(comm);
+      if (mpi.size() == 1) { return; }
+      if (mpi.rank() == 0) { return; }
+
+      int err = MPI_Irecv(this->data(), sizeof(scalar) * this->size(), MPI_CHAR,
+                          (mpi.rank() - 1) % mpi.size(), tag, mpi.comm, &recv_request);
+      if (err != MPI_SUCCESS) {
+        throw MPIError();
+      }
+    }
+
+    template<typename scalar, typename time>
+    void VectorEncapsulation<scalar, time>::recv(ICommunicator* comm, int tag, bool blocking)
+    {
+      auto& mpi = as_mpi(comm);
+      if (mpi.size() == 1) { return; }
+      if (mpi.rank() == 0) { return; }
+
+      int err;
+      if (blocking) {
+        MPI_Status stat;
+        err = MPI_Recv(this->data(), sizeof(scalar) * this->size(), MPI_CHAR,
+                       (mpi.rank() - 1) % mpi.size(), tag, mpi.comm, &stat);
+      } else {
+        MPI_Status stat;
+        err = MPI_Wait(&recv_request, &stat);
+      }
+
+      if (err != MPI_SUCCESS) {
+        throw MPIError();
+      }
+    }
+
+    template<typename scalar, typename time>
+    void VectorEncapsulation<scalar, time>::send(ICommunicator* comm, int tag, bool blocking)
+    {
+      auto& mpi = as_mpi(comm);
+      if (mpi.size() == 1) { return; }
+      if (mpi.rank() == mpi.size() - 1) { return; }
+
+      int err = MPI_SUCCESS;
+      if (blocking) {
+        err = MPI_Send(this->data(), sizeof(scalar) * this->size(), MPI_CHAR,
+                       (mpi.rank() + 1) % mpi.size(), tag, mpi.comm);
+      } else {
+        MPI_Status stat;
+        err = MPI_Wait(&send_request, &stat);
+        if (err != MPI_SUCCESS) {
+          throw MPIError();
+        }
+
+        err = MPI_Isend(this->data(), sizeof(scalar) * this->size(), MPI_CHAR,
+                        (mpi.rank() + 1) % mpi.size(), tag, mpi.comm, &send_request);
+      }
+
+      if (err != MPI_SUCCESS) {
+        throw MPIError();
+      }
+    }
+
+    template<typename scalar, typename time>
+    void VectorEncapsulation<scalar, time>::broadcast(ICommunicator* comm)
+    {
+      auto& mpi = as_mpi(comm);
+      int err = MPI_Bcast(this->data(), sizeof(scalar) * this->size(), MPI_CHAR,
+                          comm->size()-1, mpi.comm);
+
+      if (err != MPI_SUCCESS) {
+        throw MPIError();
+      }
+    }
+#endif
+
   }  // ::pfasst::encap
 }  // ::pfasst