pfasst: Add tags to status send/recv. Fix malformed MPI send/recv.

Author: memmett

include/pfasst/controller/pfasst.hpp

@@ -80,6 +80,7 @@ namespace pfasst
        * @param[in] level_iter level iterator providing information to compute the communication tag
        */
       virtual int tag(LevelIter level_iter);
+      virtual int stag(LevelIter level_iter);
 
       /**
        * Post current status and values to next processor.

include/pfasst/interfaces.hpp

@@ -143,9 +143,9 @@ namespace pfasst
       //! @}
 
       //! @{
-      virtual void post() = 0;
-      virtual void send() = 0;
-      virtual void recv() = 0;
+      virtual void post(int tag) = 0;
+      virtual void send(int tag) = 0;
+      virtual void recv(int tag) = 0;
       //! @}
   };
 
@@ -222,7 +222,7 @@ namespace pfasst
        * Perform one SDC sweep/iteration.
        *
        * Compute a correction and update solution values.
-       * Note that this function can assume that valid function values exist from a previous 
+       * Note that this function can assume that valid function values exist from a previous
        * pfasst::ISweeper::sweep() or pfasst::ISweeper::predict().
        */
       virtual void sweep() = 0;
@@ -314,7 +314,7 @@ namespace pfasst
        *
        * @param[in,out] dst            sweeper to interpolate onto (i.e. fine level)
        * @param[in]     src            sweeper to interpolate from (i.e. coarse level)
-       * @param[in]     interp_initial `true` if a delta for the initial condtion should also be 
+       * @param[in]     interp_initial `true` if a delta for the initial condtion should also be
        *   computed (PFASST)
        */
       virtual void interpolate(shared_ptr<ISweeper<time>> dst,

include/pfasst/mpi_communicator.hpp

@@ -105,9 +105,9 @@ namespace pfasst
         virtual void clear() override;
         virtual void set_converged(bool converged) override;
         virtual bool get_converged(int rank) override;
-        virtual void post();
-        virtual void send();
-        virtual void recv();
+        virtual void post(int tag) override;
+        virtual void send(int tag) override;
+        virtual void recv(int tag) override;
     };
   }  // ::pfasst::mpi
 }  // ::pfasst

src/pfasst/controller/pfasst_impl.hpp

@@ -97,13 +97,12 @@ namespace pfasst
     perform_sweeps(l.level);
 
     if (l == this->finest() && fine->converged()) {
-      this->comm->status->set_converged(true);
+      bool previous_converged = this->comm->status->previous_is_iterating();
+      this->comm->status->set_converged(!previous_converged);
     }
 
     fine->send(comm, tag(l), false);
-
     trns->restrict(crse, fine, true);
-
     trns->fas(this->get_time_step(), crse, fine);
     crse->save();
 
@@ -136,10 +135,10 @@ namespace pfasst
     if (this->comm->status->previous_is_iterating()) {
       crse->recv(comm, tag(level_iter), true);
     }
-    this->comm->status->recv();
+    this->comm->status->recv(stag(level_iter));
     this->perform_sweeps(level_iter.level);
+    this->comm->status->send(stag(level_iter));
     crse->send(comm, tag(level_iter), true);
-    this->comm->status->send();
     return level_iter + 1;
   }
 
@@ -205,13 +204,19 @@ namespace pfasst
    * @internals
    * A simple formula is used with current level index \\( L \\) (provided by @p level_iter) and
    * current iteration number \\( I \\):
-   * \\[ L * 10000 + I + 10 \\]
+   * \\[ (L+1) * 10000 + I \\]
    * @endinternals
    */
   template<typename time>
   int PFASST<time>::tag(LevelIter level_iter)
   {
-    return level_iter.level * 10000 + this->get_iteration() + 10;
+    return (level_iter.level+1) * 10000 + this->get_iteration();
+  }
+
+  template<typename time>
+  int PFASST<time>::stag(LevelIter level_iter)
+  {
+    return level_iter.level * 1000 + this->get_iteration();
   }
 
   /**
@@ -222,7 +227,7 @@ namespace pfasst
   void PFASST<time>::post()
   {
     if (this->comm->status->previous_is_iterating()) {
-      this->comm->status->post();
+      this->comm->status->post(0);
       for (auto l = this->coarsest() + 1; l <= this->finest(); ++l) {
         l.current()->post(comm, tag(l));
       }

src/pfasst/mpi_communicator_impl.hpp

@@ -96,12 +96,13 @@ namespace pfasst
       return this->converged.at(rank);
     }
 
-    void MPIStatus::post()
+    void MPIStatus::post(int tag)
     {
       // noop: send/recv for status info is blocking
+      UNUSED(tag);
     }
 
-    void MPIStatus::send()
+    void MPIStatus::send(int tag)
     {
       // don't send forward if: single processor run, or we're the last processor
       if (mpi->size() == 1) { return; }
@@ -110,13 +111,13 @@ namespace pfasst
       int iconverged = converged.at(mpi->rank()) ? IStatus::CONVERGED : IStatus::NOT_CONVERGED;
       int dest_rank = (mpi->rank() + 1) % mpi->size();
 
-      CLOG(DEBUG, "Controller") << "sending converged status to rank " << dest_rank << " with tag '1': " << ((bool)iconverged == IStatus::CONVERGED);
-      int err = MPI_Send(&iconverged, sizeof(int), MPI_INT, dest_rank, 1, mpi->comm);
+      CLOG(DEBUG, "Controller") << "sending converged status to rank " << dest_rank << " with tag " << tag << ": " << ((bool)iconverged == IStatus::CONVERGED);
+      int err = MPI_Send(&iconverged, 1, MPI_INT, dest_rank, tag, mpi->comm);
       check_mpi_error(err);
-      CLOG(DEBUG, "Controller") << "sent converged status";
+           CLOG(DEBUG, "Controller") << "sent converged status";
     }
 
-    void MPIStatus::recv()
+    void MPIStatus::recv(int tag)
     {
       // don't recv if: single processor run, or we're the first processor
       if (mpi->size() == 1) { return; }
@@ -130,10 +131,10 @@ namespace pfasst
       MPI_Status stat = MPI_Status_factory();
       int iconverged = IStatus::NOT_CONVERGED;
       int src_rank = (mpi->rank() - 1) % mpi->size();
-      CLOG(DEBUG, "Controller") << "recieving converged status from rank " << src_rank << " with tag '1'";
-      int err = MPI_Recv(&iconverged, sizeof(iconverged), MPI_INT, src_rank, 1, mpi->comm, &stat);
+      //      CLOG(DEBUG, "Controller") << "receiving converged status from rank " << src_rank << " with tag '1'";
+      int err = MPI_Recv(&iconverged, 1, MPI_INT, src_rank, tag, mpi->comm, &stat);
       check_mpi_error(err);
-      CLOG(DEBUG, "Controller") << "recieved converged status from rank " << src_rank << " with tag '1': " << ((bool)iconverged == IStatus::CONVERGED);
+      CLOG(DEBUG, "Controller") << "received converged status from rank " << src_rank << " with tag "<< tag << ": " << ((bool)iconverged == IStatus::CONVERGED);
 
       converged.at(mpi->rank() - 1) = (iconverged == IStatus::CONVERGED) ? true : false;
     }