mpi: improved debug logging and better MPIError

Signed-off-by: Torbjörn Klatt <[email protected]>

Author: torbjoernk

include/pfasst/mpi_communicator.hpp

@@ -19,8 +19,9 @@ namespace pfasst
       : public runtime_error
     {
       public:
-        explicit MPIError(const string& msg="");
+        MPIError(const string& msg="");
         virtual const char* what() const throw();
+        static MPIError from_code(const int err_code);
     };
 
 

src/pfasst/mpi_communicator_impl.hpp

@@ -16,6 +16,14 @@ namespace pfasst
       return (string("mpi error: ") + string(runtime_error::what())).c_str();
     }
 
+    MPIError MPIError::from_code(const int err_code)
+    {
+      char err_str[MPI_MAX_ERROR_STRING];
+      int err_len = 0;
+      MPI_Error_string(err_code, err_str, &err_len);
+      return MPIError("MPI Error: " + string(err_str, err_len) + " (code=" + to_string(err_code) + ")");
+    }
+
 
     MPICommunicator::MPICommunicator()
     {}
@@ -75,8 +83,10 @@ namespace pfasst
 
     void MPIStatus::set_converged(bool converged)
     {
-      CLOG(DEBUG, "Controller") << "set converged to " << boolalpha << converged;
+      CLOG(DEBUG, "Controller") << "set converged for rank " << this->comm->rank() << " to "
+                                << "'" << boolalpha << converged << "'";
       this->converged.at(this->comm->rank()) = converged;
+      assert(this->converged.at(this->comm->rank()) == converged);
     }
 
     bool MPIStatus::get_converged(int rank)
@@ -95,18 +105,11 @@ namespace pfasst
       if (mpi->size() == 1) { return; }
       if (mpi->rank() == mpi->size() - 1) { return; }
 
-      int iconverged = converged.at(mpi->rank()) ? 1 : 0;
-
+      int iconverged = converged.at(mpi->rank()) ? IStatus::CONVERGED : IStatus::NOT_CONVERGED;
       int dest_rank = (mpi->rank() + 1) % mpi->size();
-      CLOG(DEBUG, "Controller") << "sending status " << iconverged
-                                << " to " << dest_rank << " of communicator " << mpi->name();
-
-      int err = MPI_Send(&iconverged, sizeof(int), MPI_INT,
-                         dest_rank, 1, mpi->comm);
 
-      if (err != MPI_SUCCESS) {
-        throw MPIError();
-      }
+      int err = MPI_Send(&iconverged, sizeof(int), MPI_INT, dest_rank, 1, mpi->comm);
+      if (err != MPI_SUCCESS) { throw MPIError::from_code(err); }
     }
 
     void MPIStatus::recv()
@@ -115,34 +118,33 @@ namespace pfasst
       if (mpi->size() == 1) { return; }
       if (mpi->rank() == 0) { return; }
 
-      if (get_converged(mpi->rank()-1)) {
-        CLOG(DEBUG, "Controller") << "skipping status recv";
+      if (get_converged(mpi->rank() - 1)) {
+        CLOG(DEBUG, "Controller") << "skipping status recv as previous is stored as converged";
         return;
       }
 
       MPI_Status stat;
       int iconverged;
       int src_rank = (mpi->rank() - 1) % mpi->size();
-      int err = MPI_Recv(&iconverged, sizeof(iconverged), MPI_INT,
-                         src_rank, 1, mpi->comm, &stat);
-
-      if (err != MPI_SUCCESS) {
-        throw MPIError();
-      }
-
-      converged.at(mpi->rank()-1) = iconverged == 1 ? true : false;
+      int err = MPI_Recv(&iconverged, sizeof(iconverged), MPI_INT, src_rank, 1, mpi->comm, &stat);
+      if (err != MPI_SUCCESS) { throw MPIError::from_code(err); }
 
-      CLOG(DEBUG, "Controller") << "recieved status " << iconverged
-                                << " from rank " << src_rank << " of communicator " << mpi->name();
+      converged.at(mpi->rank() - 1) = (iconverged == IStatus::CONVERGED) ? true : false;
     }
   }  // ::pfasst::mpi
 }  // ::pfasst
 
 
 MAKE_LOGGABLE(MPI_Status, mpi_status, os)
 {
-  os << "MPI_Status(source=" << mpi_status.MPI_SOURCE << ", "
-                << "tag=" << mpi_status.MPI_TAG << ", "
-                << "error=" << mpi_status.MPI_ERROR << ")";
+  if (   mpi_status.MPI_TAG == MPI_ANY_TAG
+      && mpi_status.MPI_SOURCE == MPI_ANY_SOURCE
+      && mpi_status.MPI_ERROR == MPI_SUCCESS) {
+    os << "MPI_Status(empty)";
+  } else {
+    os << "MPI_Status(source=" << to_string(mpi_status.MPI_SOURCE) << ", "
+                  << "tag=" << to_string(mpi_status.MPI_TAG) << ", "
+                  << "error=" << to_string(mpi_status.MPI_ERROR) << ")";
+  }
   return os;
 }