Merge branch 'add-perfstubs'

Author: jacobmerson

CMakeLists.txt

@@ -17,6 +17,8 @@ endif()
 find_package(MPI REQUIRED)
 #adios2 adds C and Fortran depending on how it was built
 find_package(ADIOS2 CONFIG 2.7.1 REQUIRED)
+# external packages (perfstubs)
+add_subdirectory(external)
 
 set(REDEV_HEADERS
   redev.h
@@ -36,7 +38,6 @@ set(REDEV_HEADERS
 
 set(REDEV_SOURCES
   redev.cpp
-  redev_profile.cpp
   redev_time.cpp
   redev_assert.cpp
   redev_strings.cpp
@@ -45,11 +46,12 @@ set(REDEV_SOURCES
 add_library(redev ${REDEV_SOURCES})
 target_compile_features(redev PUBLIC cxx_std_17)
 target_link_libraries(redev PRIVATE redev_git_version)
-target_link_libraries(redev PUBLIC adios2::cxx11_mpi MPI::MPI_C)
+target_link_libraries(redev PUBLIC adios2::cxx11_mpi MPI::MPI_C perfstubs)
 target_compile_options(redev PRIVATE -Werror=switch)
 if(HAS_ASAN)
   target_compile_options(redev PRIVATE -fsanitize=address -fno-omit-frame-pointer)
 endif()
+target_compile_definitions(redev PUBLIC PERFSTUBS_USE_TIMERS)
 
 include(CTest)
 if(BUILD_TESTING)
@@ -148,8 +150,6 @@ if(BUILD_TESTING)
     NAME2 app PROCS2 1 EXE2 ./test_setup_classPtn ARGS2 0)
   add_exe(test_query test_query.cpp)
   mpi_test(test_query_1p 1 ./test_query)
-  add_exe(test_profile test_profile.cpp)
-  mpi_test(test_profile_1p 1 ./test_profile)
   add_exe(test_send test_send.cpp)
   dual_mpi_test(TESTNAME test_send_3p TIMEOUT ${test_timeout}
     NAME1 rdv PROCS1 4 EXE1 ./test_send ARGS1 1

external/CMakeLists.txt

@@ -0,0 +1,6 @@
+include(FetchContent)
+
+FetchContent_Declare(perfstubs
+                     GIT_REPOSITORY https://github.com/UO-OACISS/perfstubs.git
+                     GIT_TAG master)
+FetchContent_MakeAvailable(perfstubs)

redev.cpp

@@ -30,9 +30,12 @@ namespace {
 namespace redev {
 
   //TODO consider moving the ClassPtn source to another file
-  ClassPtn::ClassPtn() {}
+  ClassPtn::ClassPtn() {
+    REDEV_FUNCTION_TIMER;
+  }
 
   ClassPtn::ClassPtn(MPI_Comm comm, const redev::LOs& ranks_, const ModelEntVec& ents) {
+    REDEV_FUNCTION_TIMER;
     REDEV_ALWAYS_ASSERT(comm != MPI_COMM_NULL);
     assert(ranks_.size() == ents.size());
     if( ! ModelEntDimsValid(ents) ) exit(EXIT_FAILURE);
@@ -64,6 +67,7 @@ namespace redev {
   }
 
   bool ClassPtn::ModelEntDimsValid(const ClassPtn::ModelEntVec& ents) const {
+    REDEV_FUNCTION_TIMER;
     auto badDim = [](ModelEnt e){ return (e.first<0 || e.first>3); };
     auto res = std::find_if(begin(ents), end(ents), badDim);
     if (res != std::end(ents)) {
@@ -183,15 +187,19 @@ namespace redev {
 
 
   //TODO consider moving the RCBPtn source to another file
-  RCBPtn::RCBPtn() {}
+  RCBPtn::RCBPtn() {
+    REDEV_FUNCTION_TIMER;
+  }
 
   RCBPtn::RCBPtn(redev::LO dim_)
     : dim(dim_) {
+    REDEV_FUNCTION_TIMER;
     assert(dim>0 && dim<=3);
   }
 
   RCBPtn::RCBPtn(redev::LO dim_, std::vector<int>& ranks_, std::vector<double>& cuts_)
     : dim(dim_), ranks(ranks_), cuts(cuts_) {
+    REDEV_FUNCTION_TIMER;
     assert(dim>0 && dim<=3);
   }
 
@@ -329,6 +337,7 @@ namespace redev {
 
   Redev::Redev(MPI_Comm comm, Partition ptn, ProcessType processType, bool noClients)
     : comm(comm), adios(comm), ptn(ptn), processType(processType), noClients(noClients) {
+    PERFSTUBS_INITIALIZE();
     REDEV_FUNCTION_TIMER;
     int isInitialized = 0;
     MPI_Initialized(&isInitialized);
@@ -337,6 +346,7 @@ namespace redev {
   }
   Redev::Redev(MPI_Comm comm, ProcessType processType, bool noClients)
     : comm(comm), adios(comm), ptn(), processType(processType), noClients(noClients) {
+      PERFSTUBS_INITIALIZE();
       REDEV_FUNCTION_TIMER;
       REDEV_ALWAYS_ASSERT(processType == ProcessType::Client);
       int isInitialized = 0;
@@ -346,6 +356,7 @@ namespace redev {
     }
 
   void AdiosChannel::Setup(adios2::IO& s2cIO, adios2::Engine& s2cEngine) {
+    REDEV_FUNCTION_TIMER;
     // initialize the partition on the client based on how it's set on the server
     if (process_type_ == ProcessType::Server) {
       std::ignore = SendPartitionTypeToClient(s2cIO, s2cEngine);
@@ -354,7 +365,6 @@ namespace redev {
       auto partition_index = SendPartitionTypeToClient(s2cIO, s2cEngine);
       ConstructPartitionFromIndex(partition_index);
     }
-    REDEV_FUNCTION_TIMER;
     CheckVersion(s2cEngine,s2cIO);
     auto status = s2cEngine.BeginStep();
     REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK);

redev.h

@@ -200,6 +200,7 @@ class Redev {
   CreateAdiosChannel(std::string name, adios2::Params params,
                      TransportType transportType = TransportType::BP4,
                      std::string path = {}) {
+    REDEV_FUNCTION_TIMER;
     if(RankParticipates()) {
       return AdiosChannel{
           adios,       comm, std::move(name), std::move(params), transportType,

redev_adios_channel.h

@@ -1,6 +1,7 @@
 #ifndef REDEV_REDEV_ADIOS_CHANNEL_H
 #define REDEV_REDEV_ADIOS_CHANNEL_H
 #include "redev_assert.h"
+#include "redev_profile.h"
 #include <adios2.h>
 
 namespace redev {
@@ -14,6 +15,7 @@ class AdiosChannel {
       : comm_(comm), process_type_(processType), partition_(partition)
 
   {
+    REDEV_FUNCTION_TIMER;
     MPI_Comm_rank(comm, &rank_);
     auto s2cName = path + name + "_s2c";
     auto c2sName = path + name + "_c2s";
@@ -73,10 +75,11 @@ class AdiosChannel {
         num_server_ranks_(o.num_server_ranks_),
         comm_(std::exchange(o.comm_, MPI_COMM_NULL)),
         process_type_(o.process_type_), rank_(o.rank_),
-        partition_(o.partition_) {}
+        partition_(o.partition_) {REDEV_FUNCTION_TIMER;}
   AdiosChannel operator=(AdiosChannel &&) = delete;
   // FIXME IMPL RULE OF 5
   ~AdiosChannel() {
+    REDEV_FUNCTION_TIMER;
     // NEED TO CHECK that the engine exists before trying to close it because it
     // could be in a moved from state
     if (s2c_engine_) {
@@ -88,6 +91,7 @@ class AdiosChannel {
   }
   template <typename T>
   [[nodiscard]] BidirectionalComm<T> CreateComm(std::string name, MPI_Comm comm) {
+    REDEV_FUNCTION_TIMER;
     // TODO, remove s2c/c2s destinction on variable names then use std::move
     // name
     if(comm != MPI_COMM_NULL) {
@@ -108,6 +112,7 @@ class AdiosChannel {
   // TODO s2c/c2s Engine/IO -> send/receive Engine/IO. This removes need for all
   // the switch statements...
   void BeginSendCommunicationPhase() {
+    REDEV_FUNCTION_TIMER;
     adios2::StepStatus status;
     switch (process_type_) {
     case ProcessType::Client:
@@ -130,6 +135,7 @@ class AdiosChannel {
     }
   }
   void BeginReceiveCommunicationPhase() {
+    REDEV_FUNCTION_TIMER;
     adios2::StepStatus status;
     switch (process_type_) {
     case ProcessType::Client:
@@ -142,6 +148,7 @@ class AdiosChannel {
     REDEV_ALWAYS_ASSERT(status == adios2::StepStatus::OK);
   }
   void EndReceiveCommunicationPhase() {
+    REDEV_FUNCTION_TIMER;
     switch (process_type_) {
     case ProcessType::Client:
       s2c_engine_.EndStep();

redev_assert.h

@@ -4,7 +4,7 @@
 #define REDEV_ALWAYS_ASSERT(cond) do {          \
   if (! (cond)) {                               \
     char omsg[2048];                            \
-    snprintf(omsg,200, "%s failed at %s + %d \n",    \
+    snprintf(omsg,2048, "%s failed at %s + %d \n",    \
       #cond, __FILE__, __LINE__);               \
     redev::Redev_Assert_Fail(omsg);             \
   }                                             \

redev_bidirectional_comm.h

@@ -17,23 +17,28 @@ template <class T> class BidirectionalComm {
   BidirectionalComm(std::unique_ptr<Communicator<T>> sender_,
                     std::unique_ptr<Communicator<T>> receiver_)
       : sender(std::move(sender_)), receiver(std::move(receiver_)) {
+    REDEV_FUNCTION_TIMER;
     REDEV_ALWAYS_ASSERT(sender != nullptr);
     REDEV_ALWAYS_ASSERT(receiver != nullptr);
   }
   void SetOutMessageLayout(LOs &dest, LOs &offsets) {
+    REDEV_FUNCTION_TIMER;
     REDEV_ALWAYS_ASSERT(sender != nullptr);
     sender->SetOutMessageLayout(dest, offsets);
   }
   InMessageLayout GetInMessageLayout() {
+    REDEV_FUNCTION_TIMER;
     REDEV_ALWAYS_ASSERT(receiver != nullptr);
     return receiver->GetInMessageLayout();
   }
 
   void Send(T *msgs, Mode mode = Mode::Deferred) {
+    REDEV_FUNCTION_TIMER;
     REDEV_ALWAYS_ASSERT(sender != nullptr);
     sender->Send(msgs, mode);
   }
   std::vector<T> Recv(Mode mode = Mode::Deferred) {
+    REDEV_FUNCTION_TIMER;
     REDEV_ALWAYS_ASSERT(receiver != nullptr);
     return receiver->Recv(mode);
   }

redev_channel.h

@@ -12,13 +12,16 @@ class Channel {
       : pimpl_{new ChannelModel<std::remove_cv_t<std::remove_reference_t<T>>>(
             std::forward<T>(channel))},
         send_communication_phase_active_(false),
-        receive_communication_phase_active_(false) {}
+        receive_communication_phase_active_(false) {
+        REDEV_FUNCTION_TIMER;
+        }
 
   // For cases where we may be interested in storing the comm variant rather
   // than the exact type this function can be used to reduce the runtime
   // overhead of converting from the variant to the explicit type back to the
   // variant
   template <typename T> [[nodiscard]] CommV CreateCommV(std::string name, MPI_Comm comm) {
+    REDEV_FUNCTION_TIMER;
     return pimpl_->CreateComm(std::move(name), comm,
                               InvCommunicatorTypeMap<T>::value);
   }
@@ -27,32 +30,39 @@ class Channel {
   // that expects a typed communicator to be created.
   template <typename T>
   [[nodiscard]] BidirectionalComm<T> CreateComm(std::string name, MPI_Comm comm) {
+    REDEV_FUNCTION_TIMER;
     return std::get<BidirectionalComm<T>>(CreateCommV<T>(std::move(name), comm));
   }
   void BeginSendCommunicationPhase() {
+    REDEV_FUNCTION_TIMER;
     REDEV_ALWAYS_ASSERT(InSendCommunicationPhase() == false);
     pimpl_->BeginSendCommunicationPhase();
     send_communication_phase_active_ = true;
   }
   void EndSendCommunicationPhase() {
+    REDEV_FUNCTION_TIMER;
     REDEV_ALWAYS_ASSERT(InSendCommunicationPhase() == true);
     pimpl_->EndSendCommunicationPhase();
     send_communication_phase_active_ = false;
   }
   void BeginReceiveCommunicationPhase() {
+    REDEV_FUNCTION_TIMER;
     REDEV_ALWAYS_ASSERT(InReceiveCommunicationPhase() == false);
     pimpl_->BeginReceiveCommunicationPhase();
     receive_communication_phase_active_ = true;
   }
   void EndReceiveCommunicationPhase() {
+    REDEV_FUNCTION_TIMER;
     REDEV_ALWAYS_ASSERT(InReceiveCommunicationPhase() == true);
     pimpl_->EndReceiveCommunicationPhase();
     receive_communication_phase_active_ = false;
   }
   [[nodiscard]] bool InSendCommunicationPhase() const noexcept {
+    REDEV_FUNCTION_TIMER;
     return send_communication_phase_active_;
   }
   [[nodiscard]] bool InReceiveCommunicationPhase() const noexcept {
+    REDEV_FUNCTION_TIMER;
     return receive_communication_phase_active_;
   }
 
@@ -88,6 +98,7 @@ class Channel {
     ~ChannelModel() noexcept final {}
     [[nodiscard]] CommV CreateComm(std::string &&name, MPI_Comm comm,
                                    CommunicatorDataType type) final {
+      REDEV_FUNCTION_TIMER;
       switch (type) {
       case CommunicatorDataType::INT8:
         return CommV{impl_.template CreateComm<
@@ -145,15 +156,19 @@ class Channel {
       return {};
     }
     void BeginSendCommunicationPhase() final {
+      REDEV_FUNCTION_TIMER;
       impl_.BeginSendCommunicationPhase();
     }
     void EndSendCommunicationPhase() final {
+      REDEV_FUNCTION_TIMER;
       impl_.EndSendCommunicationPhase();
     }
     void BeginReceiveCommunicationPhase() final {
+      REDEV_FUNCTION_TIMER;
       impl_.BeginReceiveCommunicationPhase();
     }
     void EndReceiveCommunicationPhase() final {
+      REDEV_FUNCTION_TIMER;
       impl_.EndReceiveCommunicationPhase();
     }
 

redev_comm.h

@@ -8,6 +8,7 @@
 #include <stddef.h>
 #include <type_traits> // is_same
 #include <adios2.h>
+#include "redev_time.h"
 
 namespace {
 void checkStep(adios2::StepStatus status) {