Fix checkpoint action for multithreaded execution

Author: skuntz

src/sst/core/checkpointAction.cc

@@ -154,6 +154,7 @@ CheckpointAction::execute()
 void
 CheckpointAction::createCheckpoint(Simulation_impl* sim)
 {
+
     if ( 0 == rank_.rank && 0 == rank_.thread ) {
         const double now = sst_get_cpu_time();
         sim->getSimulationOutput().output(
@@ -212,9 +213,11 @@ CheckpointAction::createCheckpoint(Simulation_impl* sim)
     // No need to barrier here since rank 0 thread 0 will be the first
     // to execute in the loop below and everything else will wait
     for ( uint32_t r = 0; r < num_ranks.rank; ++r ) {
+
         if ( r == rank_.rank ) {
             // If this is my rank go ahead
             for ( uint32_t t = 0; t < num_ranks.thread; ++t ) {
+                
                 // If this is my thread go ahead
                 if ( t == rank_.thread ) {
                     sim->checkpoint_append_registry(directory + "/" + registry_name, filename);
@@ -253,13 +256,18 @@ CheckpointAction::createCheckpoint(Simulation_impl* sim)
 // SyncManager check whether a checkpoint needs to be generated
 SimTime_t
 CheckpointAction::check(SimTime_t current_time)
-{
+{ 
+#if 0
+    Simulation_impl* sim = Simulation_impl::getSimulation();
+    sim->getSimulationOutput().output(
+        "skk:T %d: checkpointAction.cc: check()\n", rank_.thread);
+#endif
     // The if-logic is a little weird, but it's trying to minimize the
     // number of branches in the normal case of no checkpoint being
     // initiated.  This will also handle the case where both a sim and
     // real-time trigger happened at the same time
     if ( (current_time == next_sim_time_) || generate_ ) {
-        Simulation_impl* sim = Simulation_impl::getSimulation();
+        Simulation_impl* sim = Simulation_impl::getSimulation();  
         createCheckpoint(sim);
         generate_ = false;
         // Only add to the simulation-interval checkpoint time if it
@@ -271,6 +279,11 @@ CheckpointAction::check(SimTime_t current_time)
     return next_sim_time_;
 }
 
+bool
+CheckpointAction::getCheckpoint()
+{
+    return generate_;
+}
 void
 CheckpointAction::setCheckpoint()
 {

src/sst/core/checkpointAction.h

@@ -76,6 +76,9 @@ class CheckpointAction : public Action
      */
     void insertIntoTimeVortex(Simulation_impl* sim);
 
+    /** Get checkpoint flag */
+    bool getCheckpoint();
+
     /** Generate a checkpoint next time check() is called */
     void setCheckpoint();
 

src/sst/core/realtime.cc

@@ -635,6 +635,12 @@ RealTimeManager::notifySignal()
     }
 
     if ( sig_alrm_from_os_ ) {
+#if 0
+        Output   sim_output = Simulation_impl::getSimulation()->getSimulationOutput();
+        RankInfo rank = Simulation_impl::getSimulation()->getRank();
+        sim_output.output(
+            "skk:T%d: realtime: sig_alrm_from_os\n", rank.thread);
+#endif
         sig_alrm_from_os_ = 0;
         if ( serial_exec_ ) {
             signal_actions_[SIGALRM]->execute();

src/sst/core/simulation.cc

@@ -1715,6 +1715,10 @@ Simulation_impl::getComponentObjectMap()
 void
 Simulation_impl::scheduleCheckpoint()
 {
+#if 0
+    sim_output.output(
+        "skk: simulation: scheduleCheckpoint\n");
+#endif
     checkpoint_action_->setCheckpoint();
 
     // Trigger checkpoint immediately in serial simulations

src/sst/core/sync/syncManager.cc

@@ -618,9 +618,28 @@ SyncManager::execute()
                 real_time_->performSignal(sig_end);
             else if ( signals_received ) {
                 if ( sig_usr ) real_time_->performSignal(sig_usr);
+#if 0
                 if ( sig_alrm ) real_time_->performSignal(sig_alrm);
+#else
+                if (sig_alrm) {
+                    //out.output("skk:syncmgr:execute: T%d: in sigalrm\n", rank_.thread);
+                    real_time_->performSignal(sig_alrm);
+                }
+#endif
+            }
+
+            // Check local checkpoint generate flag and set shared generate if needed. 
+            if (checkpoint_->getCheckpoint() == true) {
+                ckpt_generate_.store(1);
+            }
+            // Ensure everyone has written the mask before updating local generate_
+            ic_barrier_.wait();  
+            if (ckpt_generate_.load()) {
+                checkpoint_->setCheckpoint();
             }
             next_checkpoint_time = checkpoint_->check(getDeliveryTime());
+            ckpt_generate_.store(0);
+
 
             handleShutdown();           // Check if any thread set shutdown
             handleInteractiveConsole(); // Check of any thread set interactive console
@@ -730,6 +749,7 @@ SyncManager::addProfileTool(Profile::SyncProfileTool* tool)
     profile_tools_->addProfileTool(tool);
 }
 
+std::atomic<unsigned>     SyncManager::ckpt_generate_ { 0 };
 std::atomic<unsigned>     SyncManager::enter_interactive_mask_ { 0 };
 std::atomic<int>          SyncManager::current_ic_thread_ { 0 };
 std::atomic<int>          SyncManager::current_ic_state_ { 0 };

src/sst/core/sync/syncManager.h

@@ -190,6 +190,7 @@ class SyncManager : public Action
 
     RealTimeManager*                 real_time_;
     CheckpointAction*                checkpoint_;
+    static std::atomic<unsigned>     ckpt_generate_;
     static std::atomic<unsigned>     enter_interactive_mask_;
     static std::atomic<int>          current_ic_thread_;
     static std::atomic<int>          current_ic_state_;