Initial support for multithreading in debug console. Sync manager handles checking if enter interactive console is set and switching between threads. New console commands are info (get thread and rank) for current or all threads and thread <threadID> to switch to a new thread.

Author: skuntz

src/sst/core/impl/interactive/simpleDebug.cc

@@ -50,6 +50,10 @@ SimpleDebugger::SimpleDebugger(Params& params) :
             [this](std::vector<std::string>& tokens) { cmd_help(tokens); } },
         { "verbose", "v", "[mask]: set verbosity mask or print if no mask specified", ConsoleCommandGroup::GENERAL,
             [this](std::vector<std::string>& tokens) { cmd_verbose(tokens); } },
+        { "info", "info", "\"current\"|\"all\" print summary for current thread or all threads", ConsoleCommandGroup::GENERAL,
+            [this](std::vector<std::string>& tokens) { cmd_info(tokens); } },
+        { "thread", "thd", "[threadID]: switch to specified thread ID", ConsoleCommandGroup::GENERAL,
+            [this](std::vector<std::string>& tokens) { cmd_thread(tokens); } },
         { "confirm", "cfm", "<true/false>: set confirmation requests on (default) or off", ConsoleCommandGroup::GENERAL,
             [this](std::vector<std::string>& tokens) { cmd_setConfirm(tokens); } },
         { "pwd", "pwd", "print the current working directory in the object map", ConsoleCommandGroup::NAVIGATION,
@@ -216,15 +220,52 @@ SimpleDebugger::~SimpleDebugger()
 }
 
 void
+SimpleDebugger::summary()
+{
+#if 0
+    RankInfo info = getRank();
+    RankInfo nRanks = getNumRanks();
+    std::count << "\n(Rank:" << info.rank << " / " << nRanks.rank
+        << " Thread:" << info.thread << "/" << nRanks.thread
+        << ")\n";
+    std::cout << " -- Trigger Status\n";
+#endif
+    
+    std::cout << " -- Component Summary\n";
+#if 0
+    std::vector<std::string> tokens;
+    if (nullptr == obj_) {
+        obj_ = getComponentObjectMap();
+    }
+    cmd_ls(tokens);
+#else
+    SST::Core::Serialization::ObjectMap* baseObj = getComponentObjectMap();
+    auto& vars = baseObj->getVariables();
+    for (auto& x : vars) {
+        if (x.second->isFundamental()) {
+            std::cout << x.first << " = " << x.second->get() << " (" << x.second->getType() << ")" << std::endl;
+        }
+        else {
+            std::cout << x.first.c_str() << "/ (" << x.second->getType() << ")\n";
+        }
+    }
+#endif
+}
+
+int
 SimpleDebugger::execute(const std::string& msg)
 {
-    printf("Entering interactive mode at time %" PRI_SIMTIME " \n", getCurrentSimCycle());
+    RankInfo info = getRank();
+    RankInfo nRanks = getNumRanks();
+    printf("\n---- Rank%d:Thread%d: Entering interactive mode at time %" PRI_SIMTIME " \n", info.rank, info.thread, getCurrentSimCycle());
     printf("%s\n", msg.c_str());
 
     if ( nullptr == obj_ ) {
         obj_ = getComponentObjectMap();
     }
     done = false;
+    retState = -1;
+    
 
     // Select the input source and next command line
     std::string line;
@@ -236,6 +277,9 @@ SimpleDebugger::execute(const std::string& msg)
             // Logging disable has edge cases for stack push/pop
             bool squashLogging = false;
 
+            // User input prompt
+            std::cout << "R" << info.rank << ":T" << info.thread << "> " << std::flush;
+
             if ( !injectedCommand.str().empty() ) {
                 // Injected commands allow sst command line options to cause actions (currently only replay)
                 line = injectedCommand.str();
@@ -289,6 +333,7 @@ SimpleDebugger::execute(const std::string& msg)
             std::cout << "Parsing error. Ignoring " << line << std::endl;
         }
     }
+    return retState;
 }
 
 // Invoke the command.
@@ -490,6 +535,78 @@ SimpleDebugger::cmd_verbose(std::vector<std::string>& tokens)
     }
 }
 
+void
+SimpleDebugger::cmd_info(std::vector<std::string>& UNUSED(tokens)) {
+
+    if (tokens.size() != 2) {
+        printf("Invalid format for info command (info \"current\"|\"all\")\n");
+        return;
+    }
+
+    RankInfo info = getRank();
+    RankInfo nRanks = getNumRanks();
+    if (tokens[1] == "current") {
+        std::cout << "Rank " << info.rank << "/" << nRanks.rank
+            << ", Thread " << info.thread << "/" << nRanks.thread << std::endl;
+    }
+    else if (tokens[1] == "all") {
+        if (nRanks.rank == 1 && nRanks.thread == 1) {
+            std::cout << "Rank " << info.rank << "/" << nRanks.rank
+                << ", Thread " << info.thread << "/" << nRanks.thread << std::endl;
+        }
+        else {
+            // Return to syncmanager to print summary for all threads
+            retState = -2; // summary info
+            done = true;
+        }
+    }
+    else {
+        printf("Invalid argument for info command: %s (info \"current\"|\"all\")\n", tokens[1].c_str());
+        return;
+    }
+}
+
+// thread <threadID> : switches to new thread
+void
+SimpleDebugger::cmd_thread(std::vector<std::string>& tokens) {
+
+    if (tokens.size() != 2) {
+        printf("Invalid format for thread command (thread <threadID>)\n");
+        return;
+    }
+
+    RankInfo info = getRank();
+    RankInfo nRanks = getNumRanks();
+    int threadID;
+
+    // Get threadID
+    try {
+        threadID = std::stoi(tokens[1]);
+    }
+    catch (const std::invalid_argument& e) {
+        std::cout << "Invalid argument for threadID: " << tokens[1] << std::endl;
+        return;
+    }
+    catch (const std::out_of_range& e) {
+        std::cout << "Out of range for threadID: " << tokens[1] << std::endl;
+        return;
+    }
+
+    // Check if valid threadID
+    if (threadID < 0 || threadID >= nRanks.thread) {
+        printf("ThreadID %d out of range (0:%d)\n", threadID, nRanks.thread-1);
+        return;
+    }
+    
+    // If not current thread, set retState and done flag
+    if (threadID != info.thread) {
+        retState = threadID;
+        done = true;
+    }
+    return;
+}
+
+
 // pwd: print current working directory
 void
 SimpleDebugger::cmd_pwd(std::vector<std::string>& UNUSED(tokens))
@@ -541,10 +658,18 @@ SimpleDebugger::get_listing_strings(std::list<std::string>& list)
 void
 SimpleDebugger::cmd_cd(std::vector<std::string>& tokens)
 {
+#if 1
     if ( tokens.size() != 2 ) {
         printf("Invalid format for cd command (cd <obj>)\n");
         return;
     }
+#else
+    // skk This works but doesn't delete/deactivate like objmap selectParent
+    if (tokens.size() == 1) {
+        obj_ = getComponentObjectMap();
+        return;
+    }
+#endif
 
     // Allow for trailing '/'
     std::string selection = tokens[1];

src/sst/core/impl/interactive/simpleDebug.h

@@ -250,7 +250,8 @@ class SimpleDebugger : public SST::InteractiveConsole
     explicit SimpleDebugger(Params& params);
     ~SimpleDebugger();
 
-    void execute(const std::string& msg) override;
+    int execute(const std::string& msg) override;
+    void summary() override;
 
     // Callbacks from command line completions
     void get_listing_strings(std::list<std::string>&);
@@ -265,6 +266,7 @@ class SimpleDebugger : public SST::InteractiveConsole
 
     SST::Core::Serialization::ObjectMap* obj_ = nullptr;
     bool                                 done = false;
+    int                                  retState = -1; // -1 done, positive number is threadID
 
     bool autoCompleteEnable = true;
 
@@ -297,7 +299,9 @@ class SimpleDebugger : public SST::InteractiveConsole
 
     // Navigation
     void cmd_help(std::vector<std::string>& UNUSED(tokens));
-    void cmd_verbose(std::vector<std::string>&(tokens));
+    void cmd_verbose(std::vector<std::string>& (tokens));
+    void cmd_info(std::vector<std::string>& UNUSED(tokens));
+    void cmd_thread(std::vector<std::string>& tokens);
     void cmd_pwd(std::vector<std::string>& UNUSED(tokens));
     void cmd_ls(std::vector<std::string>& UNUSED(tokens));
     void cmd_cd(std::vector<std::string>& tokens);

src/sst/core/interactiveAction.h

@@ -15,6 +15,7 @@
 #include "sst/core/action.h"
 // Can include this because this header is not installed
 #include "sst/core/simulation_impl.h"
+#include "sst/core/interactiveConsole.h"
 
 #include <string>
 
@@ -31,26 +32,54 @@ class InteractiveAction : public Action
        Create a new InteractiveAction object for the simulation core to initiate interactive mode
     */
     InteractiveAction(Simulation_impl* sim, const std::string& msg) :
-        Action(),
         sim_(sim),
         msg_(msg)
     {
         setPriority(INTERACTIVEPRIOIRTY);
     }
 
     ~InteractiveAction() {}
+#if 1
+    /**
+       Indicates InteractiveAction should be inserted into the
+       TimeVortex. The insertion will only happen for serial runs, as
+       InteractiveAction is managed by the SyncManager in parallel
+       runs.
+     */
+    void insertIntoTimeVortex(SimTime_t time) {
+        // If this is a serial job, insert this into
+        // the time TimeVortex.  If it is parallel, then the
+        // InteractiveAction is managed by the SyncManager.
+        //std::cout << "skk: insertIntoTimeVortex called\n";
+        RankInfo num_ranks = sim_->getNumRanks();
+        //if (num_ranks.rank == 1 && num_ranks.thread == 1) {
+            //std::cout << "  skk: insertIntoTimeVortex insertActivity\n";
+            sim_->insertActivity(time, this);
+        //}
+    }
+#endif
+#if 0
+    /** Break to interactive console next time check() is called */
+    void setInteractiveConsole() {
+        sim_->enter_interactive_ = true;
+    }
+#endif
 
     /** Called by TimeVortex to trigger interactive mode. */
     void execute() override
     {
         sim_->enter_interactive_ = true;
-        sim_->interactive_msg_   = msg_;
+        sim_->interactive_msg_ = msg_;
         delete this;
     }
 
+
 private:
     Simulation_impl* sim_;
     std::string      msg_;
+    
+    // Do I need flag here for break to interactive like ckptAction?
+    // Currently using sim_->enter_interactive_
 };
 
 } // namespace SST

src/sst/core/interactiveConsole.cc

@@ -100,6 +100,7 @@ InteractiveConsole::getTimeConverter(const std::string& time)
 void
 InteractiveConsole::schedule_interactive(SimTime_t time_offset, const std::string& msg)
 {
+    //std::cout << "skk: IC: schedule_interactive\n";
     Simulation_impl*   sim = Simulation_impl::getSimulation();
     InteractiveAction* act = new InteractiveAction(sim, msg);
     sim->insertActivity(getCurrentSimCycle() + time_offset, act);
@@ -114,7 +115,8 @@ InteractiveConsole::getComponentObjectMap()
 void
 InteractiveConsole::simulationShutdown()
 {
-    Simulation_impl::getSimulation()->endSimulation();
+    //Simulation_impl::getSimulation()->endSimulation();
+    Simulation_impl::getSimulation()->signalShutdown(false);
 }
 
 

src/sst/core/interactiveConsole.h

@@ -58,7 +58,9 @@ class InteractiveConsole
     virtual ~InteractiveConsole() = default;
 
     /** Called by TimeVortex to trigger checkpoint on simulation clock interval - not used in parallel simulation */
-    virtual void execute(const std::string& msg) = 0;
+    virtual int execute(const std::string& msg) = 0;
+    /** Called by SyncManager to get summary info for each thread */
+    virtual void summary() = 0;
 
 protected:
     // Functions that can be called by child class