Interactive Console Updates

* Support for single-rank, multithreaded interactive debug console.
  Breaks into interactive console is at a sync point and is managed
  by the syncManager. It breaks into the interactive console when
  any thread has triggered the interactive console and starts in
  thread 0 initially. The user can navigate the object map,
  set watchpoints, etc. for components within the thread and switch
  between threads to access other components.
* Support for user-defined commands in the interactive debug console
  which are a named sequence of commands and can be nested.
* Updates to the parsing of --checkpoint-prefix and creation of the
  checkpoint directory. Ensure that the signal flags are handled
  before the checkpoint directory is created and fails immediately
  if the PREFIX is not valid.

Author: skuntz

src/sst/core/checkpointAction.cc

@@ -331,6 +331,14 @@ initializeCheckpointInfrastructure(Config* cfg, bool rt_can_ckpt, int myRank)
     std::string checkpoint_dir_name = "";
 
     if ( myRank == 0 ) {
+        // Verify prefix format (no '/')
+        const std::string prefix   = cfg->checkpoint_prefix();
+        size_t            foundPos = prefix.find('/');
+        if ( foundPos != std::string::npos ) {
+            throw std::invalid_argument("Invalid checkpoint prefix: " + prefix);
+        }
+
+        // Create checkpoint directory path
         SST::Util::Filesystem& fs = Simulation_impl::getSimulation()->filesystem;
         checkpoint_dir_name       = fs.createUniqueDirectory(cfg->checkpoint_prefix());
     }

src/sst/core/impl/interactive/simpleDebug.cc

@@ -32,7 +32,7 @@
 
 namespace SST::IMPL::Interactive {
 
-enum class ConsoleCommandGroup { GENERAL, NAVIGATION, STATE, WATCH, SIMULATION, LOGGING, MISC };
+enum class ConsoleCommandGroup { GENERAL, NAVIGATION, STATE, WATCH, SIMULATION, LOGGING, MISC, USER };
 
 const std::map<ConsoleCommandGroup, std::string> GroupText {
     { ConsoleCommandGroup::GENERAL, "General" },
@@ -42,16 +42,26 @@ const std::map<ConsoleCommandGroup, std::string> GroupText {
     { ConsoleCommandGroup::SIMULATION, "Simulation" },
     { ConsoleCommandGroup::LOGGING, "Logging" },
     { ConsoleCommandGroup::MISC, "Misc" },
+    { ConsoleCommandGroup::USER, "User Defined" },
 };
 
+// Each bit of mask enables verbosity for different debug features.
+// This is primarily for developers.
 enum class VERBOSITY_MASK : uint32_t {
     WATCHPOINTS = 0b0001'0000 // 0x10
 };
 
+enum class LINE_ENTRY_MODE : int {
+    NORMAL,   // line is executed as a command
+    DEFINE,   // line is captured in user defined command sequence
+    DOCUMENT, // documenting a user defined command
+};
+
 // Encapsulate a console command.
 class ConsoleCommand
 {
 public:
+    // Constructor for built-in commands has callback
     ConsoleCommand(std::string str_long, std::string str_short, std::string str_help, ConsoleCommandGroup group,
         std::function<void(std::vector<std::string>& tokens)> func) :
         str_long_(str_long),
@@ -60,17 +70,28 @@ class ConsoleCommand
         group_(group),
         func_(func)
     {}
-    const std::string&         str_long() const { return str_long_; }
-    const std::string&         str_short() const { return str_short_; }
-    const std::string&         str_help() const { return str_help_; }
+    // Constructor for user-defined commands
+    ConsoleCommand(std::string str_long) :
+        str_long_(str_long),
+        str_short_(str_long),
+        str_help_("user defined command"),
+        group_(ConsoleCommandGroup::USER)
+    {}
+    ConsoleCommand() {}; // default constructor
+    const std::string& str_long() const { return str_long_; }
+    const std::string& str_short() const { return str_short_; }
+    const std::string& str_help() const { return str_help_; }
+    void               setUserHelp(std::string& help) { str_help_ = help; }
+
+    // Command Execution
+    void exec(std::vector<std::string>& tokens) { return func_(tokens); }
+
     const ConsoleCommandGroup& group() const { return group_; }
-    void                       exec(std::vector<std::string>& tokens) { return func_(tokens); }
     bool                       match(const std::string& token)
     {
         std::string lctoken = toLower(token);
         if ( lctoken.size() == str_long_.size() && lctoken == toLower(str_long_) ) return true;
         if ( lctoken.size() == str_short_.size() && lctoken == toLower(str_short_) ) return true;
-
         return false;
     }
     friend std::ostream& operator<<(std::ostream& os, const ConsoleCommand c)
@@ -84,13 +105,14 @@ class ConsoleCommand
     std::string                                           str_short_;
     std::string                                           str_help_;
     ConsoleCommandGroup                                   group_;
-    std::function<void(std::vector<std::string>& tokens)> func_;
+    std::function<void(std::vector<std::string>& tokens)> func_ = {};
     std::string                                           toLower(std::string s)
     {
         std::transform(s.begin(), s.end(), s.begin(), ::tolower);
         return s;
     }
-};
+
+}; // class ConsoleCommand
 
 class CommandHistoryBuffer
 {
@@ -102,8 +124,10 @@ class CommandHistoryBuffer
     std::vector<std::string>& getBuffer();
     enum BANG_RC { INVALID, ECHO_ONLY, EXEC, NOP };
     BANG_RC bang(const std::string& token, std::string& newcmd);
+    void    enable(bool en) { en_ = en; }
 
 private:
+    bool                                             en_    = true;
     int                                              cur_   = 0;
     int                                              nxt_   = 0;
     int                                              sz_    = 0;
@@ -117,6 +141,94 @@ class CommandHistoryBuffer
     bool                     searchAny(const std::string& s, std::string& newcmd);
 };
 
+class CommandRegistry
+{
+
+public:
+    // Construction
+    CommandRegistry() {}
+    CommandRegistry(const std::vector<ConsoleCommand> in) :
+        registry(in)
+    {}
+    // Access
+    std::vector<ConsoleCommand>& getRegistryVector() { return registry; }
+    std::vector<ConsoleCommand>& getUserRegistryVector() { return user_registry; }
+    enum SEARCH_TYPE { ALL, BUILTIN, USER };
+    std::pair<ConsoleCommand, bool> const seek(std::string token, SEARCH_TYPE search_type);
+    // User defined command entry
+    bool                                  beginUserCommand(std::string name);
+    void                                  appendUserCommand(std::string token0, std::string line);
+    void                                  commitUserCommand();
+    std::vector<std::string>*             userCommandInsts(std::string key)
+    {
+        if ( user_defined_commands.find(key) == user_defined_commands.end() ) return nullptr;
+        return &user_defined_commands[key];
+    }
+    // User defined command help doc entry
+    bool beginDocCommand(std::string name);
+    void appendDocCommand(std::string line);
+    void commitDocCommand();
+    bool commandIsEmpty(const std::string key)
+    {
+        if ( user_defined_commands.find(key) == user_defined_commands.end() ) return true;
+        if ( user_defined_commands[key].size() == 0 ) return true;
+        return false;
+    };
+
+    // User defined command help from vector
+    void                               addHelp(std::string key, std::vector<std::string>& vec);
+    // Detailed Command Help (public for now)
+    std::map<std::string, std::string> cmdHelp;
+
+private:
+    // built-in commands
+    std::vector<ConsoleCommand>                     registry              = {};
+    // user defined commands
+    std::vector<ConsoleCommand>                     user_registry         = {};
+    std::map<std::string, std::vector<std::string>> user_defined_commands = {};
+    std::string                                     user_command_wip      = "";
+    std::vector<std::string>                        user_doc_wip          = {};
+
+    // Last searched command with valid indicator
+    std::pair<ConsoleCommand, bool> last_seek_command = {};
+}; // class CommandRegistry
+
+// Support for nesting user defined commands
+class ExecState
+{
+public:
+    // Constructor for entering a new user command
+    ExecState(ConsoleCommand cmd, std::vector<std::string> tokens, std::vector<std::string>* insts) :
+        cmd_(cmd),
+        tokens_(tokens),
+        insts_(insts),
+        index_(0),
+        user_(true)
+    {
+        assert(insts_->size() > 0);
+    };
+    ExecState() {};
+    bool        ret() { return ret_; }
+    // Advance state and return the next user instruction
+    std::string next()
+    {
+        assert(user_);
+        assert(!ret_);
+        assert(insts_);
+        assert(index_ < insts_->size());
+        ret_ = (index_ + 1) == insts_->size();
+        return insts_->at(index_++);
+    }
+
+private:
+    ConsoleCommand            cmd_    = {};      // user command in progress
+    std::vector<std::string>  tokens_ = {};      // command args
+    std::vector<std::string>* insts_  = nullptr; // command sequence
+    size_t                    index_  = 0;       // command pointer
+    bool                      user_   = false;   // in user command
+    bool                      ret_    = false;   // user command complete, return to caller
+}; // class ExecState
+
 class SimpleDebugger : public SST::InteractiveConsole
 {
 
@@ -138,7 +250,8 @@ class SimpleDebugger : public SST::InteractiveConsole
     explicit SimpleDebugger(Params& params);
     ~SimpleDebugger();
 
-    void execute(const std::string& msg) override;
+    int  execute(const std::string& msg) override;
+    void summary() override;
 
     // Callbacks from command line completions
     void get_listing_strings(std::list<std::string>&);
@@ -151,8 +264,9 @@ class SimpleDebugger : public SST::InteractiveConsole
     // directory as far as we can.
     std::vector<std::string> name_stack;
 
-    SST::Core::Serialization::ObjectMap* obj_ = nullptr;
-    bool                                 done = false;
+    SST::Core::Serialization::ObjectMap* obj_     = nullptr;
+    bool                                 done     = false;
+    int                                  retState = -1; // -1 DONE, -2 SUMAMRY, positive number is threadID
 
     bool autoCompleteEnable = true;
 
@@ -166,8 +280,12 @@ class SimpleDebugger : public SST::InteractiveConsole
     std::string   replayFilePath  = "sst-console.in";
     bool          enLogging       = false;
 
-    // command injection
-    std::stringstream injectedCommand; // TODO use ConsoleCommand object
+    // command injection (for sst --replay option)
+    std::stringstream injectedCommand;
+
+    // execution state management for nested user commands
+    ExecState             eState = {};
+    std::stack<ExecState> eStack = {};
 
     // Keep a pointer to the ObjectMap for the top level Component
     SST::Core::Serialization::ObjectMapDeferred<BaseComponent>* base_comp_ = nullptr;
@@ -182,6 +300,8 @@ class SimpleDebugger : public SST::InteractiveConsole
     // Navigation
     void cmd_help(std::vector<std::string>& UNUSED(tokens));
     void cmd_verbose(std::vector<std::string>&(tokens));
+    void cmd_info(std::vector<std::string>& UNUSED(tokens));
+    void cmd_thread(std::vector<std::string>& tokens);
     void cmd_pwd(std::vector<std::string>& UNUSED(tokens));
     void cmd_ls(std::vector<std::string>& UNUSED(tokens));
     void cmd_cd(std::vector<std::string>& tokens);
@@ -192,6 +312,9 @@ class SimpleDebugger : public SST::InteractiveConsole
     void cmd_time(std::vector<std::string>& tokens);
     void cmd_watch(std::vector<std::string>& tokens);
     void cmd_unwatch(std::vector<std::string>& tokens);
+#ifdef __CT_RECURSE__
+    void cmd_examine(std::vector<std::string>& tokens);
+#endif
 
     // Simulation Control
     void cmd_run(std::vector<std::string>& tokens);
@@ -219,22 +342,25 @@ class SimpleDebugger : public SST::InteractiveConsole
     // Reset terminal
     void cmd_clear(std::vector<std::string>& UNUSED(tokens));
 
+    // User defined commands
+    void cmd_define(std::vector<std::string>& tokens);
+    void cmd_document(std::vector<std::string>& tokens);
+
     // LLDB/GDB helper
     void cmd_spinThread(std::vector<std::string>& tokens);
 
+    // command entry point
     void dispatch_cmd(std::string& cmd);
 
     // Command Registry
-    std::vector<ConsoleCommand> cmdRegistry;
-
-    // Detailed Command Help
-    std::map<std::string, std::string> cmdHelp;
+    CommandRegistry cmdRegistry;
 
     // Command History
     CommandHistoryBuffer cmdHistoryBuf;
 
     // Command Line Editor
-    CmdLineEditor cmdLineEditor;
+    CmdLineEditor   cmdLineEditor;
+    LINE_ENTRY_MODE line_entry_mode = LINE_ENTRY_MODE::NORMAL;
 
     // Verbosity controlled console printing
     uint32_t verbosity = 0;

src/sst/core/impl/interactive/simpleDebug.h

@@ -14,6 +14,7 @@
 
 #include "sst/core/action.h"
 // Can include this because this header is not installed
+#include "sst/core/interactiveConsole.h"
 #include "sst/core/simulation_impl.h"
 
 #include <string>
@@ -31,14 +32,33 @@ class InteractiveAction : public Action
        Create a new InteractiveAction object for the simulation core to initiate interactive mode
     */
     InteractiveAction(Simulation_impl* sim, const std::string& msg) :
-        Action(),
         sim_(sim),
         msg_(msg)
     {
         setPriority(INTERACTIVEPRIOIRTY);
     }
 
     ~InteractiveAction() {}
+#if 1
+    /**
+       Indicates InteractiveAction should be inserted into the
+       TimeVortex. The insertion will only happen for serial runs, as
+       InteractiveAction is managed by the SyncManager in parallel
+       runs.
+     */
+    void insertIntoTimeVortex(SimTime_t time)
+    {
+        // If this is a serial job, insert this into
+        // the time TimeVortex.  If it is parallel, then the
+        // InteractiveAction is managed by the SyncManager.
+        // std::cout << "skk: insertIntoTimeVortex called\n";
+        RankInfo num_ranks = sim_->getNumRanks();
+        // if (num_ranks.rank == 1 && num_ranks.thread == 1) {
+        // std::cout << "  skk: insertIntoTimeVortex insertActivity\n";
+        sim_->insertActivity(time, this);
+        //}
+    }
+#endif
 
     /** Called by TimeVortex to trigger interactive mode. */
     void execute() override
@@ -48,6 +68,7 @@ class InteractiveAction : public Action
         delete this;
     }
 
+
 private:
     Simulation_impl* sim_;
     std::string      msg_;

src/sst/core/interactiveAction.h

@@ -114,7 +114,9 @@ InteractiveConsole::getComponentObjectMap()
 void
 InteractiveConsole::simulationShutdown()
 {
-    Simulation_impl::getSimulation()->endSimulation();
+    // Simulation_impl::getSimulation()->endSimulation();  // Only works for single thread
+    std::cout << "Simulation shutdown\n";
+    Simulation_impl::getSimulation()->signalShutdown(false);
 }
 
 

src/sst/core/interactiveConsole.cc

@@ -57,8 +57,12 @@ class InteractiveConsole
     InteractiveConsole()          = default;
     virtual ~InteractiveConsole() = default;
 
+    /** Interactive Console execute return codes: Positive number is thread ID to switch to, negative is other state */
+    enum ICretcode { DONE = -1, SUMMARY = -2 };
     /** Called by TimeVortex to trigger checkpoint on simulation clock interval - not used in parallel simulation */
-    virtual void execute(const std::string& msg) = 0;
+    virtual int  execute(const std::string& msg) = 0;
+    /** Called by SyncManager to get summary info for each thread */
+    virtual void summary()                       = 0;
 
 protected:
     // Functions that can be called by child class

src/sst/core/interactiveConsole.h

@@ -468,6 +468,11 @@ start_simulation(uint32_t tid, SimThreadInfo_t& info, Core::ThreadSafe::Barrier&
         Simulation_impl::createSimulation(info.myRank, info.world_size, restart, currentSimCycle, currentPriority);
     double start_run = 0.0;
 
+    // Setup the real time actions (all of these have to be defined on
+    // the command-line or SDL file, they will not be checkpointed and
+    // restored
+    sim->setupSimActions();
+
     // Thread zero needs to initialize the checkpoint data structures
     // if any checkpointing options were turned on.  This will return
     // an empty string if checkpointing was not enabled.
@@ -528,13 +533,6 @@ start_simulation(uint32_t tid, SimThreadInfo_t& info, Core::ThreadSafe::Barrier&
 
     } // if ( restart )
 
-
-    // Setup the real time actions (all of these have to be defined on
-    // the command-line or SDL file, they will not be checkpointed and
-    // restored
-    sim->setupSimActions();
-
-
     if ( !restart ) {
 
         // Prepare the links, which creates the ComponentInfo objects and