Merge pull request #1270 from sstsimulator/devel

Automatically Merged using SST Master Branch Merger

Author: sst-autotester

config/sst_check_filesystem.m4

@@ -0,0 +1,26 @@
+AC_DEFUN([SST_CHECK_FILESYSTEM],
+[
+  AC_LANG_SAVE
+   AC_LANG([C++])
+  # In older versions of gcc implementation of std::filesystem is kept
+  # in separate stdc++fs library. You should link it explicitly
+  AC_MSG_CHECKING([if std::filesystem requires linking stdc++fs])
+  AC_LINK_IFELSE(
+      [AC_LANG_SOURCE([
+          #include <filesystem>
+          int main() {
+              std::filesystem::create_directory("/dev/null");
+          }
+      ])],
+      [ac_cv_fs_stdlib=no],
+      [ac_cv_fs_stdlib=yes]
+  )
+  if test "x$ac_cv_fs_stdlib" = xyes; then
+      AC_MSG_RESULT(yes)
+      LIBS="$LIBS -lstdc++fs"
+  else
+      AC_MSG_RESULT(no)
+  fi
+  AC_LANG_RESTORE
+
+])

configure.ac

@@ -13,6 +13,13 @@ AM_INIT_AUTOMAKE([1.9.6 foreign dist-bzip2 subdir-objects no-define tar-pax])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 m4_pattern_allow([LT_LIBEXT])
 
+AH_TOP([
+#ifndef _SST_CONFIG_H_
+#define _SST_CONFIG_H_
+])
+AH_BOTTOM([
+#endif /* _SST_CONFIG_H_ */
+])
 AC_CONFIG_HEADERS([src/sst/core/sst_config.h])
 
 # Lets check for the standard compilers and basic options
@@ -93,6 +100,8 @@ SST_ENABLE_CORE_PROFILE()
 
 SST_CHECK_FPIC()
 
+SST_CHECK_FILESYSTEM()
+
 AC_DEFINE_UNQUOTED([SST_CPPFLAGS], ["$CPPFLAGS"], [Defines the CPPFLAGS used to build SST])
 AC_DEFINE_UNQUOTED([SST_CFLAGS], ["$CFLAGS"], [Defines the CFLAGS used to build SST])
 AC_DEFINE_UNQUOTED([SST_CXXFLAGS], ["$CXXFLAGS"], [Defines the CXXFLAGS used to build SST])

src/sst/core/Makefile.am

@@ -177,6 +177,7 @@ sst_core_sources = \
 	configBase.cc \
 	configShared.cc \
 	configGraph.cc \
+	configGraphOutput.cc \
 	cfgoutput/pythonConfigOutput.cc \
 	cfgoutput/dotConfigOutput.cc \
 	cfgoutput/xmlConfigOutput.cc \

src/sst/core/cfgoutput/jsonConfigOutput.cc

@@ -53,6 +53,19 @@ struct StatPair
     SST::ConfigComponent const*                  comp;
 };
 
+struct StatGroupPair
+{
+    std::pair<const std::string, SST::ConfigStatGroup> const& group;
+    std::vector<std::string>                                  vec;
+    SST::ConfigGraph const*                                   graph;
+};
+
+struct StatGroupParamPair
+{
+    const std::string  name;
+    const SST::Params& stat;
+};
+
 void
 to_json(json::ordered_json& j, StatPair const& sp)
 {
@@ -136,6 +149,49 @@ to_json(json::ordered_json& j, LinkConfPair const& pair)
     j["right"]["latency"]   = link->latency_str[1];
 }
 
+void
+to_json(json::ordered_json& j, StatGroupParamPair const& pair)
+{
+    auto const& outParams = pair.stat;
+
+    j["name"] = pair.name;
+
+    for ( auto const& param : outParams.getKeys() ) {
+        j["params"][param] = outParams.find<std::string>(param);
+    }
+}
+
+void
+to_json(json::ordered_json& j, StatGroupPair const& pair)
+{
+    auto const& grp   = pair.group.second;
+    auto const* graph = pair.graph;
+    auto        vec   = pair.vec;
+
+    j["name"] = grp.name;
+
+    if ( grp.outputFrequency.getValue() != 0 ) { j["frequency"] = grp.outputFrequency.toStringBestSI(); }
+
+    if ( grp.outputID != 0 ) {
+        const SST::ConfigStatOutput& out = graph->getStatOutput(grp.outputID);
+        j["output"]["type"]              = out.type;
+        if ( !out.params.empty() ) {
+            const SST::Params& outParams = out.params;
+            for ( auto const& param : vec ) {
+                j["output"]["params"][param] = outParams.find<std::string>(param);
+            }
+        }
+    }
+
+    for ( auto& i : grp.statMap ) {
+        if ( !i.second.empty() ) { j["statistics"].emplace_back(StatGroupParamPair { i.first, i.second }); }
+    }
+
+    for ( SST::ComponentId_t id : grp.components ) {
+        const SST::ConfigComponent* comp = graph->findComponent(id);
+        j["components"].emplace_back(comp->name);
+    }
+}
 } // namespace
 
 void
@@ -164,14 +220,38 @@ JSONConfigGraphOutput::generate(const Config* cfg, ConfigGraph* graph)
     outputJson["program_options"]["checkpoint-sim-period"]  = cfg->checkpoint_sim_period();
     outputJson["program_options"]["checkpoint-wall-period"] = std::to_string(cfg->checkpoint_wall_period());
 
-
     // Put in the global param sets
     for ( const auto& set : getGlobalParamSetNames() ) {
         for ( const auto& kvp : getGlobalParamSet(set) ) {
             if ( kvp.first != "<set_name>" ) outputJson["global_params"][set][kvp.first] = kvp.second;
         }
     }
 
+    // Global statistics
+    if ( 0 != graph->getStatLoadLevel() ) {
+        outputJson["statistics_options"]["statisticLoadLevel"] = (uint64_t)graph->getStatLoadLevel();
+    }
+
+    if ( !graph->getStatOutput().type.empty() ) {
+        outputJson["statistics_options"]["statisticOutput"] = graph->getStatOutput().type.c_str();
+        const Params& outParams                             = graph->getStatOutput().params;
+        if ( !outParams.empty() ) {
+            // generate the parameters
+            for ( auto const& paramsItr : getParamsLocalKeys(outParams) ) {
+                outputJson["statistics_options"]["params"][paramsItr] = outParams.find<std::string>(paramsItr);
+            }
+        }
+    }
+
+    // Generate the stat groups
+    if ( !graph->getStatGroups().empty() ) {
+        outputJson["statistics_group"];
+        for ( auto& grp : graph->getStatGroups() ) {
+            auto vec = getParamsLocalKeys(graph->getStatOutput(grp.second.outputID).params);
+            outputJson["statistics_group"].emplace_back(StatGroupPair { grp, vec, graph });
+        }
+    }
+
     // no components exist in this rank
     if ( const_cast<ConfigComponentMap_t&>(compMap).size() == 0 ) { outputJson["components"]; }
 

src/sst/core/checkpointAction.cc

@@ -21,14 +21,52 @@
 #include "sst/core/stringize.h"
 #include "sst/core/timeConverter.h"
 
-// #include <filesystem>
+#include <filesystem>
 #include <sys/stat.h>
 #include <unistd.h>
 
 namespace SST {
 
-CheckpointAction::CheckpointAction(
-    Config* UNUSED(cfg), RankInfo this_rank, Simulation_impl* sim, TimeConverter* period) :
+namespace pvt {
+
+std::string
+createNameFromFormat(const std::string& format, const std::string& prefix, uint64_t checkpoint_id, SimTime_t time)
+{
+    std::string ret;
+    bool        found_percent = false;
+    for ( const auto& x : format ) {
+        if ( found_percent ) {
+            switch ( x ) {
+            case 'p':
+                ret += prefix;
+                break;
+            case 'n':
+                ret += std::to_string(checkpoint_id);
+                break;
+            case 't':
+                ret += std::to_string(time);
+                break;
+            default:
+                // Should not happen since format string was already
+                // checked, but if it does, just delete whole %
+                // sequence (i.e. do nothing)
+                break;
+            }
+            found_percent = false;
+        }
+        else if ( x == '%' ) {
+            found_percent = true;
+        }
+        else {
+            ret += x;
+        }
+    }
+    return ret;
+}
+
+} // namespace pvt
+
+CheckpointAction::CheckpointAction(Config* cfg, RankInfo this_rank, Simulation_impl* sim, TimeConverter* period) :
     Action(),
     rank_(this_rank),
     period_(period),
@@ -53,7 +91,37 @@ CheckpointAction::CheckpointAction(
         next_sim_time_ = MAX_SIMTIME_T;
     }
 
-    if ( (0 == this_rank.rank) ) { last_cpu_time_ = sst_get_cpu_time(); }
+    // Parse the format string.  It was checked by the Config object
+    // to make sure there was no more than one directory separator (/)
+    // and that no invalid % sequences were used.
+    std::string format = cfg->checkpoint_name_format();
+    size_t      split  = format.find("/");
+    if ( split == format.npos ) {
+        dir_format_  = format;
+        file_format_ = format;
+    }
+    else {
+        dir_format_  = format.substr(0, split);
+        file_format_ = format.substr(split + 1);
+    }
+
+    if ( (0 == this_rank.rank) ) {
+        // Check to make sure that there is at least one of %n or %t to
+        // make checkpoint filenames unique.
+        bool unique = false;
+        if ( format.find("%n") != format.npos ) unique = true;
+        if ( format.find("%t") != format.npos ) unique = true;
+
+        if ( !unique ) {
+            sim->getSimulationOutput().output(
+                "WARNING: checkpoint-name-format does not include one of %%n or %%t, which means that all checkpoints "
+                "will use the same filename and previous files will be overwritten [%s].\n",
+                format.c_str());
+        }
+
+        last_cpu_time_ = sst_get_cpu_time();
+    }
+
     // Set the priority to be the same as the SyncManager so that
     // checkpointing happens in the same place for both serial and
     // parallel runs.  We will never have both a SyncManager and a
@@ -78,7 +146,7 @@ CheckpointAction::execute()
 void
 CheckpointAction::createCheckpoint(Simulation_impl* sim)
 {
-    if ( 0 == rank_.rank ) {
+    if ( 0 == rank_.rank && 0 == rank_.thread ) {
         const double now = sst_get_cpu_time();
         sim->getSimulationOutput().output(
             "# Simulation Checkpoint: Simulated Time %s (Real CPU time since last checkpoint %.5f seconds)\n",
@@ -89,36 +157,31 @@ CheckpointAction::createCheckpoint(Simulation_impl* sim)
 
     // Need to create a directory for this checkpoint
     std::string prefix   = sim->checkpoint_prefix_;
-    std::string basename = prefix + "_" + std::to_string(checkpoint_id) + "_" + std::to_string(sim->currentSimCycle);
+    std::string basename = pvt::createNameFromFormat(dir_format_, prefix, checkpoint_id, sim->currentSimCycle);
 
     // Directory is shared across threads.  Make it a static and make
     // sure we barrier in the right places
-    static std::string directory;
+    std::string directory = sim->checkpoint_directory_ + "/" + basename;
 
     // Only thread 0 will participate in setup
     if ( rank_.thread == 0 ) {
         // Rank 0 will create the directory for this checkpoint
         if ( rank_.rank == 0 ) {
-            directory = Checkpointing::createUniqueDirectory(sim->checkpoint_directory_ + "/" + basename);
-#ifdef SST_CONFIG_HAVE_MPI
-            Comms::broadcast(directory, 0);
-#endif
+            directory = sim->checkpoint_directory_ + "/" + basename;
+            std::filesystem::create_directory(directory);
         }
-        else {
-            // Get directory name (really just a barrier since each
-            // rank already knows the name and it shouldn't have to
-            // create a unique one)
 #ifdef SST_CONFIG_HAVE_MPI
-            Comms::broadcast(directory, 0);
+        Comms::broadcast(directory, 0);
 #endif
-        }
     }
-    barrier.wait();
-    if ( rank_.thread == 0 ) checkpoint_id++;
-
+    basename = pvt::createNameFromFormat(file_format_, prefix, checkpoint_id, sim->currentSimCycle);
     std::string filename =
         directory + "/" + basename + "_" + std::to_string(rank_.rank) + "_" + std::to_string(rank_.thread) + ".bin";
 
+    barrier.wait();
+
+    if ( rank_.thread == 0 ) checkpoint_id++;
+
     // Write out the checkpoints for the partitions
     sim->checkpoint(filename);
 
@@ -245,55 +308,6 @@ doesDirectoryExist(const std::string& dirName, bool include_files)
     }
 }
 
-/**
-   Function to create a directory. We need this bacause
-   std::filesystem isn't fully supported until GCC9
-*/
-bool
-createDirectory(const std::string& dirName)
-{
-    if ( mkdir(dirName.c_str(), 0755) == 0 ) {
-        return true; // Directory created successfully
-    }
-    else {
-        return false; // Failed to create directory
-    }
-}
-
-std::string
-createUniqueDirectory(const std::string basename)
-{
-    std::string dirName = basename;
-
-    // Check if the directory exists
-    // if ( std::filesystem::exists(dirName) ) {
-    if ( doesDirectoryExist(dirName, true) ) {
-        // Append a unique random set of characters to the directory name
-        std::string newDirName;
-        int         num = 0;
-        do {
-            ++num;
-            newDirName = dirName + "_" + std::to_string(num);
-            // } while ( std::filesystem::exists(newDirName) ); // Ensure the new directory name is unique
-        } while ( doesDirectoryExist(newDirName, true) ); // Ensure the new directory name is unique
-
-        dirName = newDirName;
-    }
-
-    // Create the directory
-    // if ( !std::filesystem::create_directory(dirName) ) {
-    if ( !createDirectory(dirName) ) {
-        Simulation_impl::getSimulationOutput().fatal(
-            CALL_INFO_LONG, 1, "Failed to create directory: %s\n", dirName.c_str());
-    }
-    return dirName;
-}
-
-void
-removeDirectory(const std::string UNUSED(name))
-{
-    // Implement when adding logic to keep only N checkpoints
-}
 
 std::string
 initializeCheckpointInfrastructure(Config* cfg, bool rt_can_ckpt, int myRank)
@@ -303,7 +317,11 @@ initializeCheckpointInfrastructure(Config* cfg, bool rt_can_ckpt, int myRank)
 
     std::string checkpoint_dir_name = "";
 
-    if ( myRank == 0 ) { checkpoint_dir_name = createUniqueDirectory(cfg->checkpoint_prefix()); }
+    if ( myRank == 0 ) {
+        SST::Util::Filesystem& fs = Simulation_impl::getSimulation()->filesystem;
+        checkpoint_dir_name       = fs.createUniqueDirectory(cfg->checkpoint_prefix());
+    }
+
 #ifdef SST_CONFIG_HAVE_MPI
     // Broadcast the directory name
     Comms::broadcast(checkpoint_dir_name, 0);

src/sst/core/checkpointAction.h

@@ -94,6 +94,8 @@ class CheckpointAction : public Action
     double         last_cpu_time_; // Last time a checkpoint was triggered
     bool           generate_;      // Whether a checkpoint should be done next time check() is called
     SimTime_t      next_sim_time_; // Next simulationt ime a checkpoint should trigger at or 0 if not applicable
+    std::string    dir_format_;    // Format string for checkpoint directory names
+    std::string    file_format_;   // Format string for checkpoint file names
 };
 
 } // namespace SST