Adding JSON statistics readers and writers

src/sst/core/cfgoutput/jsonConfigOutput.cc

@@ -53,6 +53,19 @@ struct StatPair
     SST::ConfigComponent const*                  comp;
 };
 
+struct StatGroupPair
+{
+    std::pair<const std::string, SST::ConfigStatGroup> const& group;
+    std::vector<std::string>                                  vec;
+    SST::ConfigGraph const*                                   graph;
+};
+
+struct StatGroupParamPair
+{
+    const std::string  name;
+    const SST::Params& stat;
+};
+
 void
 to_json(json::ordered_json& j, StatPair const& sp)
 {
@@ -136,6 +149,49 @@ to_json(json::ordered_json& j, LinkConfPair const& pair)
     j["right"]["latency"]   = link->latency_str[1];
 }
 
+void
+to_json(json::ordered_json& j, StatGroupParamPair const& pair)
+{
+    auto const& outParams = pair.stat;
+
+    j["name"] = pair.name;
+
+    for ( auto const& param : outParams.getKeys() ) {
+        j["params"][param] = outParams.find<std::string>(param);
+    }
+}
+
+void
+to_json(json::ordered_json& j, StatGroupPair const& pair)
+{
+    auto const& grp   = pair.group.second;
+    auto const* graph = pair.graph;
+    auto        vec   = pair.vec;
+
+    j["name"] = grp.name;
+
+    if ( grp.outputFrequency.getValue() != 0 ) { j["frequency"] = grp.outputFrequency.toStringBestSI(); }
+
+    if ( grp.outputID != 0 ) {
+        const SST::ConfigStatOutput& out = graph->getStatOutput(grp.outputID);
+        j["output"]["type"]              = out.type;
+        if ( !out.params.empty() ) {
+            const SST::Params& outParams = out.params;
+            for ( auto const& param : vec ) {
+                j["output"]["params"][param] = outParams.find<std::string>(param);
+            }
+        }
+    }
+
+    for ( auto& i : grp.statMap ) {
+        if ( !i.second.empty() ) { j["statistics"].emplace_back(StatGroupParamPair { i.first, i.second }); }
+    }
+
+    for ( SST::ComponentId_t id : grp.components ) {
+        const SST::ConfigComponent* comp = graph->findComponent(id);
+        j["components"].emplace_back(comp->name);
+    }
+}
 } // namespace
 
 void
@@ -164,14 +220,38 @@ JSONConfigGraphOutput::generate(const Config* cfg, ConfigGraph* graph)
     outputJson["program_options"]["checkpoint-sim-period"]  = cfg->checkpoint_sim_period();
     outputJson["program_options"]["checkpoint-wall-period"] = std::to_string(cfg->checkpoint_wall_period());
 
-
     // Put in the global param sets
     for ( const auto& set : getGlobalParamSetNames() ) {
         for ( const auto& kvp : getGlobalParamSet(set) ) {
             if ( kvp.first != "<set_name>" ) outputJson["global_params"][set][kvp.first] = kvp.second;
         }
     }
 
+    // Global statistics
+    if ( 0 != graph->getStatLoadLevel() ) {
+        outputJson["statistics_options"]["statisticLoadLevel"] = (uint64_t)graph->getStatLoadLevel();
+    }
+
+    if ( !graph->getStatOutput().type.empty() ) {
+        outputJson["statistics_options"]["statisticOutput"] = graph->getStatOutput().type.c_str();
+        const Params& outParams                             = graph->getStatOutput().params;
+        if ( !outParams.empty() ) {
+            // generate the parameters
+            for ( auto const& paramsItr : getParamsLocalKeys(outParams) ) {
+                outputJson["statistics_options"]["params"][paramsItr] = outParams.find<std::string>(paramsItr);
+            }
+        }
+    }
+
+    // Generate the stat groups
+    if ( !graph->getStatGroups().empty() ) {
+        outputJson["statistics_group"];
+        for ( auto& grp : graph->getStatGroups() ) {
+            auto vec = getParamsLocalKeys(graph->getStatOutput(grp.second.outputID).params);
+            outputJson["statistics_group"].emplace_back(StatGroupPair { grp, vec, graph });
+        }
+    }
+
     // no components exist in this rank
     if ( const_cast<ConfigComponentMap_t&>(compMap).size() == 0 ) { outputJson["components"]; }
 

src/sst/core/model/json/jsonmodel.cc

@@ -71,6 +71,7 @@ SSTJSONModelDefinition::recursiveSubcomponent(ConfigComponent* Parent, const nlo
 {
     std::string      Name;
     std::string      Type;
+    std::string      StatName;
     ConfigComponent* Comp = nullptr;
     int              Slot = 0;
 
@@ -119,6 +120,32 @@ SSTJSONModelDefinition::recursiveSubcomponent(ConfigComponent* Parent, const nlo
             }
         }
 
+        // read the statistics
+        if ( subArray.contains("statistics") ) {
+            for ( auto& stats : compArray.at("statistics") ) {
+                // -- stat name
+                if ( stats.contains("name") ) {
+                    auto sn = stats.find("name");
+                    if ( sn != stats.end() ) { StatName = sn.value(); }
+                    else {
+                        output->fatal(
+                            CALL_INFO, 1, "Error discovering component stat name from script: %s\n",
+                            scriptName.c_str());
+                    }
+                }
+
+                // -- stat params
+                Params StatParams;
+                if ( stats.contains("params") ) {
+                    for ( auto& paramArray : stats.at("params").items() ) {
+                        StatParams.insert(paramArray.key(), paramArray.value());
+                    }
+                }
+
+                Comp->enableStatistic(StatName, StatParams);
+            }
+        }
+
         // recursively build up the subcomponents
         if ( subArray.contains("subcomponents") ) {
             auto& subsubArray = subArray["subcomponents"];
@@ -132,6 +159,7 @@ SSTJSONModelDefinition::discoverComponents(const json& jFile)
 {
     std::string      Name;
     std::string      Type;
+    std::string      StatName;
     ComponentId_t    Id;
     ConfigComponent* Comp   = nullptr;
     uint32_t         rank   = 0;
@@ -185,6 +213,32 @@ SSTJSONModelDefinition::discoverComponents(const json& jFile)
             }
         }
 
+        // read the statistics
+        if ( compArray.contains("statistics") ) {
+            for ( auto& stats : compArray.at("statistics") ) {
+                // -- stat name
+                if ( stats.contains("name") ) {
+                    auto sn = stats.find("name");
+                    if ( sn != stats.end() ) { StatName = sn.value(); }
+                    else {
+                        output->fatal(
+                            CALL_INFO, 1, "Error discovering component stat name from script: %s\n",
+                            scriptName.c_str());
+                    }
+                }
+
+                // -- stat params
+                Params StatParams;
+                if ( stats.contains("params") ) {
+                    for ( auto& paramArray : stats.at("params").items() ) {
+                        StatParams.insert(paramArray.key(), paramArray.value());
+                    }
+                }
+
+                Comp->enableStatistic(StatName, StatParams);
+            }
+        }
+
         // set the rank information
         RankInfo Rank(rank, thread);
         Comp->setRank(Rank);
@@ -294,6 +348,127 @@ SSTJSONModelDefinition::discoverGlobalParams(const json& jFile)
     }
 }
 
+void
+SSTJSONModelDefinition::setStatGroupOptions(const json& jFile)
+{
+    std::string Name;
+    std::string Frequency;
+    std::string Type;
+    std::string StatName;
+
+    for ( auto& statArray : jFile.at("statistics_group") ) {
+        // -- name
+        auto x = statArray.find("name");
+        if ( x != statArray.end() ) { Name = x.value(); }
+        else {
+            output->fatal(
+                CALL_INFO, 1, "Error discovering statistics group name from script: %s\n", scriptName.c_str());
+        }
+
+        auto* csg = graph->getStatGroup(Name);
+        if ( csg == nullptr ) {
+            output->fatal(
+                CALL_INFO, 1, "Error creating statistics group from script %s; name=%s\n", scriptName.c_str(),
+                Name.c_str());
+        }
+
+        // -- frequency
+        auto f = statArray.find("frequency");
+        if ( f != statArray.end() ) {
+            Frequency = f.value();
+            if ( !csg->setFrequency(Frequency) ) {
+                output->fatal(CALL_INFO, 1, "Error setting frequency for statistics group: %s\n", Name.c_str());
+            }
+        }
+
+        // -- output
+        if ( statArray.contains("output") ) {
+            auto& statOuts = graph->getStatOutputs();
+            if ( statArray.at("output").contains("type") ) { statArray.at("output").at("type").get_to(Type); }
+            else {
+                output->fatal(
+                    CALL_INFO, 1, "Error discovering statistics group output type for group: %s\n", Name.c_str());
+            }
+
+            statOuts.emplace_back(ConfigStatOutput(Type));
+            csg->setOutput(statOuts.size() - 1);
+
+            if ( statArray.at("output").contains("params") ) {
+                for ( auto& paramArray : statArray.at("output").at("params").items() ) {
+                    statOuts.back().addParameter(paramArray.key(), paramArray.value());
+                }
+            }
+        }
+
+        // -- statistics
+        if ( statArray.contains("statistics") ) {
+            for ( auto& stats : statArray.at("statistics") ) {
+                // -- stat name
+                if ( stats.contains("name") ) {
+                    auto sn = stats.find("name");
+                    if ( sn != stats.end() ) { StatName = sn.value(); }
+                    else {
+                        output->fatal(
+                            CALL_INFO, 1, "Error discovering statistics group stat name from script: %s\n",
+                            scriptName.c_str());
+                    }
+                }
+
+                // -- stat params
+                Params StatParams;
+                if ( stats.contains("params") ) {
+                    for ( auto& paramArray : stats.at("params").items() ) {
+                        StatParams.insert(paramArray.key(), paramArray.value());
+                    }
+                }
+
+                csg->addStatistic(StatName, StatParams);
+
+                bool        verified;
+                std::string reason;
+                std::tie(verified, reason) = csg->verifyStatsAndComponents(graph);
+                if ( !verified ) {
+                    output->fatal(CALL_INFO, 1, "Error verifying statistics and components: %s\n", reason.c_str());
+                }
+            }
+        }
+
+        // -- components
+        if ( statArray.contains("components") ) {
+            for ( auto& compArray : statArray["components"].items() ) {
+                csg->addComponent(findComponentIdByName(compArray.value()));
+            }
+        }
+    }
+}
+
+void
+SSTJSONModelDefinition::discoverStatistics(const json& jFile)
+{
+    // discover the global statistics options
+    if ( jFile.contains("statistics_options") ) {
+        if ( jFile.at("statistics_options").contains("statisticLoadLevel") ) {
+            uint8_t loadLevel;
+            jFile.at("statistics_options").at("statisticLoadLevel").get_to(loadLevel);
+            graph->setStatisticLoadLevel(loadLevel);
+        }
+
+        if ( jFile.at("statistics_options").contains("statisticOutput") ) {
+            std::string output;
+            jFile.at("statistics_options").at("statisticOutput").get_to(output);
+            graph->setStatisticOutput(output);
+        }
+
+        if ( jFile.at("statistics_options").contains("params") ) {
+            for ( auto& paramArray : jFile.at("statistics_options").at("params").items() ) {
+                graph->addStatisticOutputParameter(paramArray.key(), paramArray.value());
+            }
+        }
+    }
+    // discover the statistics groups
+    if ( jFile.contains("statistics_group") ) { setStatGroupOptions(jFile); }
+}
+
 ConfigGraph*
 SSTJSONModelDefinition::createConfigGraph()
 {
@@ -322,7 +497,8 @@ SSTJSONModelDefinition::createConfigGraph()
     // discover the links
     discoverLinks(jFile);
 
-    // TODO: discover statistics
+    // discover statistics
+    discoverStatistics(jFile);
 
     return graph;
 }

src/sst/core/model/json/jsonmodel.h

@@ -65,11 +65,13 @@ class SSTJSONModelDefinition : public SSTModelDescription
     double        start_time;
 
 private:
+    void          setStatGroupOptions(const json& jFile);
     void          recursiveSubcomponent(ConfigComponent* Parent, const nlohmann::basic_json<>& compArray);
     void          discoverProgramOptions(const json& jFile);
     void          discoverComponents(const json& jFile);
     void          discoverLinks(const json& jFile);
     void          discoverGlobalParams(const json& jFile);
+    void          discoverStatistics(const json& jFile);
     ComponentId_t findComponentIdByName(const std::string& Name);
 };
 

src/sst/core/model/python/pymodel.cc

@@ -648,7 +648,7 @@ enableStatisticsForComponentName(PyObject* UNUSED(self), PyObject* args)
         apply_to_children = 0;
         // Try list version
         argOK             = PyArg_ParseTuple(
-            args, "sO!|O!i", &compName, &PyList_Type, &statList, &PyDict_Type, &statParamDict, &apply_to_children);
+                        args, "sO!|O!i", &compName, &PyList_Type, &statList, &PyDict_Type, &statParamDict, &apply_to_children);
         if ( argOK ) Py_INCREF(statList);
     }
 
@@ -765,7 +765,7 @@ enableStatisticsForComponentType(PyObject* UNUSED(self), PyObject* args)
         apply_to_children = 0;
         // Try list version
         argOK             = PyArg_ParseTuple(
-            args, "sO!|O!i", &compType, &PyList_Type, &statList, &PyDict_Type, &statParamDict, &apply_to_children);
+                        args, "sO!|O!i", &compType, &PyList_Type, &statList, &PyDict_Type, &statParamDict, &apply_to_children);
         if ( argOK ) Py_INCREF(statList);
     }
 

src/sst/core/statapi/statoutputhdf5.cc

@@ -433,7 +433,7 @@ StatisticOutputHDF5::GroupInfo::finalizeGroupRegistration()
     H5::DataSet*  idSet =
         new H5::DataSet(getFile()->createDataSet(groupName + "/ids", H5::PredType::NATIVE_UINT64, infoSpace, cparms));
     H5::DataSet* nameSet   = new H5::DataSet(getFile()->createDataSet(
-        groupName + "/names", H5::StrType(H5::PredType::C_S1, H5T_VARIABLE), infoSpace, cparms));
+          groupName + "/names", H5::StrType(H5::PredType::C_S1, H5T_VARIABLE), infoSpace, cparms));
     H5::DataSet* coordXSet = new H5::DataSet(
         getFile()->createDataSet(groupName + "/coord_x", H5::PredType::NATIVE_DOUBLE, infoSpace, cparms));
     H5::DataSet* coordYSet = new H5::DataSet(

src/sst/core/testElements/coreTest_ClockerComponent.cc

@@ -57,7 +57,8 @@ coreTestClockerComponent::coreTestClockerComponent() : Component(-1)
     // for serialization only
 }
 
-bool coreTestClockerComponent::tick(Cycle_t)
+bool
+coreTestClockerComponent::tick(Cycle_t)
 {
     clock_count--;
 

src/sst/core/testElements/coreTest_Component.cc

@@ -99,7 +99,8 @@ coreTestComponent::handleEvent(Event* ev)
 // each clock tick we do 'workPerCycle' iterations of a coreTest loop.
 // We have a 1/commFreq chance of sending an event of size commSize to
 // one of our neighbors.
-bool coreTestComponent::clockTic(Cycle_t)
+bool
+coreTestComponent::clockTic(Cycle_t)
 {
     // do work
     // loop becomes: