From a4b6e910ef11489acc2450688ebfc8b03e533728 Mon Sep 17 00:00:00 2001 From: John Leidel Date: Wed, 5 Mar 2025 13:05:43 -0600 Subject: [PATCH 1/2] adding reader and writer paths for global statistics and statistic groups; Issue #1066 --- src/sst/core/cfgoutput/jsonConfigOutput.cc | 83 +++++++++++++- src/sst/core/model/json/jsonmodel.cc | 127 ++++++++++++++++++++- src/sst/core/model/json/jsonmodel.h | 2 + 3 files changed, 210 insertions(+), 2 deletions(-) diff --git a/src/sst/core/cfgoutput/jsonConfigOutput.cc b/src/sst/core/cfgoutput/jsonConfigOutput.cc index e606754a1..ba68d860a 100644 --- a/src/sst/core/cfgoutput/jsonConfigOutput.cc +++ b/src/sst/core/cfgoutput/jsonConfigOutput.cc @@ -53,6 +53,18 @@ struct StatPair SST::ConfigComponent const* comp; }; +struct StatGroupPair +{ + std::pair const& group; + SST::ConfigGraph const* graph; +}; + +struct StatGroupParamPair +{ + const std::string name; + const SST::Params& stat; +}; + void to_json(json::ordered_json& j, StatPair const& sp) { @@ -136,6 +148,52 @@ to_json(json::ordered_json& j, LinkConfPair const& pair) j["right"]["latency"] = link->latency_str[1]; } +void +to_json(json::ordered_json& j, StatGroupParamPair const& pair) +{ + auto const& outParams = pair.stat; + + j["name"] = pair.name; + + for ( auto const& param : outParams.getKeys() ){ + j["params"][param] = outParams.find(param); + } +} + +void +to_json(json::ordered_json& j, StatGroupPair const& pair) +{ + auto const& grp = pair.group.second; + auto const* graph = pair.graph; + + j["name"] = grp.name; + + if ( grp.outputFrequency.getValue() != 0 ) { + j["frequency"] = grp.outputFrequency.toStringBestSI(); + } + + if ( grp.outputID != 0 ) { + const SST::ConfigStatOutput& out = graph->getStatOutput(grp.outputID); + j["output"]["type"] = out.type; + if ( !out.params.empty() ) { + const SST::Params& outParams = graph->getStatOutput().params; + for( auto const& param : outParams.getKeys() ){ + j["output"]["params"][param] = outParams.find(param); + } + } + } + + for ( auto& i : grp.statMap ) { + if ( !i.second.empty() ) { + j["statistics"].emplace_back( StatGroupParamPair { i.first, i.second }); + } + } + + for ( SST::ComponentId_t id : grp.components ) { + const SST::ConfigComponent* comp = graph->findComponent(id); + j["components"].emplace_back( comp->name ); + } +} } // namespace void @@ -164,7 +222,6 @@ JSONConfigGraphOutput::generate(const Config* cfg, ConfigGraph* graph) outputJson["program_options"]["checkpoint-sim-period"] = cfg->checkpoint_sim_period(); outputJson["program_options"]["checkpoint-wall-period"] = std::to_string(cfg->checkpoint_wall_period()); - // Put in the global param sets for ( const auto& set : getGlobalParamSetNames() ) { for ( const auto& kvp : getGlobalParamSet(set) ) { @@ -172,6 +229,30 @@ JSONConfigGraphOutput::generate(const Config* cfg, ConfigGraph* graph) } } + // Global statistics + if ( 0 != graph->getStatLoadLevel() ) { + outputJson["statistics_options"]["statisticLoadLevel"] = (uint64_t)graph->getStatLoadLevel(); + } + + if ( !graph->getStatOutput().type.empty() ) { + outputJson["statistics_options"]["statisticOutput"] = graph->getStatOutput().type.c_str(); + const Params& outParams = graph->getStatOutput().params; + if ( !outParams.empty() ) { + // generate the parameters + for ( auto const& paramsItr : getParamsLocalKeys(outParams) ) { + outputJson["statistics_options"]["params"][paramsItr] = outParams.find(paramsItr); + } + } + } + + // Generate the stat groups + if ( !graph->getStatGroups().empty() ) { + outputJson["statistics_group"]; + for ( auto& grp : graph->getStatGroups() ) { + outputJson["statistics_group"].emplace_back(StatGroupPair { grp, graph }); + } + } + // no components exist in this rank if ( const_cast(compMap).size() == 0 ) { outputJson["components"]; } diff --git a/src/sst/core/model/json/jsonmodel.cc b/src/sst/core/model/json/jsonmodel.cc index 41a5c1a01..65bbb3cfa 100644 --- a/src/sst/core/model/json/jsonmodel.cc +++ b/src/sst/core/model/json/jsonmodel.cc @@ -294,6 +294,130 @@ SSTJSONModelDefinition::discoverGlobalParams(const json& jFile) } } +void +SSTJSONModelDefinition::setStatGroupOptions(const json& jFile) +{ + std::string Name; + std::string Frequency; + std::string Type; + std::string StatName; + + for ( auto& statArray : jFile["statistics_group"] ) { + // -- Name + auto x = statArray.find("name"); + if ( x != statArray.end() ) { Name = x.value(); } + else { + output->fatal( + CALL_INFO, 1, "Error discovering statistics group name from script: %s\n", scriptName.c_str()); + } + auto *csg = graph->getStatGroup(Name); + if( csg == nullptr ) { + output->fatal( + CALL_INFO, 1, + "Error creating statistics group from script %s; name=%s\n", + scriptName.c_str(), Name.c_str()); + } + + // -- Frequency + auto f = statArray.find("frequency"); + if ( f != statArray.end() ) { Frequency = f.value(); } + else { + output->fatal( + CALL_INFO, 1, "Error discovering statistics group frequency from script: %s\n", scriptName.c_str()); + } + + if ( !csg->setFrequency(Frequency) ){ + output->fatal( + CALL_INFO, 1, "Error setting frequency for statistics group: %s\n", + Name.c_str() ); + } + + // -- output + if ( statArray.contains("output") ) { + auto &statOuts = graph->getStatOutputs(); + if ( statArray.at( "output" ).contains( "type" ) ) { + statArray.at( "output" ).at( "type" ).get_to( Type ); + }else{ + output->fatal( + CALL_INFO, 1, + "Error discovering statistics group output type for group: %s\n", + Name.c_str() ); + } + + statOuts.emplace_back( ConfigStatOutput( Type ) ); + + if ( statArray.at("output").contains("params") ){ + for ( auto& paramArray : statArray.at( "output" ).at( "params" ).items() ){ + statOuts.back().addParameter( paramArray.key(), paramArray.value() ); + } + } + } + + // -- statistics + if ( statArray.contains("statistics") ) { + // -- stat name + if ( statArray.at("statistics").contains("name") ) { + auto sn = statArray.at("statistics").find("name"); + if ( sn != statArray.end() ) { StatName = sn.value(); } + else { + output->fatal( + CALL_INFO, 1, + "Error discovering statistics group stat name from script: %s\n", scriptName.c_str()); + } + } + + // -- stat params + Params StatParams; + if ( statArray.at("statistics").contains("params") ){ + for ( auto& paramArray : statArray.at("statistics").at("params").items() ){ + StatParams.insert( paramArray.key(), paramArray.value() ); + } + } + + csg->addStatistic( StatName, StatParams ); + } + + // -- components + if ( statArray.contains("components") ) { + for ( auto& compArray : statArray["components"].items() ) { + csg->addComponent(findComponentIdByName(compArray.value())); + } + } + } +} + +void +SSTJSONModelDefinition::discoverStatistics(const json& jFile) +{ + // discover the global statistics options + if ( jFile.contains("statistics_options") ) { + for ( auto& option : jFile["statistics_options"] ) { + // -- statisticLoadLevel + auto ll = option.find("statisticLoadLevel"); + if( ll != option.end() ) { + graph->setStatisticLoadLevel( ll.value() ); + } + + // -- statisticOutput + auto so = option.find("statisticOutput"); + if( so != option.end() ) { + graph->setStatisticOutput( so.value() ); + } + + // -- params + if ( option.contains("params") ){ + for ( auto& paramArray : option.at("params").items() ){ + graph->addStatisticOutputParameter( paramArray.key(), paramArray.value() ); + } + } + } + } + // discover the statistics groups + if ( jFile.contains("statistics_group") ) { + setStatGroupOptions(jFile); + } +} + ConfigGraph* SSTJSONModelDefinition::createConfigGraph() { @@ -322,7 +446,8 @@ SSTJSONModelDefinition::createConfigGraph() // discover the links discoverLinks(jFile); - // TODO: discover statistics + // discover statistics + discoverStatistics(jFile); return graph; } diff --git a/src/sst/core/model/json/jsonmodel.h b/src/sst/core/model/json/jsonmodel.h index b0d68081d..fb3ae0693 100644 --- a/src/sst/core/model/json/jsonmodel.h +++ b/src/sst/core/model/json/jsonmodel.h @@ -65,11 +65,13 @@ class SSTJSONModelDefinition : public SSTModelDescription double start_time; private: + void setStatGroupOptions(const json& jFile); void recursiveSubcomponent(ConfigComponent* Parent, const nlohmann::basic_json<>& compArray); void discoverProgramOptions(const json& jFile); void discoverComponents(const json& jFile); void discoverLinks(const json& jFile); void discoverGlobalParams(const json& jFile); + void discoverStatistics(const json& jFile); ComponentId_t findComponentIdByName(const std::string& Name); }; From 28d5b7d710db4484f2e75402be507d9bee12b9f5 Mon Sep 17 00:00:00 2001 From: John Leidel Date: Wed, 5 Mar 2025 13:22:42 -0600 Subject: [PATCH 2/2] formatting source per clang format rules --- src/sst/core/cfgoutput/jsonConfigOutput.cc | 60 ++++++------ src/sst/core/model/json/jsonmodel.cc | 106 +++++++++------------ 2 files changed, 75 insertions(+), 91 deletions(-) diff --git a/src/sst/core/cfgoutput/jsonConfigOutput.cc b/src/sst/core/cfgoutput/jsonConfigOutput.cc index ba68d860a..76ba26d62 100644 --- a/src/sst/core/cfgoutput/jsonConfigOutput.cc +++ b/src/sst/core/cfgoutput/jsonConfigOutput.cc @@ -56,12 +56,12 @@ struct StatPair struct StatGroupPair { std::pair const& group; - SST::ConfigGraph const* graph; + SST::ConfigGraph const* graph; }; struct StatGroupParamPair { - const std::string name; + const std::string name; const SST::Params& stat; }; @@ -155,43 +155,39 @@ to_json(json::ordered_json& j, StatGroupParamPair const& pair) j["name"] = pair.name; - for ( auto const& param : outParams.getKeys() ){ - j["params"][param] = outParams.find(param); + for ( auto const& param : outParams.getKeys() ) { + j["params"][param] = outParams.find(param); } } void to_json(json::ordered_json& j, StatGroupPair const& pair) { - auto const& grp = pair.group.second; + auto const& grp = pair.group.second; auto const* graph = pair.graph; j["name"] = grp.name; - if ( grp.outputFrequency.getValue() != 0 ) { - j["frequency"] = grp.outputFrequency.toStringBestSI(); - } + if ( grp.outputFrequency.getValue() != 0 ) { j["frequency"] = grp.outputFrequency.toStringBestSI(); } if ( grp.outputID != 0 ) { - const SST::ConfigStatOutput& out = graph->getStatOutput(grp.outputID); - j["output"]["type"] = out.type; - if ( !out.params.empty() ) { - const SST::Params& outParams = graph->getStatOutput().params; - for( auto const& param : outParams.getKeys() ){ - j["output"]["params"][param] = outParams.find(param); + const SST::ConfigStatOutput& out = graph->getStatOutput(grp.outputID); + j["output"]["type"] = out.type; + if ( !out.params.empty() ) { + const SST::Params& outParams = graph->getStatOutput().params; + for ( auto const& param : outParams.getKeys() ) { + j["output"]["params"][param] = outParams.find(param); + } } - } } for ( auto& i : grp.statMap ) { - if ( !i.second.empty() ) { - j["statistics"].emplace_back( StatGroupParamPair { i.first, i.second }); - } + if ( !i.second.empty() ) { j["statistics"].emplace_back(StatGroupParamPair { i.first, i.second }); } } for ( SST::ComponentId_t id : grp.components ) { - const SST::ConfigComponent* comp = graph->findComponent(id); - j["components"].emplace_back( comp->name ); + const SST::ConfigComponent* comp = graph->findComponent(id); + j["components"].emplace_back(comp->name); } } } // namespace @@ -231,26 +227,26 @@ JSONConfigGraphOutput::generate(const Config* cfg, ConfigGraph* graph) // Global statistics if ( 0 != graph->getStatLoadLevel() ) { - outputJson["statistics_options"]["statisticLoadLevel"] = (uint64_t)graph->getStatLoadLevel(); + outputJson["statistics_options"]["statisticLoadLevel"] = (uint64_t)graph->getStatLoadLevel(); } if ( !graph->getStatOutput().type.empty() ) { - outputJson["statistics_options"]["statisticOutput"] = graph->getStatOutput().type.c_str(); - const Params& outParams = graph->getStatOutput().params; - if ( !outParams.empty() ) { - // generate the parameters - for ( auto const& paramsItr : getParamsLocalKeys(outParams) ) { - outputJson["statistics_options"]["params"][paramsItr] = outParams.find(paramsItr); + outputJson["statistics_options"]["statisticOutput"] = graph->getStatOutput().type.c_str(); + const Params& outParams = graph->getStatOutput().params; + if ( !outParams.empty() ) { + // generate the parameters + for ( auto const& paramsItr : getParamsLocalKeys(outParams) ) { + outputJson["statistics_options"]["params"][paramsItr] = outParams.find(paramsItr); + } } - } } // Generate the stat groups if ( !graph->getStatGroups().empty() ) { - outputJson["statistics_group"]; - for ( auto& grp : graph->getStatGroups() ) { - outputJson["statistics_group"].emplace_back(StatGroupPair { grp, graph }); - } + outputJson["statistics_group"]; + for ( auto& grp : graph->getStatGroups() ) { + outputJson["statistics_group"].emplace_back(StatGroupPair { grp, graph }); + } } // no components exist in this rank diff --git a/src/sst/core/model/json/jsonmodel.cc b/src/sst/core/model/json/jsonmodel.cc index 65bbb3cfa..b29f1289a 100644 --- a/src/sst/core/model/json/jsonmodel.cc +++ b/src/sst/core/model/json/jsonmodel.cc @@ -297,10 +297,10 @@ SSTJSONModelDefinition::discoverGlobalParams(const json& jFile) void SSTJSONModelDefinition::setStatGroupOptions(const json& jFile) { - std::string Name; - std::string Frequency; - std::string Type; - std::string StatName; + std::string Name; + std::string Frequency; + std::string Type; + std::string StatName; for ( auto& statArray : jFile["statistics_group"] ) { // -- Name @@ -310,12 +310,11 @@ SSTJSONModelDefinition::setStatGroupOptions(const json& jFile) output->fatal( CALL_INFO, 1, "Error discovering statistics group name from script: %s\n", scriptName.c_str()); } - auto *csg = graph->getStatGroup(Name); - if( csg == nullptr ) { - output->fatal( - CALL_INFO, 1, - "Error creating statistics group from script %s; name=%s\n", - scriptName.c_str(), Name.c_str()); + auto* csg = graph->getStatGroup(Name); + if ( csg == nullptr ) { + output->fatal( + CALL_INFO, 1, "Error creating statistics group from script %s; name=%s\n", scriptName.c_str(), + Name.c_str()); } // -- Frequency @@ -326,61 +325,56 @@ SSTJSONModelDefinition::setStatGroupOptions(const json& jFile) CALL_INFO, 1, "Error discovering statistics group frequency from script: %s\n", scriptName.c_str()); } - if ( !csg->setFrequency(Frequency) ){ - output->fatal( - CALL_INFO, 1, "Error setting frequency for statistics group: %s\n", - Name.c_str() ); + if ( !csg->setFrequency(Frequency) ) { + output->fatal(CALL_INFO, 1, "Error setting frequency for statistics group: %s\n", Name.c_str()); } // -- output if ( statArray.contains("output") ) { - auto &statOuts = graph->getStatOutputs(); - if ( statArray.at( "output" ).contains( "type" ) ) { - statArray.at( "output" ).at( "type" ).get_to( Type ); - }else{ - output->fatal( - CALL_INFO, 1, - "Error discovering statistics group output type for group: %s\n", - Name.c_str() ); - } + auto& statOuts = graph->getStatOutputs(); + if ( statArray.at("output").contains("type") ) { statArray.at("output").at("type").get_to(Type); } + else { + output->fatal( + CALL_INFO, 1, "Error discovering statistics group output type for group: %s\n", Name.c_str()); + } - statOuts.emplace_back( ConfigStatOutput( Type ) ); + statOuts.emplace_back(ConfigStatOutput(Type)); - if ( statArray.at("output").contains("params") ){ - for ( auto& paramArray : statArray.at( "output" ).at( "params" ).items() ){ - statOuts.back().addParameter( paramArray.key(), paramArray.value() ); + if ( statArray.at("output").contains("params") ) { + for ( auto& paramArray : statArray.at("output").at("params").items() ) { + statOuts.back().addParameter(paramArray.key(), paramArray.value()); + } } - } } // -- statistics if ( statArray.contains("statistics") ) { - // -- stat name - if ( statArray.at("statistics").contains("name") ) { - auto sn = statArray.at("statistics").find("name"); - if ( sn != statArray.end() ) { StatName = sn.value(); } - else { - output->fatal( - CALL_INFO, 1, - "Error discovering statistics group stat name from script: %s\n", scriptName.c_str()); + // -- stat name + if ( statArray.at("statistics").contains("name") ) { + auto sn = statArray.at("statistics").find("name"); + if ( sn != statArray.end() ) { StatName = sn.value(); } + else { + output->fatal( + CALL_INFO, 1, "Error discovering statistics group stat name from script: %s\n", + scriptName.c_str()); + } } - } - // -- stat params - Params StatParams; - if ( statArray.at("statistics").contains("params") ){ - for ( auto& paramArray : statArray.at("statistics").at("params").items() ){ - StatParams.insert( paramArray.key(), paramArray.value() ); - } - } + // -- stat params + Params StatParams; + if ( statArray.at("statistics").contains("params") ) { + for ( auto& paramArray : statArray.at("statistics").at("params").items() ) { + StatParams.insert(paramArray.key(), paramArray.value()); + } + } - csg->addStatistic( StatName, StatParams ); + csg->addStatistic(StatName, StatParams); } // -- components if ( statArray.contains("components") ) { for ( auto& compArray : statArray["components"].items() ) { - csg->addComponent(findComponentIdByName(compArray.value())); + csg->addComponent(findComponentIdByName(compArray.value())); } } } @@ -394,28 +388,22 @@ SSTJSONModelDefinition::discoverStatistics(const json& jFile) for ( auto& option : jFile["statistics_options"] ) { // -- statisticLoadLevel auto ll = option.find("statisticLoadLevel"); - if( ll != option.end() ) { - graph->setStatisticLoadLevel( ll.value() ); - } + if ( ll != option.end() ) { graph->setStatisticLoadLevel(ll.value()); } // -- statisticOutput auto so = option.find("statisticOutput"); - if( so != option.end() ) { - graph->setStatisticOutput( so.value() ); - } + if ( so != option.end() ) { graph->setStatisticOutput(so.value()); } // -- params - if ( option.contains("params") ){ - for ( auto& paramArray : option.at("params").items() ){ - graph->addStatisticOutputParameter( paramArray.key(), paramArray.value() ); - } + if ( option.contains("params") ) { + for ( auto& paramArray : option.at("params").items() ) { + graph->addStatisticOutputParameter(paramArray.key(), paramArray.value()); + } } } } // discover the statistics groups - if ( jFile.contains("statistics_group") ) { - setStatGroupOptions(jFile); - } + if ( jFile.contains("statistics_group") ) { setStatGroupOptions(jFile); } } ConfigGraph*