diff --git a/src/sst/core/cfgoutput/jsonConfigOutput.cc b/src/sst/core/cfgoutput/jsonConfigOutput.cc index e606754a1..e1d7b414a 100644 --- a/src/sst/core/cfgoutput/jsonConfigOutput.cc +++ b/src/sst/core/cfgoutput/jsonConfigOutput.cc @@ -53,6 +53,19 @@ struct StatPair SST::ConfigComponent const* comp; }; +struct StatGroupPair +{ + std::pair const& group; + std::vector vec; + SST::ConfigGraph const* graph; +}; + +struct StatGroupParamPair +{ + const std::string name; + const SST::Params& stat; +}; + void to_json(json::ordered_json& j, StatPair const& sp) { @@ -136,6 +149,49 @@ to_json(json::ordered_json& j, LinkConfPair const& pair) j["right"]["latency"] = link->latency_str[1]; } +void +to_json(json::ordered_json& j, StatGroupParamPair const& pair) +{ + auto const& outParams = pair.stat; + + j["name"] = pair.name; + + for ( auto const& param : outParams.getKeys() ) { + j["params"][param] = outParams.find(param); + } +} + +void +to_json(json::ordered_json& j, StatGroupPair const& pair) +{ + auto const& grp = pair.group.second; + auto const* graph = pair.graph; + auto vec = pair.vec; + + j["name"] = grp.name; + + if ( grp.outputFrequency.getValue() != 0 ) { j["frequency"] = grp.outputFrequency.toStringBestSI(); } + + if ( grp.outputID != 0 ) { + const SST::ConfigStatOutput& out = graph->getStatOutput(grp.outputID); + j["output"]["type"] = out.type; + if ( !out.params.empty() ) { + const SST::Params& outParams = out.params; + for ( auto const& param : vec ) { + j["output"]["params"][param] = outParams.find(param); + } + } + } + + for ( auto& i : grp.statMap ) { + if ( !i.second.empty() ) { j["statistics"].emplace_back(StatGroupParamPair { i.first, i.second }); } + } + + for ( SST::ComponentId_t id : grp.components ) { + const SST::ConfigComponent* comp = graph->findComponent(id); + j["components"].emplace_back(comp->name); + } +} } // namespace void @@ -164,7 +220,6 @@ JSONConfigGraphOutput::generate(const Config* cfg, ConfigGraph* graph) outputJson["program_options"]["checkpoint-sim-period"] = cfg->checkpoint_sim_period(); outputJson["program_options"]["checkpoint-wall-period"] = std::to_string(cfg->checkpoint_wall_period()); - // Put in the global param sets for ( const auto& set : getGlobalParamSetNames() ) { for ( const auto& kvp : getGlobalParamSet(set) ) { @@ -172,6 +227,31 @@ JSONConfigGraphOutput::generate(const Config* cfg, ConfigGraph* graph) } } + // Global statistics + if ( 0 != graph->getStatLoadLevel() ) { + outputJson["statistics_options"]["statisticLoadLevel"] = (uint64_t)graph->getStatLoadLevel(); + } + + if ( !graph->getStatOutput().type.empty() ) { + outputJson["statistics_options"]["statisticOutput"] = graph->getStatOutput().type.c_str(); + const Params& outParams = graph->getStatOutput().params; + if ( !outParams.empty() ) { + // generate the parameters + for ( auto const& paramsItr : getParamsLocalKeys(outParams) ) { + outputJson["statistics_options"]["params"][paramsItr] = outParams.find(paramsItr); + } + } + } + + // Generate the stat groups + if ( !graph->getStatGroups().empty() ) { + outputJson["statistics_group"]; + for ( auto& grp : graph->getStatGroups() ) { + auto vec = getParamsLocalKeys(graph->getStatOutput(grp.second.outputID).params); + outputJson["statistics_group"].emplace_back(StatGroupPair { grp, vec, graph }); + } + } + // no components exist in this rank if ( const_cast(compMap).size() == 0 ) { outputJson["components"]; } diff --git a/src/sst/core/model/json/jsonmodel.cc b/src/sst/core/model/json/jsonmodel.cc index 41a5c1a01..f7745e3d9 100644 --- a/src/sst/core/model/json/jsonmodel.cc +++ b/src/sst/core/model/json/jsonmodel.cc @@ -71,6 +71,7 @@ SSTJSONModelDefinition::recursiveSubcomponent(ConfigComponent* Parent, const nlo { std::string Name; std::string Type; + std::string StatName; ConfigComponent* Comp = nullptr; int Slot = 0; @@ -119,6 +120,32 @@ SSTJSONModelDefinition::recursiveSubcomponent(ConfigComponent* Parent, const nlo } } + // read the statistics + if ( subArray.contains("statistics") ) { + for ( auto& stats : compArray.at("statistics") ) { + // -- stat name + if ( stats.contains("name") ) { + auto sn = stats.find("name"); + if ( sn != stats.end() ) { StatName = sn.value(); } + else { + output->fatal( + CALL_INFO, 1, "Error discovering component stat name from script: %s\n", + scriptName.c_str()); + } + } + + // -- stat params + Params StatParams; + if ( stats.contains("params") ) { + for ( auto& paramArray : stats.at("params").items() ) { + StatParams.insert(paramArray.key(), paramArray.value()); + } + } + + Comp->enableStatistic(StatName, StatParams); + } + } + // recursively build up the subcomponents if ( subArray.contains("subcomponents") ) { auto& subsubArray = subArray["subcomponents"]; @@ -132,6 +159,7 @@ SSTJSONModelDefinition::discoverComponents(const json& jFile) { std::string Name; std::string Type; + std::string StatName; ComponentId_t Id; ConfigComponent* Comp = nullptr; uint32_t rank = 0; @@ -185,6 +213,32 @@ SSTJSONModelDefinition::discoverComponents(const json& jFile) } } + // read the statistics + if ( compArray.contains("statistics") ) { + for ( auto& stats : compArray.at("statistics") ) { + // -- stat name + if ( stats.contains("name") ) { + auto sn = stats.find("name"); + if ( sn != stats.end() ) { StatName = sn.value(); } + else { + output->fatal( + CALL_INFO, 1, "Error discovering component stat name from script: %s\n", + scriptName.c_str()); + } + } + + // -- stat params + Params StatParams; + if ( stats.contains("params") ) { + for ( auto& paramArray : stats.at("params").items() ) { + StatParams.insert(paramArray.key(), paramArray.value()); + } + } + + Comp->enableStatistic(StatName, StatParams); + } + } + // set the rank information RankInfo Rank(rank, thread); Comp->setRank(Rank); @@ -294,6 +348,127 @@ SSTJSONModelDefinition::discoverGlobalParams(const json& jFile) } } +void +SSTJSONModelDefinition::setStatGroupOptions(const json& jFile) +{ + std::string Name; + std::string Frequency; + std::string Type; + std::string StatName; + + for ( auto& statArray : jFile.at("statistics_group") ) { + // -- name + auto x = statArray.find("name"); + if ( x != statArray.end() ) { Name = x.value(); } + else { + output->fatal( + CALL_INFO, 1, "Error discovering statistics group name from script: %s\n", scriptName.c_str()); + } + + auto* csg = graph->getStatGroup(Name); + if ( csg == nullptr ) { + output->fatal( + CALL_INFO, 1, "Error creating statistics group from script %s; name=%s\n", scriptName.c_str(), + Name.c_str()); + } + + // -- frequency + auto f = statArray.find("frequency"); + if ( f != statArray.end() ) { + Frequency = f.value(); + if ( !csg->setFrequency(Frequency) ) { + output->fatal(CALL_INFO, 1, "Error setting frequency for statistics group: %s\n", Name.c_str()); + } + } + + // -- output + if ( statArray.contains("output") ) { + auto& statOuts = graph->getStatOutputs(); + if ( statArray.at("output").contains("type") ) { statArray.at("output").at("type").get_to(Type); } + else { + output->fatal( + CALL_INFO, 1, "Error discovering statistics group output type for group: %s\n", Name.c_str()); + } + + statOuts.emplace_back(ConfigStatOutput(Type)); + csg->setOutput(statOuts.size() - 1); + + if ( statArray.at("output").contains("params") ) { + for ( auto& paramArray : statArray.at("output").at("params").items() ) { + statOuts.back().addParameter(paramArray.key(), paramArray.value()); + } + } + } + + // -- statistics + if ( statArray.contains("statistics") ) { + for ( auto& stats : statArray.at("statistics") ) { + // -- stat name + if ( stats.contains("name") ) { + auto sn = stats.find("name"); + if ( sn != stats.end() ) { StatName = sn.value(); } + else { + output->fatal( + CALL_INFO, 1, "Error discovering statistics group stat name from script: %s\n", + scriptName.c_str()); + } + } + + // -- stat params + Params StatParams; + if ( stats.contains("params") ) { + for ( auto& paramArray : stats.at("params").items() ) { + StatParams.insert(paramArray.key(), paramArray.value()); + } + } + + csg->addStatistic(StatName, StatParams); + + bool verified; + std::string reason; + std::tie(verified, reason) = csg->verifyStatsAndComponents(graph); + if ( !verified ) { + output->fatal(CALL_INFO, 1, "Error verifying statistics and components: %s\n", reason.c_str()); + } + } + } + + // -- components + if ( statArray.contains("components") ) { + for ( auto& compArray : statArray["components"].items() ) { + csg->addComponent(findComponentIdByName(compArray.value())); + } + } + } +} + +void +SSTJSONModelDefinition::discoverStatistics(const json& jFile) +{ + // discover the global statistics options + if ( jFile.contains("statistics_options") ) { + if ( jFile.at("statistics_options").contains("statisticLoadLevel") ) { + uint8_t loadLevel; + jFile.at("statistics_options").at("statisticLoadLevel").get_to(loadLevel); + graph->setStatisticLoadLevel(loadLevel); + } + + if ( jFile.at("statistics_options").contains("statisticOutput") ) { + std::string output; + jFile.at("statistics_options").at("statisticOutput").get_to(output); + graph->setStatisticOutput(output); + } + + if ( jFile.at("statistics_options").contains("params") ) { + for ( auto& paramArray : jFile.at("statistics_options").at("params").items() ) { + graph->addStatisticOutputParameter(paramArray.key(), paramArray.value()); + } + } + } + // discover the statistics groups + if ( jFile.contains("statistics_group") ) { setStatGroupOptions(jFile); } +} + ConfigGraph* SSTJSONModelDefinition::createConfigGraph() { @@ -322,7 +497,8 @@ SSTJSONModelDefinition::createConfigGraph() // discover the links discoverLinks(jFile); - // TODO: discover statistics + // discover statistics + discoverStatistics(jFile); return graph; } diff --git a/src/sst/core/model/json/jsonmodel.h b/src/sst/core/model/json/jsonmodel.h index b0d68081d..fb3ae0693 100644 --- a/src/sst/core/model/json/jsonmodel.h +++ b/src/sst/core/model/json/jsonmodel.h @@ -65,11 +65,13 @@ class SSTJSONModelDefinition : public SSTModelDescription double start_time; private: + void setStatGroupOptions(const json& jFile); void recursiveSubcomponent(ConfigComponent* Parent, const nlohmann::basic_json<>& compArray); void discoverProgramOptions(const json& jFile); void discoverComponents(const json& jFile); void discoverLinks(const json& jFile); void discoverGlobalParams(const json& jFile); + void discoverStatistics(const json& jFile); ComponentId_t findComponentIdByName(const std::string& Name); }; diff --git a/src/sst/core/model/python/pymodel.cc b/src/sst/core/model/python/pymodel.cc index 28a1b234c..15b0cd461 100644 --- a/src/sst/core/model/python/pymodel.cc +++ b/src/sst/core/model/python/pymodel.cc @@ -648,7 +648,7 @@ enableStatisticsForComponentName(PyObject* UNUSED(self), PyObject* args) apply_to_children = 0; // Try list version argOK = PyArg_ParseTuple( - args, "sO!|O!i", &compName, &PyList_Type, &statList, &PyDict_Type, &statParamDict, &apply_to_children); + args, "sO!|O!i", &compName, &PyList_Type, &statList, &PyDict_Type, &statParamDict, &apply_to_children); if ( argOK ) Py_INCREF(statList); } @@ -765,7 +765,7 @@ enableStatisticsForComponentType(PyObject* UNUSED(self), PyObject* args) apply_to_children = 0; // Try list version argOK = PyArg_ParseTuple( - args, "sO!|O!i", &compType, &PyList_Type, &statList, &PyDict_Type, &statParamDict, &apply_to_children); + args, "sO!|O!i", &compType, &PyList_Type, &statList, &PyDict_Type, &statParamDict, &apply_to_children); if ( argOK ) Py_INCREF(statList); } diff --git a/src/sst/core/statapi/statoutputhdf5.cc b/src/sst/core/statapi/statoutputhdf5.cc index 81735c38d..f51f0b89b 100644 --- a/src/sst/core/statapi/statoutputhdf5.cc +++ b/src/sst/core/statapi/statoutputhdf5.cc @@ -433,7 +433,7 @@ StatisticOutputHDF5::GroupInfo::finalizeGroupRegistration() H5::DataSet* idSet = new H5::DataSet(getFile()->createDataSet(groupName + "/ids", H5::PredType::NATIVE_UINT64, infoSpace, cparms)); H5::DataSet* nameSet = new H5::DataSet(getFile()->createDataSet( - groupName + "/names", H5::StrType(H5::PredType::C_S1, H5T_VARIABLE), infoSpace, cparms)); + groupName + "/names", H5::StrType(H5::PredType::C_S1, H5T_VARIABLE), infoSpace, cparms)); H5::DataSet* coordXSet = new H5::DataSet( getFile()->createDataSet(groupName + "/coord_x", H5::PredType::NATIVE_DOUBLE, infoSpace, cparms)); H5::DataSet* coordYSet = new H5::DataSet( diff --git a/src/sst/core/testElements/coreTest_ClockerComponent.cc b/src/sst/core/testElements/coreTest_ClockerComponent.cc index e0d6058b7..90b1008fd 100644 --- a/src/sst/core/testElements/coreTest_ClockerComponent.cc +++ b/src/sst/core/testElements/coreTest_ClockerComponent.cc @@ -57,7 +57,8 @@ coreTestClockerComponent::coreTestClockerComponent() : Component(-1) // for serialization only } -bool coreTestClockerComponent::tick(Cycle_t) +bool +coreTestClockerComponent::tick(Cycle_t) { clock_count--; diff --git a/src/sst/core/testElements/coreTest_Component.cc b/src/sst/core/testElements/coreTest_Component.cc index cc040108d..e4b83c95b 100644 --- a/src/sst/core/testElements/coreTest_Component.cc +++ b/src/sst/core/testElements/coreTest_Component.cc @@ -99,7 +99,8 @@ coreTestComponent::handleEvent(Event* ev) // each clock tick we do 'workPerCycle' iterations of a coreTest loop. // We have a 1/commFreq chance of sending an event of size commSize to // one of our neighbors. -bool coreTestComponent::clockTic(Cycle_t) +bool +coreTestComponent::clockTic(Cycle_t) { // do work // loop becomes: diff --git a/src/sst/core/testElements/coreTest_LookupTableComponent.cc b/src/sst/core/testElements/coreTest_LookupTableComponent.cc index c1fe82ad9..d80fe9b25 100644 --- a/src/sst/core/testElements/coreTest_LookupTableComponent.cc +++ b/src/sst/core/testElements/coreTest_LookupTableComponent.cc @@ -80,7 +80,8 @@ void coreTestLookupTableComponent::finish() {} -bool coreTestLookupTableComponent::tick(SST::Cycle_t) +bool +coreTestLookupTableComponent::tick(SST::Cycle_t) { bool done = false; static const size_t nPerRow = 8; diff --git a/src/sst/core/testElements/coreTest_MessageGeneratorComponent.cc b/src/sst/core/testElements/coreTest_MessageGeneratorComponent.cc index 2bd773788..69694022d 100644 --- a/src/sst/core/testElements/coreTest_MessageGeneratorComponent.cc +++ b/src/sst/core/testElements/coreTest_MessageGeneratorComponent.cc @@ -68,7 +68,8 @@ coreTestMessageGeneratorComponent::handleEvent(Event* event) // each clock tick we do 'workPerCycle' iterations of a coreTest loop. // We have a 1/commFreq chance of sending an event of size commSize to // one of our neighbors. -bool coreTestMessageGeneratorComponent::tick(Cycle_t) +bool +coreTestMessageGeneratorComponent::tick(Cycle_t) { coreTestMessage* msg = new coreTestMessage(); remote_component->send(msg); diff --git a/src/sst/core/testElements/coreTest_Module.cc b/src/sst/core/testElements/coreTest_Module.cc index 7254c0301..8366db866 100644 --- a/src/sst/core/testElements/coreTest_Module.cc +++ b/src/sst/core/testElements/coreTest_Module.cc @@ -130,7 +130,8 @@ coreTestModuleLoader::coreTestModuleLoader() : Component() { /* For serialization ONLY*/ } -bool coreTestModuleLoader::tick(SST::Cycle_t) +bool +coreTestModuleLoader::tick(SST::Cycle_t) { uint32_t next = rng_module->getNext(); rng_count++; diff --git a/src/sst/core/testElements/coreTest_PerfComponent.cc b/src/sst/core/testElements/coreTest_PerfComponent.cc index 61557587c..2a6936d4c 100644 --- a/src/sst/core/testElements/coreTest_PerfComponent.cc +++ b/src/sst/core/testElements/coreTest_PerfComponent.cc @@ -101,7 +101,8 @@ coreTestPerfComponent::handleEvent(Event* ev) // each clock tick we do 'workPerCycle' iterations of a coreTest loop. // We have a 1/commFreq chance of sending an event of size commSize to // one of our neighbors. -bool coreTestPerfComponent::clockTic(Cycle_t) +bool +coreTestPerfComponent::clockTic(Cycle_t) { // do work // loop becomes: diff --git a/src/sst/core/testElements/coreTest_RNGComponent.cc b/src/sst/core/testElements/coreTest_RNGComponent.cc index 25c2a344e..9aa33a92e 100644 --- a/src/sst/core/testElements/coreTest_RNGComponent.cc +++ b/src/sst/core/testElements/coreTest_RNGComponent.cc @@ -82,7 +82,8 @@ coreTestRNGComponent::coreTestRNGComponent() : Component(-1) // for serialization only } -bool coreTestRNGComponent::tick(Cycle_t) +bool +coreTestRNGComponent::tick(Cycle_t) { double nU = rng->nextUniform(); uint32_t U32 = rng->generateNextUInt32(); diff --git a/src/sst/core/testElements/coreTest_SharedObjectComponent.cc b/src/sst/core/testElements/coreTest_SharedObjectComponent.cc index f23d171b3..9df89e31d 100644 --- a/src/sst/core/testElements/coreTest_SharedObjectComponent.cc +++ b/src/sst/core/testElements/coreTest_SharedObjectComponent.cc @@ -328,7 +328,8 @@ coreTestSharedObjectsComponent::finish() } } -bool coreTestSharedObjectsComponent::tick(SST::Cycle_t) +bool +coreTestSharedObjectsComponent::tick(SST::Cycle_t) { if ( check ) { diff --git a/src/sst/core/testElements/message_mesh/enclosingComponent.cc b/src/sst/core/testElements/message_mesh/enclosingComponent.cc index 5be20d75f..66b6ace8d 100644 --- a/src/sst/core/testElements/message_mesh/enclosingComponent.cc +++ b/src/sst/core/testElements/message_mesh/enclosingComponent.cc @@ -118,7 +118,8 @@ RouteMessage::RouteMessage( ports(parent_ports), my_id(nid) { - rng = new SST::RNG::MersenneRNG(my_id + 100); + rng = new SST::RNG::MersenneRNG(my_id + 100); + mcnt = registerStatistic("msg_count"); } void @@ -127,4 +128,5 @@ RouteMessage::send(MessageEvent* ev, int UNUSED(incoming_port)) // Route to random port int nextport = rng->generateNextUInt32() % ports.size(); ports[nextport]->send(ev); + mcnt->addData(1); } diff --git a/src/sst/core/testElements/message_mesh/enclosingComponent.h b/src/sst/core/testElements/message_mesh/enclosingComponent.h index ba0141d20..2ec46109a 100644 --- a/src/sst/core/testElements/message_mesh/enclosingComponent.h +++ b/src/sst/core/testElements/message_mesh/enclosingComponent.h @@ -212,6 +212,7 @@ class RouteMessage : public RouteInterface ) SST_ELI_DOCUMENT_STATISTICS( + {"msg_count", "Message counter", "count", 1}, ) SST_ELI_DOCUMENT_PORTS( @@ -229,6 +230,8 @@ class RouteMessage : public RouteInterface const std::vector ports; int my_id; SST::RNG::Random* rng; + + Statistic* mcnt; }; } // namespace SST::CoreTest::MessageMesh diff --git a/tests/test_MessageMesh.py b/tests/test_MessageMesh.py index e17de821c..798da4720 100644 --- a/tests/test_MessageMesh.py +++ b/tests/test_MessageMesh.py @@ -18,6 +18,9 @@ x_size = int(sys.argv[1]) y_size = int(sys.argv[2]) +sst.setStatisticOutput("sst.statOutputCSV") +sst.enableAllStatisticsForAllComponents() + # Calculate number of routers and endpoints num_routers = x_size * y_size