Report multipopulation (#778)

* Add multipopulation support for compartment reports.
* Remove old population name and population offset.

Co-authored-by: Pramod Kumbhar <[email protected]>

Author: jorblancoa

coreneuron/apps/main1.cpp

@@ -428,12 +428,13 @@ static void trajectory_return() {
     }
 }
 
-std::unique_ptr<ReportHandler> create_report_handler(ReportConfiguration& config) {
+std::unique_ptr<ReportHandler> create_report_handler(ReportConfiguration& config,
+                                                     const SpikesInfo& spikes_info) {
     std::unique_ptr<ReportHandler> report_handler;
     if (config.format == "Bin") {
         report_handler = std::make_unique<BinaryReportHandler>(config);
     } else if (config.format == "SONATA") {
-        report_handler = std::make_unique<SonataReportHandler>(config);
+        report_handler = std::make_unique<SonataReportHandler>(config, spikes_info);
     } else {
         if (nrnmpi_myid == 0) {
             printf(" WARNING : Report name '%s' has unknown format: '%s'.\n",
@@ -524,7 +525,7 @@ extern "C" int run_solve_core(int argc, char** argv) {
 
     std::vector<ReportConfiguration> configs;
     std::vector<std::unique_ptr<ReportHandler>> report_handlers;
-    std::vector<std::pair<std::string, int>> spikes_population_name_offset;
+    SpikesInfo spikes_info;
     bool reports_needs_finalize = false;
 
     if (!corenrn_param.is_quiet()) {
@@ -539,7 +540,7 @@ extern "C" int run_solve_core(int argc, char** argv) {
     if (!corenrn_param.reportfilepath.empty()) {
         configs = create_report_configurations(corenrn_param.reportfilepath,
                                                corenrn_param.outpath,
-                                               spikes_population_name_offset);
+                                               spikes_info);
         reports_needs_finalize = !configs.empty();
     }
 
@@ -592,7 +593,8 @@ extern "C" int run_solve_core(int argc, char** argv) {
         // register all reports into reportinglib
         double min_report_dt = INT_MAX;
         for (size_t i = 0; i < configs.size(); i++) {
-            std::unique_ptr<ReportHandler> report_handler = create_report_handler(configs[i]);
+            std::unique_ptr<ReportHandler> report_handler = create_report_handler(configs[i],
+                                                                                  spikes_info);
             if (report_handler) {
                 report_handler->create_report(dt, tstop, delay);
                 report_handlers.push_back(std::move(report_handler));
@@ -645,7 +647,7 @@ extern "C" int run_solve_core(int argc, char** argv) {
     // write spike information to outpath
     {
         Instrumentor::phase p("output-spike");
-        output_spikes(output_dir.c_str(), spikes_population_name_offset);
+        output_spikes(output_dir.c_str(), spikes_info);
     }
 
     // copy weights back to NEURON NetCon

coreneuron/io/output_spikes.cpp

@@ -154,10 +154,9 @@ static void sort_spikes(std::vector<double>& spikevec_time, std::vector<int>& sp
 /** Split spikevec_time and spikevec_gid by populations
  *  Add spike data with population name and gid offset tolibsonatareport API
  */
-static void output_spike_populations(
-    const std::vector<std::pair<std::string, int>>& population_name_offset) {
+void output_spike_populations(const SpikesInfo& spikes_info) {
     // Write spikes with default population name and offset
-    if (population_name_offset.empty()) {
+    if (spikes_info.population_info.empty()) {
         sonata_add_spikes_population("All",
                                      0,
                                      spikevec_time.data(),
@@ -166,15 +165,15 @@ static void output_spike_populations(
                                      spikevec_gid.size());
         return;
     }
-    int n_populations = population_name_offset.size();
+    int n_populations = spikes_info.population_info.size();
     for (int idx = 0; idx < n_populations; idx++) {
-        auto cur_pop = population_name_offset[idx];
-        std::string population_name = cur_pop.first;
-        int population_offset = cur_pop.second;
+        const auto& curr_pop = spikes_info.population_info[idx];
+        std::string population_name = curr_pop.first;
+        int population_offset = curr_pop.second;
         int gid_lower = population_offset;
         int gid_upper = std::numeric_limits<int>::max();
         if (idx != n_populations - 1) {
-            gid_upper = population_name_offset[idx + 1].second - 1;
+            gid_upper = spikes_info.population_info[idx + 1].second - 1;
         }
         std::vector<double> pop_spikevec_time;
         std::vector<int> pop_spikevec_gid;
@@ -198,10 +197,7 @@ static void output_spike_populations(
  *  \todo : MPI related code should be factored into nrnmpi.c
  *          Check spike record length which is set to 64 chars
  */
-static void output_spikes_parallel(
-    const char* outpath,
-    const char* filename,
-    const std::vector<std::pair<std::string, int>>& population_name_offset) {
+static void output_spikes_parallel(const char* outpath, const SpikesInfo& spikes_info) {
     std::stringstream ss;
     ss << outpath << "/out.dat";
     std::string fname = ss.str();
@@ -211,8 +207,8 @@ static void output_spikes_parallel(
         remove(fname.c_str());
     }
 #ifdef ENABLE_SONATA_REPORTS
-    sonata_create_spikefile(outpath, filename);
-    output_spike_populations(population_name_offset);
+    sonata_create_spikefile(outpath, spikes_info.file_name.data());
+    output_spike_populations(spikes_info);
     sonata_write_spike_populations();
     sonata_close_spikefile();
 #endif  // ENABLE_SONATA_REPORTS
@@ -280,16 +276,15 @@ static void output_spikes_serial(const char* outpath) {
     fclose(f);
 }
 
-void output_spikes(const char* outpath,
-                   const std::vector<std::pair<std::string, int>>& population_name_offset) {
+void output_spikes(const char* outpath, const SpikesInfo& spikes_info) {
     // try to transfer spikes to NEURON. If successfull, don't write out.dat
     if (all_spikes_return(spikevec_time, spikevec_gid)) {
         clear_spike_vectors();
         return;
     }
 #if NRNMPI
     if (corenrn_param.mpi_enable && nrnmpi_initialized()) {
-        output_spikes_parallel(outpath, "out", population_name_offset);
+        output_spikes_parallel(outpath, spikes_info);
     } else
 #endif
     {

coreneuron/io/output_spikes.hpp

@@ -11,9 +11,9 @@
 #include <string>
 #include <vector>
 #include <utility>
+#include "coreneuron/io/reports/nrnreport.hpp"
 namespace coreneuron {
-void output_spikes(const char* outpath,
-                   const std::vector<std::pair<std::string, int>>& population_name_offset);
+void output_spikes(const char* outpath, const SpikesInfo& spikes_info);
 void mk_spikevec_buffer(int);
 
 extern std::vector<double> spikevec_time;

coreneuron/io/reports/nrnreport.hpp

@@ -39,6 +39,11 @@ struct SummationReportMapping {
     std::unordered_map<std::string, SummationReport> summation_reports_;
 };
 
+struct SpikesInfo {
+    std::string file_name = "out";
+    std::vector<std::pair<std::string, int>> population_info;
+};
+
 // name of the variable in mod file that is used to indicate which synapse
 // is enabled or disable for reporting
 #define SELECTED_VAR_MOD_NAME "selected_for_report"
@@ -87,7 +92,6 @@ struct ReportConfiguration {
     std::string unit;                     // unit of the report
     std::string format;                   // format of the report (Bin, hdf5, SONATA)
     std::string type_str;                 // type of report string
-    std::string population_name;          // population name of the report
     TargetType target_type;               // type of the target
     ReportType type;                      // type of the report
     SectionType section_type;             // type of section report
@@ -97,15 +101,13 @@ struct ReportConfiguration {
     double stop;                          // stop time of report
     int num_gids;                         // total number of gids
     int buffer_size;                      // hint on buffer size used for this report
-    uint64_t population_offset;           // offset of the node ids in the population
     std::set<int> target;                 // list of gids for this report
 };
 
 void setup_report_engine(double dt_report, double mindelay);
-std::vector<ReportConfiguration> create_report_configurations(
-    const std::string& filename,
-    const std::string& output_dir,
-    std::vector<std::pair<std::string, int>>& spikes_population_name);
+std::vector<ReportConfiguration> create_report_configurations(const std::string& filename,
+                                                              const std::string& output_dir,
+                                                              SpikesInfo& spikes_info);
 void finalize_report();
 void nrn_flush_reports(double t);
 void set_report_buffer_size(int n);

coreneuron/io/reports/report_configuration_parser.cpp

@@ -103,10 +103,9 @@ void register_target_type(ReportConfiguration& report, ReportType report_type) {
     }
 }
 
-std::vector<ReportConfiguration> create_report_configurations(
-    const std::string& conf_file,
-    const std::string& output_dir,
-    std::vector<std::pair<std::string, int>>& spikes_population_name_offset) {
+std::vector<ReportConfiguration> create_report_configurations(const std::string& conf_file,
+                                                              const std::string& output_dir,
+                                                              SpikesInfo& spikes_info) {
     std::vector<ReportConfiguration> reports;
     std::string report_on;
     int target;
@@ -120,8 +119,7 @@ std::vector<ReportConfiguration> create_report_configurations(
 
         report_conf >> report.name >> report.target_name >> report.type_str >> report_on >>
             report.unit >> report.format >> target >> report.report_dt >> report.start >>
-            report.stop >> report.num_gids >> report.buffer_size >> report.population_name >>
-            report.population_offset;
+            report.stop >> report.num_gids >> report.buffer_size;
 
         report.target_type = static_cast<TargetType>(target);
         std::transform(report.type_str.begin(),
@@ -179,9 +177,10 @@ std::vector<ReportConfiguration> create_report_configurations(
             report_conf >> spikes_population_name;
             spikes_population_offset = 0;
         }
-        spikes_population_name_offset.emplace_back(
+        spikes_info.population_info.emplace_back(
             std::make_pair(spikes_population_name, spikes_population_offset));
     }
+    report_conf >> spikes_info.file_name;
 
     return reports;
 }

coreneuron/io/reports/sonata_report_handler.cpp

@@ -38,6 +38,22 @@ void SonataReportHandler::register_custom_report(const NrnThread& nt,
     register_report(nt, config, vars_to_report);
 }
 
+std::pair<std::string, int> SonataReportHandler::get_population_info(int gid) {
+    if (m_spikes_info.population_info.empty()) {
+        return std::make_pair("All", 0);
+    }
+    std::pair<std::string, int> prev = m_spikes_info.population_info.front();
+    for (const auto& name_offset: m_spikes_info.population_info) {
+        std::string pop_name = name_offset.first;
+        int pop_offset = name_offset.second;
+        if (pop_offset > gid) {
+            break;
+        }
+        prev = name_offset;
+    }
+    return prev;
+}
+
 void SonataReportHandler::register_report(const NrnThread& nt,
                                           ReportConfiguration& config,
                                           const VarsToReport& vars_to_report) {
@@ -55,14 +71,14 @@ void SonataReportHandler::register_report(const NrnThread& nt,
         if (!vars.size())
             continue;
 
-        sonata_add_node(config.output_path.data(),
-                        config.population_name.data(),
-                        config.population_offset,
-                        gid);
+        const auto& pop_info = get_population_info(gid);
+        std::string population_name = pop_info.first;
+        int population_offset = pop_info.second;
+        sonata_add_node(config.output_path.data(), population_name.data(), population_offset, gid);
         sonata_set_report_max_buffer_size_hint(config.output_path.data(), config.buffer_size);
         for (const auto& variable: vars) {
             sonata_add_element(config.output_path.data(),
-                               config.population_name.data(),
+                               population_name.data(),
                                gid,
                                variable.id,
                                variable.var_value);

coreneuron/io/reports/sonata_report_handler.hpp

@@ -17,8 +17,9 @@ namespace coreneuron {
 
 class SonataReportHandler: public ReportHandler {
   public:
-    SonataReportHandler(ReportConfiguration& config)
-        : ReportHandler(config) {}
+    SonataReportHandler(ReportConfiguration& config, const SpikesInfo& spikes_info)
+        : ReportHandler(config)
+        , m_spikes_info(spikes_info) {}
 
     void create_report(double dt, double tstop, double delay) override;
 #ifdef ENABLE_SONATA_REPORTS
@@ -34,7 +35,11 @@ class SonataReportHandler: public ReportHandler {
     void register_report(const NrnThread& nt,
                          ReportConfiguration& config,
                          const VarsToReport& vars_to_report);
+    std::pair<std::string, int> get_population_info(int gid);
 #endif  // ENABLE_SONATA_REPORTS
+
+  private:
+    SpikesInfo m_spikes_info;
 };
 
 }  // Namespace coreneuron

tests/integration/reportinglib/1.check.in

@@ -27,7 +27,7 @@ if [ "$sonata_reports" = "ON" ]
 then
   if [ -f test_2.h5 ]
   then
-    h5dump_diff=$(@H5DUMP_EXECUTABLE@ -d /report/All/data -y -O test_2.h5 | sed '1d;$d;s/,//g;s/ //g' | diff $test_ref -)
+    h5dump_diff=$(@H5DUMP_EXECUTABLE@ -d /report/PopA/data -y -O test_2.h5 | sed '1d;$d;s/,//g;s/ //g' | diff $test_ref -)
 
     if [ $? -ne 0 ]
     then
@@ -38,6 +38,11 @@ then
      echo "[ERROR] Expected SONATA soma file 'test_2.h5' doesn't exist. Test failed!" >&2
      exit $FAILED
   fi
+  if [ ! -f spikes.h5 ]
+  then
+     echo "[ERROR] Expected SONATA spike file 'spikes.h5' doesn't exist. Test failed!" >&2
+     exit $FAILED
+  fi
 fi
 
 # If we reach this point, all tests were successful