Improve memory usage in report initialization (#882)

* Use a vector for the report target gids in order to save memory
* Add intersection function for the report gids
* Clean up target vector after initialization
* sync gids and compartment id types to sonata

Author: jorblancoa

coreneuron/apps/main1.cpp

@@ -428,13 +428,13 @@ static void trajectory_return() {
     }
 }
 
-std::unique_ptr<ReportHandler> create_report_handler(ReportConfiguration& config,
+std::unique_ptr<ReportHandler> create_report_handler(const ReportConfiguration& config,
                                                      const SpikesInfo& spikes_info) {
     std::unique_ptr<ReportHandler> report_handler;
     if (config.format == "Bin") {
-        report_handler = std::make_unique<BinaryReportHandler>(config);
+        report_handler = std::make_unique<BinaryReportHandler>();
     } else if (config.format == "SONATA") {
-        report_handler = std::make_unique<SonataReportHandler>(config, spikes_info);
+        report_handler = std::make_unique<SonataReportHandler>(spikes_info);
     } else {
         if (nrnmpi_myid == 0) {
             printf(" WARNING : Report name '%s' has unknown format: '%s'.\n",
@@ -595,7 +595,7 @@ extern "C" int run_solve_core(int argc, char** argv) {
             std::unique_ptr<ReportHandler> report_handler = create_report_handler(configs[i],
                                                                                   spikes_info);
             if (report_handler) {
-                report_handler->create_report(dt, tstop, delay);
+                report_handler->create_report(configs[i], dt, tstop, delay);
                 report_handlers.push_back(std::move(report_handler));
             }
             if (configs[i].report_dt < min_report_dt) {

coreneuron/io/reports/binary_report_handler.cpp

@@ -13,11 +13,14 @@
 
 namespace coreneuron {
 
-void BinaryReportHandler::create_report(double dt, double tstop, double delay) {
+void BinaryReportHandler::create_report(ReportConfiguration& config,
+                                        double dt,
+                                        double tstop,
+                                        double delay) {
 #ifdef ENABLE_BIN_REPORTS
     records_set_atomic_step(dt);
 #endif  // ENABLE_BIN_REPORTS
-    ReportHandler::create_report(dt, tstop, delay);
+    ReportHandler::create_report(config, dt, tstop, delay);
 }
 
 #ifdef ENABLE_BIN_REPORTS
@@ -44,22 +47,22 @@ static void create_custom_extra(const CellMapping& mapping, std::array<int, 5>&
 }
 
 void BinaryReportHandler::register_section_report(const NrnThread& nt,
-                                                  ReportConfiguration& config,
+                                                  const ReportConfiguration& config,
                                                   const VarsToReport& vars_to_report,
                                                   bool is_soma_target) {
     create_extra_func create_extra = is_soma_target ? create_soma_extra : create_compartment_extra;
     register_report(nt, config, vars_to_report, create_extra);
 }
 
 void BinaryReportHandler::register_custom_report(const NrnThread& nt,
-                                                 ReportConfiguration& config,
+                                                 const ReportConfiguration& config,
                                                  const VarsToReport& vars_to_report) {
     create_extra_func create_extra = create_custom_extra;
     register_report(nt, config, vars_to_report, create_extra);
 }
 
 void BinaryReportHandler::register_report(const NrnThread& nt,
-                                          ReportConfiguration& config,
+                                          const ReportConfiguration& config,
                                           const VarsToReport& vars_to_report,
                                           create_extra_func& create_extra) {
     int sizemapping = 1;

coreneuron/io/reports/binary_report_handler.hpp

@@ -20,23 +20,20 @@ namespace coreneuron {
 
 class BinaryReportHandler: public ReportHandler {
   public:
-    BinaryReportHandler(ReportConfiguration& config)
-        : ReportHandler(config) {}
-
-    void create_report(double dt, double tstop, double delay) override;
+    void create_report(ReportConfiguration& config, double dt, double tstop, double delay) override;
 #ifdef ENABLE_BIN_REPORTS
     void register_section_report(const NrnThread& nt,
-                                 ReportConfiguration& config,
+                                 const ReportConfiguration& config,
                                  const VarsToReport& vars_to_report,
                                  bool is_soma_target) override;
     void register_custom_report(const NrnThread& nt,
-                                ReportConfiguration& config,
+                                const ReportConfiguration& config,
                                 const VarsToReport& vars_to_report) override;
 
   private:
     using create_extra_func = std::function<void(const CellMapping&, std::array<int, 5>&)>;
     void register_report(const NrnThread& nt,
-                         ReportConfiguration& config,
+                         const ReportConfiguration& config,
                          const VarsToReport& vars_to_report,
                          create_extra_func& create_extra);
 #endif  // ENABLE_BIN_REPORTS

coreneuron/io/reports/nrnreport.hpp

@@ -101,7 +101,7 @@ struct ReportConfiguration {
     double stop;                          // stop time of report
     int num_gids;                         // total number of gids
     int buffer_size;                      // hint on buffer size used for this report
-    std::set<int> target;                 // list of gids for this report
+    std::vector<int> target;              // list of gids for this report
 };
 
 void setup_report_engine(double dt_report, double mindelay);

coreneuron/io/reports/report_configuration_parser.cpp

@@ -106,15 +106,14 @@ void register_target_type(ReportConfiguration& report, ReportType report_type) {
 std::vector<ReportConfiguration> create_report_configurations(const std::string& conf_file,
                                                               const std::string& output_dir,
                                                               SpikesInfo& spikes_info) {
-    std::vector<ReportConfiguration> reports;
     std::string report_on;
     int target;
     std::ifstream report_conf(conf_file);
 
     int num_reports = 0;
     report_conf >> num_reports;
-    for (int i = 0; i < num_reports; i++) {
-        ReportConfiguration report;
+    std::vector<ReportConfiguration> reports(num_reports);
+    for (auto& report: reports) {
         report.buffer_size = 4;  // default size to 4 Mb
 
         report_conf >> report.name >> report.target_name >> report.type_str >> report_on >>
@@ -147,15 +146,13 @@ std::vector<ReportConfiguration> create_report_configurations(const std::string&
             parse_filter_string(report_on, report);
         }
         if (report.num_gids) {
-            std::vector<int> new_gids(report.num_gids);
+            report.target.resize(report.num_gids);
             report_conf.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
-            report_conf.read(reinterpret_cast<char*>(new_gids.data()),
+            report_conf.read(reinterpret_cast<char*>(report.target.data()),
                              report.num_gids * sizeof(int));
-            report.target = std::set<int>(new_gids.begin(), new_gids.end());
             // extra new line: skip
             report_conf.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
         }
-        reports.push_back(report);
     }
     // read population information for spike report
     int num_populations;

coreneuron/io/reports/report_event.hpp

@@ -20,15 +20,15 @@ namespace coreneuron {
 
 #if defined(ENABLE_BIN_REPORTS) || defined(ENABLE_SONATA_REPORTS)
 struct VarWithMapping {
-    int id;
+    uint32_t id;
     double* var_value;
     VarWithMapping(int id_, double* v_)
         : id(id_)
         , var_value(v_) {}
 };
 
 // mapping the set of variables pointers to report to its gid
-using VarsToReport = std::unordered_map<int, std::vector<VarWithMapping>>;
+using VarsToReport = std::unordered_map<uint64_t, std::vector<VarWithMapping>>;
 
 class ReportEvent: public DiscreteEvent {
   public: