Support for multi-report (#74)

* Implementation of synapse report in coreneuron
    - Add mapping between synapses and variable names to register
    - Add a GID to compartment mapping
    - Add support for gids selection to report
    - Support for multiple reportEvents
    - Correct extra mapping to have reports binary identical compared to neurodamus reports

* Few bug fixes and improvements while testing multiple reports:
    - DiscreteEvent now has require_checkpoint method to indicate
      whether event needs to be checkpointed. This helps when we
      have user-defined events like reportevent and we can skip.
    - Checkpoint state "store" wasn't working in multi-threaded
      multi-file scenario. This is because the static variable
      becomes thread specific and could't be modified by other
      threads. Storing now state in FileHandler.
    - Reportinglib was giving warnings for extra steps. This was
      due to floating point inconsistencies and "int" step. Change
      step to double similar to neurodamus.
    - Default cell permute to 0 (avoid troubles with reports)
    - Avoid std::endl which does flush internally and causing terrible i/o performance.
    - Tau instrumentation for selective profiling

Author: fouriaux

CMakeLists.txt

@@ -63,7 +63,6 @@ option(ENABLE_OMP_RUNTIME_SCHEDULE "Use runtime schedule for OpenMP" OFF)
 
 set(FRONTEND_C_COMPILER gcc CACHE FILEPATH "C compiler for building mod2c [frontend executable]")
 set(FRONTEND_CXX_COMPILER g++ CACHE FILEPATH "C++ compiler for building mod2c [frontend executable]")
-
 ## mech / mod options
 set(ADDITIONAL_MECHPATH "" CACHE PATH "Search path for optional additional mechanism MOD files")
 set(ADDITIONAL_MECHS "" CACHE FILEPATH "File containing list of additional mechanism MOD files")

coreneuron/CMakeLists.txt

@@ -29,7 +29,7 @@ include_directories(utils/randoms)
 include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
 include_directories(${MPI_INCLUDE_PATH})
 
-FILE(GLOB_RECURSE coreneuron_all_headers "*.h")
+FILE(GLOB_RECURSE coreneuron_all_headers "*.h*")
 FILE(GLOB_RECURSE coreneuron_all_templates "*.ipp")
 FILE(GLOB_RECURSE coreneuron_all_c_files "*.c*")
 file(GLOB_RECURSE coreneuron_cuda_files "*.cu")

coreneuron/nrniv/main1.cpp

@@ -32,6 +32,8 @@ THE POSSIBILITY OF SUCH DAMAGE.
  * @brief File containing main driver routine for CoreNeuron
  */
 
+#include <vector>
+#include <string.h>
 #include "coreneuron/utils/randoms/nrnran123.h"
 #include "coreneuron/nrnconf.h"
 #include "coreneuron/nrnoc/multicore.h"
@@ -52,6 +54,7 @@ THE POSSIBILITY OF SUCH DAMAGE.
 #include "coreneuron/nrniv/multisend.h"
 #include "coreneuron/utils/file_utils.h"
 #include <string.h>
+#include <climits>
 
 #if 0
 #include <fenv.h>
@@ -66,6 +69,7 @@ int main1(int argc, char* argv[], char** env);
 void call_prcellstate_for_prcellgid(int prcellgid, int compute_gpu, int is_init);
 void nrn_init_and_load_data(int argc,
                             char* argv[],
+                            bool is_mapping_needed = false,
                             bool nrnmpi_under_nrncontrol = true,
                             bool run_setup_cleanup = true) {
 #if defined(NRN_FEEXCEPT)
@@ -90,9 +94,6 @@ void nrn_init_and_load_data(int argc,
     // create mutex for nrn123, protect instance_count_
     nrnran123_mutconstruct();
 
-    // read command line parameters and parameter config files
-    nrnopt_parse(argc, (const char**)argv);
-
     // set global variables
     // precedence is: set by user, globals.dat, 34.0
     celsius = nrnopt_get_dbl("--celsius");
@@ -158,8 +159,8 @@ void nrn_init_and_load_data(int argc,
     use_interleave_permute = nrnopt_get_int("--cell-permute");
     cellorder_nwarp = nrnopt_get_int("--nwarp");
     use_solve_interleave = nrnopt_get_int("--cell-permute");
-#if LAYOUT==1
-    //permuting not allowed for AoS
+#if LAYOUT == 1
+    // permuting not allowed for AoS
     use_interleave_permute = 0;
     use_solve_interleave = 0;
 #endif
@@ -174,7 +175,7 @@ void nrn_init_and_load_data(int argc,
     use_phase2_ = (nrnopt_get_int("--ms-phases") == 2) ? 1 : 0;
 
     // reading *.dat files and setting up the data structures, setting mindelay
-    nrn_setup(filesdat.c_str(), nrn_need_byteswap, run_setup_cleanup);
+    nrn_setup(filesdat.c_str(), is_mapping_needed, nrn_need_byteswap, run_setup_cleanup);
 
     // Allgather spike compression and  bin queuing.
     nrn_use_bin_queue_ = nrnopt_get_flag("--binqueue");
@@ -242,8 +243,28 @@ void call_prcellstate_for_prcellgid(int prcellgid, int compute_gpu, int is_init)
 int main1(int argc, char** argv, char** env) {
     (void)env; /* unused */
 
+#if NRNMPI
+    nrnmpi_init(1, &argc, &argv);
+#endif
+
+    // read command line parameters and parameter config files
+    nrnopt_parse(argc, (const char**)argv);
+    std::vector<ReportConfiguration> configs;
+    bool reports_needs_finalize = false;
+
+    if (nrnopt_get_str("--report-conf").size()) {
+        if (nrnopt_get_int("--multiple") > 1) {
+            if (nrnmpi_myid == 0)
+                printf("\n WARNING! : Can't enable reports with model duplications feature! \n");
+        } else {
+            configs = create_report_configurations(nrnopt_get_str("--report-conf").c_str(),
+                                                   nrnopt_get_str("--outpath").c_str());
+            reports_needs_finalize = configs.size();
+        }
+    }
+
     // initializationa and loading functions moved to separate
-    nrn_init_and_load_data(argc, argv);
+    nrn_init_and_load_data(argc, argv, configs.size() > 0);
     std::string checkpoint_path = nrnopt_get_str("--checkpoint");
     if (strlen(checkpoint_path.c_str())) {
         nrn_checkpoint_arg_exists = true;
@@ -264,35 +285,34 @@ int main1(int argc, char** argv, char** env) {
     {
         double v = nrnopt_get_dbl("--voltage");
 
-        // finitialize is not called if running in checkpoint-restore mode
+        // TODO : if some ranks are empty then restore will go in deadlock
+        // phase (as some ranks won't have restored anything and hence return
+        // false in checkpoint_initialize
         if (!checkpoint_initialize()) {
             nrn_finitialize(v != 1000., v);
         }
 
         report_mem_usage("After nrn_finitialize");
+        double dt = nrnopt_get_dbl("--dt");
+        double delay = nrnopt_get_dbl("--mindelay");
+        double tstop = nrnopt_get_dbl("--tstop");
+
+        if (tstop < t && nrnmpi_myid == 0) {
+            printf("Error: Stop time (%lf) < Start time (%lf), restoring from checkpoint? \n",
+                   tstop, t);
+            abort();
+        }
 
-#ifdef ENABLE_REPORTING
-        ReportGenerator* r = NULL;
-#endif
-
-        // if reports are enabled using ReportingLib
-        if (nrnopt_get_flag("--report")) {
-#ifdef ENABLE_REPORTING
-            if (nrnopt_get_int("--multiple") > 1) {
-                if (nrnmpi_myid == 0)
-                    printf(
-                        "\n WARNING! : Can't enable reports with model duplications feature! \n");
-            } else {
-                r = new ReportGenerator(nrnopt_get_int("--report"), t, nrnopt_get_dbl("--tstop"),
-                                        nrnopt_get_dbl("--dt"), nrnopt_get_dbl("--mindelay"),
-                                        nrnopt_get_dbl("--dt_report"), nrnopt_get_str("--outpath"));
-                r->register_report();
+        // register all reports into reportinglib
+        double min_report_dt = INT_MAX;
+        for (int i = 0; i < configs.size(); i++) {
+            register_report(dt, tstop, delay, configs[i]);
+            if (configs[i].report_dt < min_report_dt) {
+                min_report_dt = configs[i].report_dt;
             }
-#else
-            if (nrnmpi_myid == 0)
-                printf("\n WARNING! : Can't enable reports, recompile with ReportingLib! \n");
-#endif
         }
+        setup_report_engine(min_report_dt, delay);
+        configs.clear();
 
         // call prcellstate for prcellgid
         call_prcellstate_for_prcellgid(nrnopt_get_int("--prcellgid"), compute_gpu, 0);
@@ -311,24 +331,24 @@ int main1(int argc, char** argv, char** env) {
         // Report global cell statistics
         report_cell_stats();
 
-#ifdef ENABLE_SELECTIVE_PROFILING
-        stop_profile();
-#endif
-
         // prcellstate after end of solver
         call_prcellstate_for_prcellgid(nrnopt_get_int("--prcellgid"), compute_gpu, 0);
-
-#ifdef ENABLE_REPORTING
-        if (nrnopt_get_int("--report") && r)
-            delete r;
-#endif
     }
 
-    write_checkpoint(nrn_threads, nrn_nthread, checkpoint_path.c_str(), nrn_need_byteswap);
-
     // write spike information to outpath
     output_spikes(output_dir.c_str());
 
+    write_checkpoint(nrn_threads, nrn_nthread, checkpoint_path.c_str(), nrn_need_byteswap);
+
+#ifdef ENABLE_SELECTIVE_PROFILING
+    stop_profile();
+#endif
+
+    // must be done after checkpoint (to avoid deleting events)
+    if (reports_needs_finalize) {
+        finalize_report();
+    }
+
     // Cleaning the memory
     nrn_cleanup();
 

coreneuron/nrniv/netcon.h

@@ -60,7 +60,9 @@ class DiscreteEvent {
     virtual int type() {
         return DiscreteEventType;
     }
-
+    virtual bool require_checkpoint() {
+        return true;
+    }
     virtual void pr(const char*, double t, NetCvode*);
 };
 

coreneuron/nrniv/node_permute.h

@@ -1,6 +1,9 @@
 #ifndef node_permute_h
 #define node_permute_h
 
+struct NrnThread;
+struct Memb_list;
+
 // determine ml->_permute and permute the ml->nodeindices accordingly
 void permute_nodeindices(Memb_list* ml, int* permute);
 

coreneuron/nrniv/nrn_checkpoint.cpp

@@ -77,9 +77,9 @@ class FileHandlerWrap {
     template <typename T>
     void write_array(T* p, size_t nb_elements) {
         // G first before chkpnt is incremented
-        G << "chkpnt " << F.checkpoint() << std::endl;
+        G << "chkpnt " << F.checkpoint() << "\n";
         for (size_t i = 0; i < nb_elements; ++i) {
-            G << std::setprecision(8) << p[i] << std::endl;
+            G << std::setprecision(8) << p[i] << "\n";
         }
         F.write_array(p, nb_elements);  // chkpnt incremented
     }
@@ -176,6 +176,9 @@ void write_checkpoint(NrnThread* nt, int nb_threads, const char* dir, bool swap_
     if (nrnmpi_myid == 0) {
         write_time(output_dir);
     }
+#if NRNMPI
+    nrnmpi_barrier();
+#endif
 }
 
 static void write_phase2(NrnThread& nt, FileHandlerWrap& fh) {
@@ -595,7 +598,7 @@ static void write_phase2(NrnThread& nt, FileHandlerWrap& fh) {
 static void write_time(const char* output_dir) {
     std::ostringstream filename;
     FileHandler f;
-    filename << output_dir << "/" << "time.dat";
+    filename << output_dir << "/time.dat";
     f.open(filename.str().c_str(), swap_bytes, std::ios::out);
     f.write_array(&t, 1);
     f.close();
@@ -607,7 +610,7 @@ double restore_time(const char* restore_dir) {
     if (strlen(restore_dir)) {
         std::ostringstream filename;
         FileHandler f;
-        filename << restore_dir << "/" << "time.dat";
+        filename << restore_dir << "/time.dat";
         f.open(filename.str().c_str(), swap_bytes, std::ios::in);
         f.read_array(&rtime, 1);
         f.close();
@@ -617,9 +620,14 @@ double restore_time(const char* restore_dir) {
 
 static void write_tqueue(TQItem* q, NrnThread& nt, FileHandlerWrap& fh) {
     DiscreteEvent* d = (DiscreteEvent*)q->data_;
+
     // printf("  p %.20g %d\n", q->t_, d->type());
     // d->pr("", q->t_, net_cvode_instance);
 
+    if (!d->require_checkpoint()) {
+        return;
+    }
+
     fh << d->type() << "\n";
     fh.write_array(&q->t_, 1);
 

coreneuron/nrniv/nrn_filehandler.cpp

@@ -34,6 +34,7 @@ extern "C" void check_bbcore_write_version(const char*);
 FileHandler::FileHandler(const char* filename, bool reorder) {
     this->open(filename, reorder);
     checkpoint(0);
+    stored_chkpnt = 0;
 }
 
 void FileHandler::open(const char* filename, bool reorder, std::ios::openmode mode) {
@@ -50,7 +51,7 @@ void FileHandler::open(const char* filename, bool reorder, std::ios::openmode mo
         check_bbcore_write_version(version);
     }
     if (current_mode & std::ios::out) {
-        F << bbcore_write_version << std::endl;
+        F << bbcore_write_version << "\n";
     }
 }
 

coreneuron/nrniv/nrn_filehandler.h

@@ -58,6 +58,7 @@ class FileHandler {
     bool reorder_bytes;                    //!< True if we need to reorder for native endiannes.
     std::ios_base::openmode current_mode;  //!< File open mode (not stored in fstream)
     int chkpnt;                            //!< Current checkpoint number state.
+    int stored_chkpnt;                     //!< last "remembered" checkpoint number state.
     /** Read a checkpoint line, bump our chkpnt counter, and assert equality.
      *
      * Checkpoint information is represented by a sequence "checkpt %d\n"
@@ -71,7 +72,7 @@ class FileHandler {
     FileHandler& operator=(const FileHandler&);
 
   public:
-    FileHandler() : reorder_bytes(false), chkpnt(0) {
+    FileHandler() : reorder_bytes(false), chkpnt(0), stored_chkpnt(0) {
     }
 
     explicit FileHandler(const char* filename, bool reorder = false);
@@ -97,6 +98,16 @@ class FileHandler {
         chkpnt = c;
     }
 
+    /** Record current chkpnt state. */
+    void record_checkpoint() {
+        stored_chkpnt = chkpnt;
+    }
+
+    /** Restored last recorded chkpnt state. */
+    void restore_checkpoint() {
+        chkpnt = stored_chkpnt;
+    }
+
     /** Parse a single integer entry.
      *
      * Single integer entries are represented by their standard
@@ -265,7 +276,7 @@ class FileHandler {
     /* write_checkpoint is callable only for our internal uses, making it accesible to user, makes
      * file format unpredictable */
     void write_checkpoint() {
-        F << "chkpnt " << chkpnt++ << std::endl;
+        F << "chkpnt " << chkpnt++ << "\n";
     }
 };
 

coreneuron/nrniv/nrn_setup.cpp

@@ -47,7 +47,6 @@ THE POSSIBILITY OF SUCH DAMAGE.
 #include "coreneuron/nrniv/nrn_checkpoint.h"
 #include "coreneuron/nrniv/node_permute.h"
 #include "coreneuron/nrniv/cellorder.h"
-#include "coreneuron/utils/reports/nrnreport.h"
 #include "coreneuron/utils/reports/nrnsection_mapping.h"
 
 // file format defined in cooperation with nrncore/src/nrniv/nrnbbcore_write.cpp
@@ -559,7 +558,10 @@ void nrn_setup_cleanup() {
     neg_gid2out.clear();
 }
 
-void nrn_setup(const char* filesdat, int byte_swap, bool run_setup_cleanup) {
+void nrn_setup(const char* filesdat,
+               bool is_mapping_needed,
+               int byte_swap,
+               bool run_setup_cleanup) {
     /// Number of local cell groups
     int ngroup = 0;
 
@@ -653,7 +655,7 @@ void nrn_setup(const char* filesdat, int byte_swap, bool run_setup_cleanup) {
     /* nrn_multithread_job supports serial, pthread, and openmp. */
     coreneuron::phase_wrapper<(coreneuron::phase)2>();
 
-    if (nrnopt_get_flag("--report"))
+    if (is_mapping_needed)
         coreneuron::phase_wrapper<(coreneuron::phase)3>();
 
     double mindelay = set_mindelay(nrnopt_get_dbl("--mindelay"));
@@ -1286,7 +1288,7 @@ void read_phase2(FileHandler& F, int imult, NrnThread& nt) {
                 int etype = s;
                 int elayout = nrn_mech_data_layout_[etype];
                 /* if ion is SoA, must recalculate pdata values */
-		/* if ion is AoS, have to deal with offset */
+                /* if ion is AoS, have to deal with offset */
                 Memb_list* eml = nt._ml_list[etype];
                 int edata0 = eml->data - nt._data;
                 int ecnt = eml->nodecount;
@@ -1760,6 +1762,9 @@ for (int i=0; i < nt.end; ++i) {
         nt._vecplay[i] = new VecPlayContinuous(ml->data + ix, yvec, tvec, NULL, nt.id);
     }
 
+    // store current checkpoint state to continue reading mapping
+    F.record_checkpoint();
+
     // If not at end of file, then this must be a checkpoint and restore tqueue.
     if (!F.eof()) {
         checkpoint_restore_tqueue(nt, F);
@@ -1828,6 +1833,9 @@ for (int i=0; i < nt.end; ++i) {
 void read_phase3(FileHandler& F, int imult, NrnThread& nt) {
     (void)imult;
 
+    /** restore checkpoint state (before restoring queue items */
+    F.restore_checkpoint();
+
     /** mapping information for all neurons in single NrnThread */
     NrnThreadMappingInfo* ntmapping = new NrnThreadMappingInfo();
 

coreneuron/nrniv/nrn_setup.h

@@ -51,6 +51,7 @@ static void setup_ThreadData(NrnThread& nt);
 // Functions to load and clean data;
 extern void nrn_init_and_load_data(int argc,
                                    char** argv,
+                                   bool is_mapping_needed = false,
                                    bool nrnmpi_under_nrncontrol = true,
                                    bool run_setup_cleanup = true);
 extern void nrn_setup_cleanup();