(Minimum) Changes to avoid floating-point exceptions (#491)

  * If coreneuron is compiled with MPI but run without --mpi
    flag then coreneuron crashes. This doesn't allow NEURON
    to run existing tests (e.g. using python API)
  * Instead of using MPI function, using wrappers from mpispike.cpp
  * Updated all reduce wrapper to accept size_t / unsigned long int
    than just size_t
  * Update cast and summary printf
  * Add test that can run without MPI launcher

Co-authored-by: Olli Lupton <[email protected]>

Author: pramodk

coreneuron/io/mech_report.cpp

@@ -19,7 +19,7 @@ namespace coreneuron {
 void write_mech_report() {
     /// mechanim count across all gids, local to rank
     const auto n_memb_func = corenrn.get_memb_funcs().size();
-    std::vector<unsigned long long> local_mech_count(n_memb_func, 0);
+    std::vector<long> local_mech_count(n_memb_func, 0);
 
     /// each gid record goes on separate row, only check non-empty threads
     for (size_t i = 0; i < nrn_nthread; i++) {
@@ -31,16 +31,14 @@ void write_mech_report() {
         }
     }
 
-    std::vector<unsigned long long> total_mech_count(n_memb_func);
+    std::vector<long> total_mech_count(n_memb_func);
 
 #if NRNMPI
     /// get global sum of all mechanism instances
-    MPI_Allreduce(&local_mech_count[0],
-                  &total_mech_count[0],
-                  local_mech_count.size(),
-                  MPI_UNSIGNED_LONG_LONG,
-                  MPI_SUM,
-                  nrnmpi_comm);
+    nrnmpi_long_allreduce_vec(&local_mech_count[0],
+                              &total_mech_count[0],
+                              local_mech_count.size(),
+                              1);
 
 #else
     total_mech_count = local_mech_count;
@@ -50,7 +48,7 @@ void write_mech_report() {
         printf("\n================ MECHANISMS COUNT BY TYPE ==================\n");
         printf("%4s %20s %10s\n", "Id", "Name", "Count");
         for (size_t i = 0; i < total_mech_count.size(); i++) {
-            printf("%4lu %20s %10lld\n", i, nrn_get_mechname(i), total_mech_count[i]);
+            printf("%4lu %20s %10ld\n", i, nrn_get_mechname(i), total_mech_count[i]);
         }
         printf("=============================================================\n");
     }

coreneuron/io/nrn_setup.cpp

@@ -996,18 +996,18 @@ size_t input_presyn_size(void) {
 }
 
 size_t model_size(bool detailed_report) {
-    size_t nbyte = 0;
+    long nbyte = 0;
     size_t sz_nrnThread = sizeof(NrnThread);
     size_t sz_presyn = sizeof(PreSyn);
     size_t sz_input_presyn = sizeof(InputPreSyn);
     size_t sz_netcon = sizeof(NetCon);
     size_t sz_pntproc = sizeof(Point_process);
     size_t nccnt = 0;
 
-    std::vector<size_t> size_data(13, 0);
-    std::vector<size_t> global_size_data_min(13, 0);
-    std::vector<size_t> global_size_data_max(13, 0);
-    std::vector<size_t> global_size_data_sum(13, 0);
+    std::vector<long> size_data(13, 0);
+    std::vector<long> global_size_data_min(13, 0);
+    std::vector<long> global_size_data_max(13, 0);
+    std::vector<long> global_size_data_sum(13, 0);
     std::vector<float> global_size_data_avg(13, 0.0);
 
     for (int i = 0; i < nrn_nthread; ++i) {
@@ -1081,24 +1081,10 @@ size_t model_size(bool detailed_report) {
     if (detailed_report) {
         size_data[12] = nbyte;
 #if NRNMPI
-        MPI_Allreduce(&size_data[0],
-                      &global_size_data_min[0],
-                      13,
-                      MPI_UNSIGNED_LONG_LONG,
-                      MPI_MIN,
-                      nrnmpi_comm);
-        MPI_Allreduce(&size_data[0],
-                      &global_size_data_max[0],
-                      13,
-                      MPI_UNSIGNED_LONG_LONG,
-                      MPI_MAX,
-                      nrnmpi_comm);
-        MPI_Allreduce(&size_data[0],
-                      &global_size_data_sum[0],
-                      13,
-                      MPI_UNSIGNED_LONG_LONG,
-                      MPI_SUM,
-                      nrnmpi_comm);
+        // last arg is op type where 1 is sum, 2 is max and any other value is min
+        nrnmpi_long_allreduce_vec(&size_data[0], &global_size_data_sum[0], 13, 1);
+        nrnmpi_long_allreduce_vec(&size_data[0], &global_size_data_max[0], 13, 2);
+        nrnmpi_long_allreduce_vec(&size_data[0], &global_size_data_min[0], 13, 3);
         for (int i = 0; i < 13; i++) {
             global_size_data_avg[i] = global_size_data_sum[i] / float(nrnmpi_numprocs);
         }
@@ -1201,10 +1187,9 @@ size_t model_size(bool detailed_report) {
     }
 
 #if NRNMPI
-    size_t global_nbyte = 0;
-    MPI_Allreduce(&nbyte, &global_nbyte, 1, MPI_UNSIGNED_LONG_LONG, MPI_SUM, nrnmpi_comm);
+    long global_nbyte = 0;
+    nrnmpi_long_allreduce_vec(&nbyte, &global_nbyte, 1, 1);
     nbyte = global_nbyte;
-
 #endif
 
     return nbyte;

coreneuron/utils/nrn_stats.cpp

@@ -33,20 +33,20 @@ void report_cell_stats(void) {
     long stat_array[NUM_STATS] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, gstat_array[NUM_STATS];
 
     for (int ith = 0; ith < nrn_nthread; ++ith) {
-        stat_array[0] += (long) nrn_threads[ith].ncell;           // number of cells
-        stat_array[10] += (long) nrn_threads[ith].end;            // number of compartments
-        stat_array[1] += (long) nrn_threads[ith].n_presyn;        // number of presyns
-        stat_array[2] += (long) nrn_threads[ith].n_input_presyn;  // number of input presyns
-        stat_array[3] += (long) nrn_threads[ith].n_netcon;        // number of netcons, synapses
-        stat_array[4] += (long) nrn_threads[ith].n_pntproc;       // number of point processes
+        stat_array[0] += nrn_threads[ith].ncell;           // number of cells
+        stat_array[10] += nrn_threads[ith].end;            // number of compartments
+        stat_array[1] += nrn_threads[ith].n_presyn;        // number of presyns
+        stat_array[2] += nrn_threads[ith].n_input_presyn;  // number of input presyns
+        stat_array[3] += nrn_threads[ith].n_netcon;        // number of netcons, synapses
+        stat_array[4] += nrn_threads[ith].n_pntproc;       // number of point processes
         if (nrn_partrans::transfer_thread_data_) {
             size_t n = nrn_partrans::transfer_thread_data_[ith].tar_indices.size();
-            stat_array[11] += (long) n;  // number of transfer targets
+            stat_array[11] += n;  // number of transfer targets
             n = nrn_partrans::transfer_thread_data_[ith].src_indices.size();
-            stat_array[12] += (long) n;  // number of transfer sources
+            stat_array[12] += n;  // number of transfer sources
         }
     }
-    stat_array[5] = (long) spikevec_gid.size();  // number of spikes
+    stat_array[5] = spikevec_gid.size();  // number of spikes
 
     int spikevec_positive_gid_size = 0;
     for (std::size_t i = 0; i < spikevec_gid.size(); ++i) {
@@ -55,7 +55,7 @@ void report_cell_stats(void) {
         }
     }
 
-    stat_array[6] = (long) spikevec_positive_gid_size;  // number of non-negative gid spikes
+    stat_array[6] = spikevec_positive_gid_size;  // number of non-negative gid spikes
 
 #if NRNMPI
     nrnmpi_long_allreduce_vec(stat_array, gstat_array, NUM_STATS, 1);

tests/integration/CMakeLists.txt

@@ -30,6 +30,18 @@ set(TEST_CASES_WITH_ARGS
     "ring_gap_permute2!${RING_GAP_COMMON_ARGS} ${GPU_ARGS} --outpath ${CMAKE_CURRENT_BINARY_DIR}/ring_gap_permute2 ${PERMUTE2_ARGS}"
 )
 
+# ~~~
+# As reports require MPI, do not add test if report is enabled.
+# GPU tests need SLURM allocation and hence they won't run without srun
+# ~~~
+if(NOT CORENRN_ENABLE_REPORTING AND NOT CORENRN_ENABLE_GPU)
+  list(
+    APPEND
+    TEST_CASES_WITH_ARGS
+    "ring_serial!--tstop 100. --celsius 6.3 --datpath ${RING_DATASET_DIR} ${MODEL_STATS_ARG} --outpath ${CMAKE_CURRENT_BINARY_DIR}/ring_serial"
+  )
+endif()
+
 set(NEGATIVE_TEST_CASES_WITH_ARGS
     "ring_different_seed!${RING_COMMON_ARGS} ${GPU_ARGS} --outpath ${CMAKE_CURRENT_BINARY_DIR}/ring_different_seed --seed 123456"
 )
@@ -41,6 +53,7 @@ set(NEGATIVE_TEST_CASES_WITH_ARGS
 foreach(data_dir "ring" "ring_gap")
   foreach(
     test_suffix
+    "serial"
     "multisend"
     "binqueue"
     "savestate_permute0"
@@ -67,7 +80,12 @@ file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/ring/out.dat.ref"
 foreach(args_line ${TEST_CASES_WITH_ARGS})
   string(REPLACE "!" ";" string_line ${args_line})
   if(MPI_FOUND)
-    string(REPLACE ";" " " SRUN_PREFIX "${TEST_MPI_EXEC_BIN};-n;2")
+    # serial test run without srun or mpiexec
+    if(args_line MATCHES "ring_serial.*")
+      string(REPLACE ";" " " SRUN_PREFIX "")
+    else()
+      string(REPLACE ";" " " SRUN_PREFIX "${TEST_MPI_EXEC_BIN};-n;2")
+    endif()
   endif()
   list(GET string_line 0 TEST_NAME)
   list(GET string_line 1 TEST_ARGS)