If a netcon_srcgid is negative, need to determine the thread. (#390)

* If a netcon_srcgid is negative, need to determine the thread.

Only for direct transfer mode is it allowed that a negative srcgid is not
in the same thread as the NetCon target.

To enable thread determination an std::vector<int> involving the name
netcon_negsrcgid_tid is associated with netcon_srcgid in that when a negative
gid appears in netcon_srcgid, the tid is the value of the element in
netcon_negsrcgid_tid.

* always resize netcon_negsrcgid_tid

* link netcon_negsrcgid_tid usage to corenrn_embedded

* nrnthreads_netcon_srcgid is more meaninfgul name.

* Revert "link netcon_negsrcgid_tid usage to corenrn_embedded"

Comments indicate that nrnthreads_netcon_negsrc_gid_tid subvectors are
definitely empty when single thread or file transfer. (will be empty
anyway if there are no negative gids)

This reverts commit c870bf3.

Co-authored-by: Alexandru Savulescu <[email protected]>

Author: nrnhines

coreneuron/io/nrn2core_direct.h

@@ -30,7 +30,8 @@ extern int (*nrn2core_get_dat1_)(int tid,
                                  int& n_presyn,
                                  int& n_netcon,
                                  int*& output_gid,
-                                 int*& netcon_srcgid);
+                                 int*& netcon_srcgid,
+                                 std::vector<int>& netcon_negsrcgid_tid);
 
 extern int (*nrn2core_get_dat2_1_)(int tid,
                                    int& ngid,

coreneuron/io/nrn_setup.cpp

@@ -113,6 +113,14 @@ void (*nrn2core_all_weights_return_)(std::vector<double*>& weights);
 // encode the thread number into the negative gid
 // (i.e -ith - nth*(type +1000*index)) failed due to not large enough
 // integer domain size.
+// Note that for file transfer it is an error if a negative srcgid is
+// not in the same thread as the target. This is because there it may
+// not be the case that threads in a NEURON process end up on same process
+// in CoreNEURON. NEURON will raise an error if this
+// is the case. However, for direct memory transfer, it is allowed that
+// a negative srcgid may be in a different thread than the target. So
+// nrn2core_get_dat1 has a last arg netcon_negsrcgid_tid that specifies
+// for the negative gids in netcon_srcgid (in that order) the source thread.
 //
 // <firstgid>_2.dat
 // n_output n_real_output, nnode
@@ -178,7 +186,11 @@ std::map<int, InputPreSyn*> gid2in;
 std::vector<NetCon*> netcon_in_presyn_order_;
 
 /// Only for setup vector of netcon source gids
-std::vector<int*> netcon_srcgid;
+std::vector<int*> nrnthreads_netcon_srcgid;
+
+/// If a nrnthreads_netcon_srcgid is negative, need to determine the thread when
+/// in order to use the correct neg_gid2out[tid] map
+std::vector<std::vector<int> > nrnthreads_netcon_negsrcgid_tid;
 
 /* read files.dat file and distribute cellgroups to all mpi ranks */
 void nrn_read_filesdat(int& ngrp, int*& grp, const char* filesdat) {
@@ -275,8 +287,11 @@ void determine_inputpresyn() {
         NrnThread& nt = nrn_threads[ith];
         // associate gid with InputPreSyn and increase PreSyn and InputPreSyn count
         nt.n_input_presyn = 0;
+        // if single thread or file transfer then definitely empty.
+        std::vector<int>& negsrcgid_tid = nrnthreads_netcon_negsrcgid_tid[ith];
+        size_t i_tid = 0;
         for (int i = 0; i < nt.n_netcon; ++i) {
-            int gid = netcon_srcgid[ith][i];
+            int gid = nrnthreads_netcon_srcgid[ith][i];
             if (gid >= 0) {
                 /// If PreSyn or InputPreSyn is already in the map
                 auto gid2out_it = gid2out.find(gid);
@@ -299,8 +314,12 @@ void determine_inputpresyn() {
                 inputpresyn_.push_back(psi);
                 ++nt.n_input_presyn;
             } else {
-                auto gid2out_it = neg_gid2out[nt.id].find(gid);
-                if (gid2out_it != neg_gid2out[nt.id].end()) {
+                int tid = nt.id;
+                if (!negsrcgid_tid.empty()) {
+                    tid = negsrcgid_tid[i_tid++];
+                }
+                auto gid2out_it = neg_gid2out[tid].find(gid);
+                if (gid2out_it != neg_gid2out[tid].end()) {
                     /// Increase negative PreSyn count
                     ++gid2out_it->second->nc_cnt_;
                 }
@@ -361,16 +380,23 @@ void determine_inputpresyn() {
 
     // fill the netcon_in_presyn_order and recompute nc_cnt_
     // note that not all netcon_in_presyn will be filled if there are netcon
-    // with no presyn (ie. netcon_srcgid[nt.id][i] = -1) but that is ok since they are
+    // with no presyn (ie. nrnthreads_netcon_srcgid[nt.id][i] = -1) but that is ok since they are
     // only used via ps.nc_index_ and ps.nc_cnt_;
     for (int ith = 0; ith < nrn_nthread; ++ith) {
         NrnThread& nt = nrn_threads[ith];
+        // if single thread or file transfer then definitely empty.
+        std::vector<int>& negsrcgid_tid = nrnthreads_netcon_negsrcgid_tid[ith];
+        size_t i_tid = 0;
         for (int i = 0; i < nt.n_netcon; ++i) {
             NetCon* nc = nt.netcons + i;
-            int gid = netcon_srcgid[ith][i];
+            int gid = nrnthreads_netcon_srcgid[ith][i];
+            int tid = ith;
+            if (!negsrcgid_tid.empty() && gid < -1) {
+              tid = negsrcgid_tid[i_tid++];
+            }
             PreSyn* ps;
             InputPreSyn* psi;
-            netpar_tid_gid2ps(ith, gid, &ps, &psi);
+            netpar_tid_gid2ps(tid, gid, &ps, &psi);
             if (ps) {
                 netcon_in_presyn_order_[ps->nc_index_ + ps->nc_cnt_] = nc;
                 ++ps->nc_cnt_;
@@ -390,10 +416,11 @@ void determine_inputpresyn() {
 /// Clean up
 void nrn_setup_cleanup() {
     for (int ith = 0; ith < nrn_nthread; ++ith) {
-        if (netcon_srcgid[ith])
-            delete[] netcon_srcgid[ith];
+        if (nrnthreads_netcon_srcgid[ith])
+            delete[] nrnthreads_netcon_srcgid[ith];
     }
-    netcon_srcgid.clear();
+    nrnthreads_netcon_srcgid.clear();
+    nrnthreads_netcon_negsrcgid_tid.clear();
     neg_gid2out.clear();
 }
 
@@ -448,9 +475,9 @@ void nrn_setup(const char* filesdat,
     /// std::map<int, PreSyn*> gid2out;
     gid2out.clear();
 
-    netcon_srcgid.resize(nrn_nthread);
+    nrnthreads_netcon_srcgid.resize(nrn_nthread);
     for (int i = 0; i < nrn_nthread; ++i)
-        netcon_srcgid[i] = nullptr;
+        nrnthreads_netcon_srcgid[i] = nullptr;
 
     // gap junctions
     if (nrn_have_gaps) {
@@ -469,6 +496,7 @@ void nrn_setup(const char* filesdat,
         nrn_partrans::gap_mpi_setup(userParams.ngroup);
     }
 
+    nrnthreads_netcon_negsrcgid_tid.resize(nrn_nthread);
     if (!corenrn_embedded) {
         coreneuron::phase_wrapper<coreneuron::phase::one>(userParams);
     } else {
@@ -480,7 +508,7 @@ void nrn_setup(const char* filesdat,
         });
     }
 
-    // from the gid2out map and the netcon_srcgid array,
+    // from the gid2out map and the nrnthreads_netcon_srcgid array,
     // fill the gid2in, and from the number of entries,
     // allocate the process wide InputPreSyn array
     determine_inputpresyn();

coreneuron/io/phase1.cpp

@@ -10,7 +10,8 @@ int (*nrn2core_get_dat1_)(int tid,
                           int& n_presyn,
                           int& n_netcon,
                           int*& output_gid,
-                          int*& netcon_srcgid);
+                          int*& netcon_srcgid,
+                          std::vector<int>& netcon_negsrcgid_tid);
 
 namespace coreneuron {
 void Phase1::read_file(FileHandler& F) {
@@ -20,6 +21,8 @@ void Phase1::read_file(FileHandler& F) {
 
     this->output_gids = F.read_vector<int>(n_presyn);
     this->netcon_srcgids = F.read_vector<int>(n_netcon);
+    // For file mode transfer, it is not allowed that negative gids exist
+    // in different threads. So this->netcon_tids remains clear.
 
     F.close();
 }
@@ -32,7 +35,7 @@ void Phase1::read_direct(int thread_id) {
 
     // TODO : check error codes for NEURON - CoreNEURON communication
     int valid =
-        (*nrn2core_get_dat1_)(thread_id, n_presyn, n_netcon, output_gids, netcon_srcgid);
+        (*nrn2core_get_dat1_)(thread_id, n_presyn, n_netcon, output_gids, netcon_srcgid, this->netcon_negsrcgid_tid);
     if (!valid) {
         return;
     }
@@ -47,9 +50,12 @@ void Phase1::populate(NrnThread& nt, OMP_Mutex& mut) {
     nt.n_presyn = this->output_gids.size();
     nt.n_netcon = this->netcon_srcgids.size();
 
-    netcon_srcgid[nt.id] = new int[nt.n_netcon];
+    nrnthreads_netcon_srcgid[nt.id] = new int[nt.n_netcon];
     std::copy(this->netcon_srcgids.begin(), this->netcon_srcgids.end(),
-              netcon_srcgid[nt.id]);
+              nrnthreads_netcon_srcgid[nt.id]);
+
+    // netcon_negsrcgid_tid is empty if file transfer or single thread
+    coreneuron::nrnthreads_netcon_negsrcgid_tid[nt.id] = this->netcon_negsrcgid_tid;
 
     nt.netcons = new NetCon[nt.n_netcon];
     nt.presyns_helper = (PreSynHelper*)ecalloc_align(nt.n_presyn, sizeof(PreSynHelper));

coreneuron/io/phase1.hpp

@@ -18,6 +18,7 @@ class Phase1 {
     private:
     std::vector<int> output_gids;
     std::vector<int> netcon_srcgids;
+    std::vector<int> netcon_negsrcgid_tid; // entries only for negative srcgids
 };
 
 }  // namespace coreneuron

coreneuron/network/netpar.cpp

@@ -641,10 +641,17 @@ double set_mindelay(double maxdelay) {
 
     for (int ith = 0; ith < nrn_nthread; ++ith) {
         NrnThread& nt = nrn_threads[ith];
+        // if single thread or file transfer then definitely empty.
+        std::vector<int>& negsrcgid_tid = nrnthreads_netcon_negsrcgid_tid[ith];
+        size_t i_tid = 0;
         for (int i = 0; i < nt.n_netcon; ++i) {
             NetCon* nc = nt.netcons + i;
             bool chk = false;  // ignore nc.delay_
-            int gid = netcon_srcgid[ith][i];
+            int gid = nrnthreads_netcon_srcgid[ith][i];
+            int tid = ith;
+            if (!negsrcgid_tid.empty() && gid < -1) {
+              tid = negsrcgid_tid[i_tid++];
+            }  
             PreSyn* ps;
             InputPreSyn* psi;
             netpar_tid_gid2ps(ith, gid, &ps, &psi);

coreneuron/nrniv/nrniv_decl.h

@@ -48,8 +48,11 @@ extern std::map<int, InputPreSyn*> gid2in;
 
 /// InputPreSyn.nc_index_ to + InputPreSyn.nc_cnt_ give the NetCon*
 extern std::vector<NetCon*> netcon_in_presyn_order_;
-/// Only for setup vector of netcon source gids
-extern std::vector<int*> netcon_srcgid;
+/// Only for setup vector of netcon source gids and mindelay determination
+extern std::vector<int*> nrnthreads_netcon_srcgid;
+/// Companion to nrnthreads_netcon_srcgid when src gid is negative to allow
+/// determination of the NrnThread of the source PreSyn.
+extern std::vector<std::vector<int> > nrnthreads_netcon_negsrcgid_tid;
 
 extern void mk_mech(const char* path);
 extern void set_globals(const char* path, bool cli_global_seed, int cli_global_seed_value);