Bugfix : Support for using CoreNEURON in embedded mode with external mechanisms (#275)

  - modl_reg was not called when corenrn_embedded_run was used
    for running simulations with in-memory mode transfer
  - move corenrn_embedded_run() into enginemech.cpp to avoid
    link error for calling coreneuron::modl_reg
  - add test for external mechanism in embedded mode
  - source setup-env.sh to use neuron@develop module

fixes #274

Author: pramodk

coreneuron/apps/main1.cpp

@@ -126,25 +126,6 @@ char* prepare_args(int& argc, char**& argv, int use_mpi, const char* arg) {
     return first;
 }
 
-int corenrn_embedded_run(int nthread, int have_gaps, int use_mpi, int use_fast_imem, const char* arg) {
-    corenrn_embedded = true;
-    corenrn_embedded_nthread = nthread;
-    coreneuron::nrn_have_gaps = have_gaps != 0;
-    if (use_fast_imem != 0) {
-        coreneuron::nrn_use_fast_imem = true;
-    }
-
-    set_openmp_threads(nthread);
-    int argc = 0;
-    char** argv;
-    char* new_arg = prepare_args(argc, argv, use_mpi, arg);
-    mk_mech_init(argc, argv);
-    run_solve_core(argc, argv);
-    free(new_arg);
-    delete[] argv;
-
-    return corenrn_embedded ? 1 : 0;
-}
 }
 
 namespace coreneuron {

coreneuron/mechanism/mech/enginemech.cpp

@@ -1,21 +1,107 @@
-/// Corenrnmech is a wrapper lib providing a single solve_core function
-/// which initializes the solver, loads the external mechanisms and launches the simulation
+/**
+ * \file
+ * \brief Provides interface function for CoreNEURON mechanism library and NEURON
+ *
+ * libcorenrnmech is a interface library provided to building standalone executable
+ * special-core. Also, it is used by NEURON to run CoreNEURON via dlopen to execute
+ * models via in-memory transfer.
+ */
 
+#include <cstdlib>
 #include <coreneuron/engine.h>
 
-#ifdef ADDITIONAL_MECHS
 namespace coreneuron {
+
+/** Mechanism registration function
+ *
+ * If external mechanisms present then use modl_reg function generated
+ * in mod_func.cpp otherwise use empty one.
+ */
+#ifdef ADDITIONAL_MECHS
 extern void modl_reg();
+#else
+void modl_reg() {
 }
 #endif
 
+/// variables defined in coreneuron library
+extern bool nrn_have_gaps;
+extern bool nrn_use_fast_imem;
+
+}  // namespace coreneuron
+
+/** Initialize mechanisms and run simulation using CoreNEURON
+ *
+ * This is mainly used to build nrniv-core executable
+ */
 int solve_core(int argc, char** argv) {
     mk_mech_init(argc, argv);
+    coreneuron::modl_reg();
+    return run_solve_core(argc, argv);
+}
 
-#ifdef ADDITIONAL_MECHS
-    /// Initializing additional Neurodamus mechanisms (in mod_func.c, built by mech/mod_func.c.pl)
+extern "C" {
+
+/// global variables from coreneuron library
+extern bool corenrn_embedded;
+extern int corenrn_embedded_nthread;
+
+/// parse arguments from neuron and prepare new one for coreneuron
+char* prepare_args(int& argc, char**& argv, int use_mpi, const char* nrn_arg);
+
+/// initialize standard mechanisms from coreneuron
+void mk_mech_init(int argc, char** argv);
+
+/// set openmp threads equal to neuron's pthread
+void set_openmp_threads(int nthread);
+
+/** Run CoreNEURON in embedded mode with NEURON
+ *
+ * @param nthread Number of Pthreads on NEURON side
+ * @param have_gaps True if gap junctions are used
+ * @param use_mpi True if MPI is used on NEURON side
+ * @param use_fast_imem True if fast imembrance calculation enabled
+ * @param nrn_arg Command line arguments passed by NEURON
+ * @return 1 if embedded mode is used otherwise 0
+ * \todo Change return type semantics
+ */
+int corenrn_embedded_run(int nthread,
+                         int have_gaps,
+                         int use_mpi,
+                         int use_fast_imem,
+                         const char* nrn_arg) {
+    // set coreneuron's internal variable based on neuron arguments
+    corenrn_embedded = true;
+    corenrn_embedded_nthread = nthread;
+    coreneuron::nrn_have_gaps = have_gaps != 0;
+
+    if (use_fast_imem != 0) {
+        coreneuron::nrn_use_fast_imem = true;
+    }
+
+    // set number of openmp threads
+    set_openmp_threads(nthread);
+
+    // pre-process argumnets from neuron and prepare new for coreneuron
+    int argc;
+    char** argv;
+    char* new_arg = prepare_args(argc, argv, use_mpi, nrn_arg);
+
+    // initialize internal arguments
+    mk_mech_init(argc, argv);
+
+    // initialize extra arguments built into special-core
     coreneuron::modl_reg();
-#endif
 
-    return run_solve_core(argc, argv);
+    // run simulation
+    run_solve_core(argc, argv);
+
+    // free temporary string created from prepare_args
+    free(new_arg);
+
+    // delete array for argv
+    delete[] argv;
+
+    return corenrn_embedded ? 1 : 0;
+}
 }

tests/jenkins/_env_setup.sh

@@ -9,8 +9,11 @@ INSTALL_HOME="${WORKSPACE}/INSTALL_HOME"
 export SPACK_ROOT="${BUILD_HOME}/spack"
 export SPACK_INSTALL_PREFIX="${SPACK_INSTALL_PREFIX:-${INSTALL_HOME}}"
 export SOFTS_DIR_PATH=$SPACK_INSTALL_PREFIX  # Deprecated, but might still be reqd
-export PATH=$SPACK_ROOT/bin:/usr/bin:$PATH
-export MODULEPATH=$SPACK_INSTALL_PREFIX/modules/tcl/$(spack arch):$MODULEPATH
+export PATH=/usr/bin:$PATH
+
+if [ -d "$SPACK_ROOT" ]; then
+    source $SPACK_ROOT/share/spack/setup-env.sh
+fi
 
 # Common init
 unset $(env|awk -F= '/^(PMI|SLURM)_/ {if ($1 != "SLURM_ACCOUNT") print $1}')

tests/jenkins/install_neuron.sh

@@ -21,7 +21,7 @@ patch_neuron() (
 set -e
 source ${JENKINS_DIR:-.}/_env_setup.sh
 
-set -x
 patch_neuron
 spack install neuron+debug@develop
+source $SPACK_ROOT/share/spack/setup-env.sh
 module av neuron

tests/jenkins/mod/CaDynamics_E2.mod

@@ -0,0 +1,36 @@
+: Dynamics that track inside calcium concentration
+: modified from Destexhe et al. 1994
+
+NEURON	{
+	SUFFIX CaDynamics_E2
+	USEION ca READ ica WRITE cai
+	RANGE decay, gamma, minCai, depth
+}
+
+UNITS	{
+	(mV) = (millivolt)
+	(mA) = (milliamp)
+	FARADAY = (faraday) (coulombs)
+	(molar) = (1/liter)
+	(mM) = (millimolar)
+	(um)	= (micron)
+}
+
+PARAMETER	{
+	gamma = 0.05 : percent of free calcium (not buffered)
+	decay = 80 (ms) : rate of removal of calcium
+	depth = 0.1 (um) : depth of shell
+	minCai = 1e-4 (mM)
+}
+
+ASSIGNED	{ica (mA/cm2)}
+
+STATE	{
+	cai (mM)
+	}
+
+BREAKPOINT	{ SOLVE states METHOD cnexp }
+
+DERIVATIVE states	{
+	cai' = -(10000)*(ica*gamma/(2*FARADAY*depth)) - (cai - minCai)/decay
+}

tests/jenkins/neuron_direct.py

@@ -10,6 +10,9 @@
 ic.dur = 0.1
 ic.amp = 0.3
 
+# for testing external mod file
+h.soma.insert("CaDynamics_E2")
+
 h.cvode.use_fast_imem(1)
 h.cvode.cache_efficient(1)
 

tests/jenkins/nrnivmodl.sh

@@ -2,7 +2,7 @@
 
 set -e
 source ${JENKINS_DIR:-.}/_env_setup.sh
-module load neuron
+module load neuron/develop
 
 set -x
 TEST_DIR="$1"

tests/jenkins/run_neuron_direct.sh

@@ -2,9 +2,24 @@
 
 set -e
 source ${JENKINS_DIR:-.}/_env_setup.sh
-module load neuron
+module load neuron/develop intel
 
 set -x
 CORENRN_TYPE="$1"
-export CORENEURONLIB=$WORKSPACE/install_${CORENRN_TYPE}/lib/libcoreneuron.so
+export PATH=$WORKSPACE/install_${CORENRN_TYPE}/bin:$PATH
+
+# temporary build directory
+build_dir=$(mktemp -d $(pwd)/build_XXXX)
+cd $build_dir
+
+# build special and special-core
+nrnivmodl ../tests/jenkins/mod
+nrnivmodl-core ../tests/jenkins/mod
+ls -la x86_64
+
+# run test sim with external mechanism
 python $WORKSPACE/tests/jenkins/neuron_direct.py
+
+# remove build directory
+cd -
+rm -rf $build_dir

tests/jenkins/spack_setup.sh

@@ -23,7 +23,7 @@ install_spack() (
     # Use BBP configs
     mkdir -p $SPACK_ROOT/etc/spack/defaults/linux
     cp /gpfs/bbp.cscs.ch/apps/hpc/jenkins/config/*.yaml $SPACK_ROOT/etc/spack/
-
+    sed -i -e  's/neuron+mpi~debug%intel/neuron+mpi/g' $SPACK_ROOT/etc/spack/modules.yaml
     # Remove configs from $HOME/.spack
     rm -rf $HOME/.spack
 )