Minor improvements and clarifications for GPU build (#452)

* Minor improvements and clarifications for GPU build
  - Fix pthread link error (#428)
  - Error if GPU support is enabled without PGI compiler
    Currently GPU support is only enabled via OpenACC and
    tested only with PGI compiler only.
  - Change CUDA to 9.0 as we are no longer testing/developing
    with old CUDA. And PGI compilers are also shipping newer
    CUDA libraries.
- Build internal mod files in serial fashion to avoid mixing parallllel builds
  (see neuronsimulator/nrn/issues/889)

fixes #428

Author: pramodk

CMake/OpenAccHelper.cmake

@@ -34,7 +34,8 @@ if(CORENRN_ENABLE_GPU)
     set(CMAKE_CXX11_STANDARD_COMPILE_OPTION --c++11)
     set(CMAKE_CXX14_STANDARD_COMPILE_OPTION --c++14)
   else()
-    message(WARNING "Non-PGI compiler : make sure to add required compiler flags to enable OpenACC")
+    message(FATAL_ERROR "GPU support is available via OpenACC using PGI/NVIDIA compilers."
+                        " Use NVIDIA HPC SDK with -DCMAKE_C_COMPILER=nvc -DCMAKE_CXX_COMPILER=nvc++")
   endif()
 
   # set property for neuron to link with coreneuron libraries
@@ -49,7 +50,7 @@ if(CORENRN_ENABLE_GPU)
   if(POLICY CMP0074)
     cmake_policy(SET CMP0074 NEW)
   endif()
-  find_package(CUDA 5.0 REQUIRED)
+  find_package(CUDA 9.0 REQUIRED)
   set(CUDA_SEPARABLE_COMPILATION ON)
   set(CUDA_PROPAGATE_HOST_FLAGS OFF)
   add_definitions(-DCUDA_PROFILING)

README.md

@@ -10,14 +10,14 @@ CoreNEURON is a compute engine for the [NEURON](https://www.neuron.yale.edu/neur
 CoreNEURON is designed as a library within the NEURON simulator and can transparently handle all spiking network simulations including gap junction coupling with the **fixed time step method**. In order to run a NEURON model with CoreNEURON:
 
 * MOD files should be THREADSAFE
-* MOD files must use the Random123 random number generator (instead of MCellRan4)
+* If random number generator is used then Random123 should be used instead of MCellRan4
 * POINTER variables need to be converted to BBCOREPOINTER ([details here](http://bluebrain.github.io/CoreNeuron/index.html))
 
 ## Dependencies
 * [CMake 3.7+](https://cmake.org)
-* [MPI 2.0+](http://mpich.org) [Optional]
-* [PGI OpenACC Compiler >=19.0](https://www.pgroup.com/resources/accel.htm) [Optional, for GPU support]
-* [CUDA Toolkit >=6.0](https://developer.nvidia.com/cuda-toolkit-60) [Optional, for GPU support]
+* MPI Library [Optional, for MPI support]
+* [PGI OpenACC Compiler / NVIDIA HPC SDK](https://developer.nvidia.com/hpc-sdk) [Optional, for GPU support]
+* [CUDA Toolkit >=9.0](https://developer.nvidia.com/cuda-downloads) [Optional, for GPU support]
 
 In addition to this, you will need other [NEURON dependencies](https://github.com/neuronsimulator/nrn) such as Python, Flex, Bison etc.
 
@@ -64,7 +64,8 @@ Note that if you are building on Cray system with the GNU toolchain, you have to
    -DNRN_ENABLE_RX3D=OFF \
    -DCMAKE_INSTALL_PREFIX=$HOME/install
   ```
-If you would like to enable GPU support with OpenACC, make sure to use `-DCORENRN_ENABLE_GPU=ON` option and use the PGI compiler with CUDA.
+If you would like to enable GPU support with OpenACC, make sure to use `-DCORENRN_ENABLE_GPU=ON` option and use the PGI/NVIDIA HPC SDK compilers with CUDA.
+
 > NOTE : if the CMake command files, please make sure to delete temporary CMake cache files (`CMakeCache.txt`) before rerunning CMake.
 
 4. Build and Install :  once the configure step is done, you can build and install the project as:
@@ -161,6 +162,8 @@ nrn_spike_gids = nrn_spike_gids.to_python()
 # now run CoreNEURON
 from neuron import coreneuron
 coreneuron.enable = True
+# for GPU support
+# coreneuron.gpu = True
 coreneuron.verbose = 0
 h.stdinit()
 corenrn_all_spike_t = h.Vector()
@@ -214,6 +217,8 @@ cmake .. -DCMAKE_CXX_FLAGS="-O2 -Minline=size:1000,levels:100,totalsize:40000,ma
          -DCORENRN_ENABLE_GPU=ON -DCMAKE_INSTALL_PREFIX=$HOME/install
 ```
 
+For other errors, please [open an issue](https://github.com/BlueBrain/CoreNeuron/issues).
+
 
 ## Developer Build
 

coreneuron/CMakeLists.txt

@@ -169,6 +169,7 @@ add_custom_target(
   nrniv-core ALL
   COMMAND ${CMAKE_BINARY_DIR}/bin/nrnivmodl-core -b STATIC
           -m ${CORENRN_MOD2CPP_BINARY}
+          -p 1
           ${CORENEURON_PROJECT_SOURCE_DIR}/tests/integration/ring_gap/mod
   WORKING_DIRECTORY
     ${CMAKE_BINARY_DIR}/bin

coreneuron/gpu/nrn_acc_manager.cpp

@@ -566,8 +566,6 @@ void update_net_send_buffer_on_host(NrnThread* nt, NetSendBuffer_t* nsb) {
 void update_nrnthreads_on_host(NrnThread* threads, int nthreads) {
 #ifdef _OPENACC
 
-    printf("\n --- Copying to Host! --- \n");
-
     for (int i = 0; i < nthreads; i++) {
         NrnThread* nt = threads + i;
 
@@ -662,8 +660,6 @@ void update_nrnthreads_on_host(NrnThread* threads, int nthreads) {
 void update_nrnthreads_on_device(NrnThread* threads, int nthreads) {
 #ifdef _OPENACC
 
-    printf("\n --- Copying to Device! --- \n");
-
     for (int i = 0; i < nthreads; i++) {
         NrnThread* nt = threads + i;
 
@@ -820,7 +816,6 @@ void update_matrix_to_gpu(NrnThread* _nt) {
         /* while discussion with Michael we found that RHS is also needed on
          * gpu because nrn_cap_jacob uses rhs which is being updated on GPU
          */
-        // printf("\n Copying voltage to GPU ... ");
         double* v = _nt->_actual_v;
         double* rhs = _nt->_actual_rhs;
         int ne = nrn_soa_padded_size(_nt->end, 0);

extra/CMakeLists.txt

@@ -22,9 +22,12 @@ endif()
 # imported target to link line
 # ~~~
 list(REMOVE_ITEM CORENRN_LINK_LIBS "Threads::Threads")
-
 # replicate CMake magic to transform system libs to -l<libname>
 foreach(link_lib ${CORENRN_LINK_LIBS})
+  if(${link_lib} MATCHES "\-l.*")
+      string(APPEND CORENRN_LINK_DEFS " ${link_lib}")
+      continue()
+  endif()
   get_filename_component(path ${link_lib} DIRECTORY)
   if(NOT path)
     string(APPEND CORENRN_LINK_DEFS " -l${link_lib}")