Various CMake changes to improve GPU build

 - CORENRN_NMODL_FLAGS set automatically from CMake
 - PGI flags added via CMake : inlining, diagnostics and language compliant flag -A removed
 - Cmake copy set to new to avoid warning from find_cuda
 - nrnivmodl_core_makefile.in to build libcorenrnmech static library
 - tests were not run, fix cmake

Author: pramodk

CMake/OpenAccHelper.cmake

@@ -5,21 +5,38 @@
 # =============================================================================
 
 if(CORENRN_ENABLE_GPU)
+
   set(COMPILE_LIBRARY_TYPE "STATIC")
+
   if(CORENRN_ENABLE_CUDA_UNIFIED_MEMORY)
     add_definitions(-DUNIFIED_MEMORY)
   endif()
+
+  if(NOT CUDA_HOST_COMPILER)
+    find_program(GCC_BIN gcc)
+    set(CUDA_HOST_COMPILER ${GCC_BIN} CACHE FILEPATH "" FORCE)
+  endif()
+
   if(${CMAKE_C_COMPILER_ID} STREQUAL "PGI")
     add_definitions(-DPG_ACC_BUGS)
-    set(ACC_FLAGS "-acc -Minline=size:200,levels:10")
+    set(ACC_FLAGS "-acc")
+    set(PGI_DIAG_FLAGS "--diag_suppress 177")
+    set(PGI_INLINE_FLAGS "-Minline=size:200,levels:10")
     set(CMAKE_C_FLAGS "${ACC_FLAGS} ${CMAKE_C_FLAGS}")
-    set(CMAKE_CXX_FLAGS "${ACC_FLAGS} ${CMAKE_CXX_FLAGS}")
+    set(CMAKE_CXX_FLAGS "${ACC_FLAGS} ${CMAKE_CXX_FLAGS} ${PGI_DIAG_FLAGS}")
+    set(CMAKE_CXX11_STANDARD_COMPILE_OPTION  --c++11)
+    set(CMAKE_CXX14_STANDARD_COMPILE_OPTION  --c++14)
   else()
     message(WARNING "Non-PGI compiler : make sure to add required compiler flags to enable OpenACC")
   endif()
 
+  if(POLICY CMP0074)
+    cmake_policy(SET CMP0074 NEW)
+  endif()
   find_package(CUDA 5.0 REQUIRED)
   set(CUDA_SEPARABLE_COMPILATION ON)
+  set(CUDA_PROPAGATE_HOST_FLAGS OFF)
+
   add_definitions(-DCUDA_PROFILING)
 else(CORENRN_ENABLE_GPU)
   # OpenACC pragmas are not guarded, disable all unknown pragm warnings

CMakeLists.txt

@@ -192,7 +192,11 @@ if(CORENRN_ENABLE_NMODL)
     set(CORENRN_NMODL_INCLUDE ${CMAKE_BINARY_DIR}/include)
   endif()
   include_directories(${CORENRN_NMODL_INCLUDE})
-  set(CORENRN_NMODL_FLAGS "" CACHE STRING "Extra NMODL options such as passes")
+  if(CORENRN_ENABLE_GPU)
+    set(CORENRN_NMODL_FLAGS "acc --oacc" CACHE STRING "Extra NMODL options such as passes")
+  else()
+    set(CORENRN_NMODL_FLAGS "" CACHE STRING "Extra NMODL options such as passes")
+  endif()
   separate_arguments(NMODL_EXTRA_FLAGS_LIST UNIX_COMMAND "${CORENRN_NMODL_FLAGS}")
 else()
   include(AddMod2cSubmodule)
@@ -236,6 +240,15 @@ endif()
 install(FILES CMake/coreneuron-config.cmake DESTINATION share/cmake)
 install(EXPORT coreneuron DESTINATION share/cmake)
 
+# just for printing the compiler flags in the build status
+string(TOUPPER ${CMAKE_BUILD_TYPE} BUILD_TYPE_UPPER)
+if(BUILD_TYPE_UPPER MATCHES "CUSTOM")
+  set(COMPILER_FLAGS "${CMAKE_CXX_FLAGS}")
+else()
+  set(COMPILER_FLAGS "${CMAKE_CXX_FLAGS_${BUILD_TYPE_UPPER}}")
+endif()
+
+
 # =============================================================================
 # Build status
 # =============================================================================
@@ -253,6 +266,7 @@ if(cmake_generator_tolower MATCHES "makefile")
   message(STATUS " Build option       | Status")
   message(STATUS "--------------------+--------------------------------------------------------")
 
+  message(STATUS "COMPILE FLAGS       | ${COMPILER_FLAGS} ${CMAKE_CXX_FLAGS}")
   message(STATUS "MPI                 | ${CORENRN_ENABLE_MPI}")
   if(CORENRN_ENABLE_MPI)
     message(STATUS "  INC               | ${MPI_C_INCLUDE_PATH}")
@@ -262,11 +276,16 @@ if(cmake_generator_tolower MATCHES "makefile")
   if(CORENRN_ENABLE_NMODL)
     message(STATUS "  PATH              | ${CORENRN_NMODL_BINARY}")
     message(STATUS "  ISPC              | ${CORENRN_ENABLE_ISPC}")
+    message(STATUS "  FLAGS             | ${CORENRN_NMODL_FLAGS}")
   else()
     message(STATUS "MOD2C               | ${MOD2C_PROJ}")
   endif()
   message(STATUS "GPU Support         | ${CORENRN_ENABLE_GPU}")
-  message(STATUS "CUDA Unified Memory | ${CORENRN_ENABLE_CUDA_UNIFIED_MEMORY}")
+  if(CORENRN_ENABLE_GPU)
+    message(STATUS "  CUDA              | ${CUDA_TOOLKIT_ROOT_DIR}")
+    message(STATUS "  Unified Memory    | ${CORENRN_ENABLE_CUDA_UNIFIED_MEMORY}")
+    message(STATUS "  HOST COMPILER     | ${CUDA_HOST_COMPILER}")
+  endif()
   message(STATUS "Auto Timeout        | ${CORENRN_ENABLE_TIMEOUT}")
   message(STATUS "Wrap exp()          | ${CORENRN_ENABLE_HOC_EXP}")
   message(STATUS "SplayTree Queue     | ${CORENRN_ENABLE_SPLAYTREE_QUEUING}")

README.md

@@ -80,12 +80,7 @@ module load pgi/18.4 cuda/9.0.176 cmake intel-mpi # change pgi, cuda and mpi mod
 export CC=mpicc
 export CXX=mpicxx
 
-cmake ..  -DCMAKE_C_FLAGS:STRING="-O2" \
-          -DCMAKE_CXX_FLAGS:STRING="-O2" \
-          -DCOMPILE_LIBRARY_TYPE=STATIC \
-          -DCUDA_HOST_COMPILER=`which gcc` \
-          -DCUDA_PROPAGATE_HOST_FLAGS=OFF \
-          -DCORENRN_ENABLE_GPU=ON
+cmake .. -DCORENRN_ENABLE_GPU=ON
 ```
 
 Note that the CUDA Toolkit version should be compatible with PGI compiler installed on your system. Otherwise you have to add extra C/C++ flags. For example, if we are using CUDA Toolkit 9.0 installation but PGI default target is CUDA 8.0 then we have to add :

extra/nrnivmodl_core_makefile.in

@@ -40,7 +40,7 @@ INCLUDES += $(if @reportinglib_INCLUDE_DIR@, -I$(subst ;, -I,@reportinglib_INCLU
 CC = @CMAKE_C_COMPILER@
 CXX = @CMAKE_CXX_COMPILER@
 CFLAGS = @BUILD_TYPE_C_FLAGS@ @CMAKE_C_FLAGS@
-CXXFLAGS = @BUILD_TYPE_CXX_FLAGS@ @CMAKE_CXX_FLAGS@ @CXX14_STANDARD_COMPILE_OPTION@
+CXXFLAGS = @BUILD_TYPE_CXX_FLAGS@ @CMAKE_CXX_FLAGS@ @CXX14_STANDARD_COMPILE_OPTION@ @PGI_INLINE_FLAGS@
 
 COMPILE = $(CC) $(CFLAGS) @CORENRN_COMPILE_DEFS@ $(INCLUDES)
 CXXCOMPILE = $(CXX) $(CXXFLAGS) @CORENRN_COMPILE_DEFS@ $(INCLUDES)
@@ -96,7 +96,12 @@ dimplic_o  = $(OBJS_DIR)/_dimplic.o
 
 special  = $(OUTPUT)/special-core
 coremech_libname = corenrnmech$(if $(MECH_NAME),_$(MECH_NAME),)
-coremech_lib = $(OUTPUT)/lib$(coremech_libname)@CMAKE_SHARED_LIBRARY_SUFFIX@
+ifeq (@COMPILE_LIBRARY_TYPE@, STATIC)
+    library_suffix = @CMAKE_STATIC_LIBRARY_SUFFIX@
+else
+    library_suffix = @CMAKE_SHARED_LIBRARY_SUFFIX@
+endif
+coremech_lib = $(OUTPUT)/lib$(coremech_libname)$(library_suffix)
 
 # If no DESTDIR (we are probably just building) we use $ORIGIN (@loader_path in OSX)
 _ORIGIN := $(if $(filter Darwin,$(OS_NAME)),@loader_path,$$ORIGIN)
@@ -106,7 +111,6 @@ DESTDIR_RPATH = $(if $(DESTDIR),$(DESTDIR)/lib,$(_ORIGIN))
 C_RESET := \033[0m
 C_GREEN := \033[32m
 
-
 # ======== MAIN BUILD RULES ============
 
 # Take the main and link with nrnmech.
@@ -119,9 +123,15 @@ $(special): $(coremech_lib)
 
 $(coremech_lib): $(mod_func_o) $(dimplic_o) $(mod_c_objs) $(mod_ispc_objs) build_always
 	@printf " => $(C_GREEN)LINKING$(C_RESET) library $(coremech_lib) Mod files: $(mod_files)\n"
-	$(CXX_LINK_SHARED) $(INCFLAGS) -I $(incdir) -DADDITIONAL_MECHS $(datadir)/enginemech.cpp -o ${coremech_lib} ${_SONAME} \
-	  $(mod_func_o) $(dimplic_o) $(mod_c_objs) $(mod_ispc_objs) $(libdir)/libscopmath.a $(CORENRNLIB_FLAGS) -Wl,-rpath,$(libdir) $(LDFLAGS)
-	(rm -f $(OUTPUT)/.libs/libcorenrnmech@CMAKE_SHARED_LIBRARY_SUFFIX@ ; mkdir -p $(OUTPUT)/.libs ; ln -s ../../${coremech_lib} $(OUTPUT)/.libs/libcorenrnmech@CMAKE_SHARED_LIBRARY_SUFFIX@)
+	@rm -f ${coremech_lib}
+	$(CXXCOMPILE) @CMAKE_CXX_COMPILE_OPTIONS_PIC@ -c -DADDITIONAL_MECHS $(datadir)/enginemech.cpp
+	@if [ "@COMPILE_LIBRARY_TYPE@" = "SHARED" ]; then\
+		$(CXX_LINK_SHARED) $(INCFLAGS) -I $(incdir) enginemech.o -o ${coremech_lib} ${_SONAME} \
+			$(mod_func_o) $(dimplic_o) $(mod_c_objs) $(mod_ispc_objs) $(libdir)/libscopmath.a $(CORENRNLIB_FLAGS) -Wl,-rpath,$(libdir) $(LDFLAGS);\
+	else\
+		ar cq ${coremech_lib} enginemech.o $(mod_func_o) $(dimplic_o) $(mod_c_objs) $(mod_ispc_objs);\
+	fi
+	(rm -f $(OUTPUT)/.libs/libcorenrnmech$(library_suffix) ; mkdir -p $(OUTPUT)/.libs ; ln -s ../../${coremech_lib} $(OUTPUT)/.libs/libcorenrnmech$(library_suffix))
 
 
 # Generic build cpp->.o Need PIC for shared lib

tests/integration/CMakeLists.txt

@@ -8,14 +8,14 @@ set(COMMON_ARGS "--tstop 100. --celsius 6.3 --mpi")
 set(RING_DATASET_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ring")
 set(RING_COMMON_ARGS "--datpath ${RING_DATASET_DIR} ${COMMON_ARGS}")
 set(RING_GAP_COMMON_ARGS "--datpath ${CMAKE_CURRENT_SOURCE_DIR}/ring_gap ${COMMON_ARGS}")
-set(PERMUTE1_ARGS "--cell-permute 1 --nwarp 8")
-set(PERMUTE2_ARGS "--cell-permute 2 --nwarp 16")
+set(PERMUTE1_ARGS "--cell-permute 1")
+set(PERMUTE2_ARGS "--cell-permute 2")
 if(CORENRN_ENABLE_GPU)
   set(GPU_ARGS "--nwarp 8 --gpu")
 endif()
 
 # List of tests with arguments
-set(TEST_CASE_ARGS
+set(TEST_CASES_WITH_ARGS
     "ring!${RING_COMMON_ARGS} ${GPU_ARGS} --outpath ${CMAKE_CURRENT_BINARY_DIR}/ring"
     "ring_binqueue!${RING_COMMON_ARGS} ${GPU_ARGS} --outpath ${CMAKE_CURRENT_BINARY_DIR}/ring_binqueue --binqueue"
     "ring_multisend!${RING_COMMON_ARGS} ${GPU_ARGS} --outpath ${CMAKE_CURRENT_BINARY_DIR}/ring_multisend --multisend"
@@ -30,7 +30,7 @@ set(TEST_CASE_ARGS
     "ring_gap_permute2!${RING_GAP_COMMON_ARGS} ${GPU_ARGS} --outpath ${CMAKE_CURRENT_BINARY_DIR}/ring_gap_permute2 ${PERMUTE2_ARGS}"
 )
 
-set(NEGATIVE_TEST_CASE_ARGS "ring_different_seed!${RING_COMMON_ARGS} ${GPU_ARGS} --outpath ${CMAKE_CURRENT_BINARY_DIR}/ring_different_seed --seed 123456")
+set(NEGATIVE_TEST_CASES_WITH_ARGS "ring_different_seed!${RING_COMMON_ARGS} ${GPU_ARGS} --outpath ${CMAKE_CURRENT_BINARY_DIR}/ring_different_seed --seed 123456")
 
 # There are no directories for permute and multisend related tests, create them and copy ref spikes
 foreach(data_dir "ring" "ring_gap")

tests/jenkins/install_coreneuron.sh

@@ -23,11 +23,6 @@ cd $WORKSPACE/build_${CORENRN_TYPE}
 echo "${CORENRN_TYPE} build"
 if [ "${CORENRN_TYPE}" = "GPU-non-unified" ]; then
     cmake \
-        -DCMAKE_C_FLAGS:STRING="-O2" \
-        -DCMAKE_CXX_FLAGS:STRING="-O2 -D_GLIBCXX_USE_CXX11_ABI=0 -DR123_USE_SSE=0" \
-        -DCOMPILE_LIBRARY_TYPE=STATIC  \
-        -DCUDA_HOST_COMPILER=`which gcc` \
-        -DCUDA_PROPAGATE_HOST_FLAGS=OFF \
         -DCORENRN_ENABLE_GPU=ON \
         -DCORENRN_ENABLE_CUDA_UNIFIED_MEMORY=OFF \
         -DCMAKE_INSTALL_PREFIX=$WORKSPACE/install_${CORENRN_TYPE}/ \
@@ -38,11 +33,6 @@ if [ "${CORENRN_TYPE}" = "GPU-non-unified" ]; then
         $WORKSPACE/
 elif [ "${CORENRN_TYPE}" = "GPU-unified" ]; then
     cmake \
-        -DCMAKE_C_FLAGS:STRING="-O2" \
-        -DCMAKE_CXX_FLAGS:STRING="-O2 -D_GLIBCXX_USE_CXX11_ABI=0 -DR123_USE_SSE=0" \
-        -DCOMPILE_LIBRARY_TYPE=STATIC  \
-        -DCUDA_HOST_COMPILER=`which gcc` \
-        -DCUDA_PROPAGATE_HOST_FLAGS=OFF \
         -DCORENRN_ENABLE_GPU=ON \
         -DCORENRN_ENABLE_CUDA_UNIFIED_MEMORY=ON \
         -DCMAKE_INSTALL_PREFIX=$WORKSPACE/install_${CORENRN_TYPE}/ \