Random123 + OpenACC: do not compile as CUDA (#797)

* Random123 + OpenACC: compile as C++, not CUDA.
* Sprinkle magic attributes for nvc++/OpenMP.
* cuda_profile.cu: inline contents in other .cpp files.

Author: olupton

CMakeLists.txt

@@ -132,7 +132,7 @@ option(CORENRN_ENABLE_CALIPER_PROFILING "Enable Caliper instrumentation" OFF)
 option(CORENRN_ENABLE_LIKWID_PROFILING "Enable LIKWID instrumentation" OFF)
 option(CORENRN_ENABLE_CUDA_UNIFIED_MEMORY "Enable CUDA unified memory support" OFF)
 option(CORENRN_ENABLE_UNIT_TESTS "Enable unit tests execution" ON)
-option(CORENRN_ENABLE_GPU "Enable GPU support using OpenACC" OFF)
+option(CORENRN_ENABLE_GPU "Enable GPU support using OpenACC or OpenMP" OFF)
 option(CORENRN_ENABLE_SHARED "Enable shared library build" ON)
 option(CORENRN_ENABLE_LEGACY_UNITS "Enable legacy FARADAY, R, etc" OFF)
 option(CORENRN_ENABLE_PRCELLSTATE "Enable NRN_PRCELLSTATE debug feature" OFF)

coreneuron/CMakeLists.txt

@@ -31,7 +31,6 @@ file(
   "utils/*.cpp"
   "utils/*/*.c"
   "utils/*/*.cpp")
-file(GLOB_RECURSE CORENEURON_CUDA_FILES "*.cu")
 set(SCOPMATH_CODE_FILES
     "sim/scopmath/abort.cpp" "sim/scopmath/crout_thread.cpp" "sim/scopmath/newton_thread.cpp"
     "sim/scopmath/sparse_thread.cpp" "sim/scopmath/ssimplic_thread.cpp")
@@ -116,31 +115,15 @@ if(CORENRN_ENABLE_GPU)
       ${CMAKE_CURRENT_BINARY_DIR}/netstim.cpp
       ${CMAKE_CURRENT_BINARY_DIR}/netstim_inhpoisson.cpp
       ${CMAKE_CURRENT_BINARY_DIR}/pattern.cpp
-      ${CMAKE_CURRENT_SOURCE_DIR}/utils/randoms/nrnran123.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/io/nrn_setup.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/io/setup_fornetcon.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/io/corenrn_data_return.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/io/global_vars.cpp)
 
   set_source_files_properties(${OPENACC_EXCLUDED_FILES} PROPERTIES COMPILE_FLAGS
                                                                    "-DDISABLE_OPENACC")
-  # nrnran123.cpp is a symlink to nrnran123.cu, in GPU builds we compile this as CUDA code (so we
-  # want to remove the .cpp here), while in non-GPU builds we compile it as plain C++. Unfortunately
-  # CMake <v3.20 does not pass explicit -x <lang> options based on the LANGUAGE property
-  # (https://cmake.org/cmake/help/latest/policy/CMP0119.html), so using a single .cu file and
-  # setting LANGUAGE=CXX in non-GPU builds does not work.
-  list(APPEND CORENEURON_CODE_FILES ${CORENEURON_CUDA_FILES})
-  if(CORENRN_ACCELERATOR_OFFLOAD STREQUAL "OpenMP")
-    list(REMOVE_ITEM CORENEURON_CODE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/utils/randoms/nrnran123.cu")
-  elseif(CORENRN_ACCELERATOR_OFFLOAD STREQUAL "OpenACC")
-    list(REMOVE_ITEM CORENEURON_CODE_FILES
-         "${CMAKE_CURRENT_SOURCE_DIR}/utils/randoms/nrnran123.cpp")
-  else()
-    message(
-      FATAL_ERROR
-        "Don't know how to build Random123 compatibility layer for GPU with ${CORENRN_ACCELERATOR_OFFLOAD} offload."
-    )
-  endif()
+  # Only compile the explicit CUDA implementation of the Hines solver in GPU builds.
+  list(APPEND CORENEURON_CODE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/permute/cellorder.cu)
 
   # Eigen-3.5+ provides better GPU support. However, some functions cannot be called directly from
   # within an OpenACC region. Therefore, we need to wrap them in a special API (decorate them with

coreneuron/gpu/nrn_acc_manager.cpp

@@ -17,7 +17,6 @@
 #include "coreneuron/utils/profile/profiler_interface.h"
 #include "coreneuron/permute/cellorder.hpp"
 #include "coreneuron/permute/data_layout.hpp"
-#include "coreneuron/utils/profile/cuda_profile.h"
 #include "coreneuron/sim/scopmath/newton_struct.h"
 #include "coreneuron/coreneuron.hpp"
 #include "coreneuron/utils/nrnoc_aux.hpp"

coreneuron/utils/memory_utils.cpp

@@ -25,7 +25,6 @@
 #include <fstream>
 #include <unistd.h>
 #include "coreneuron/utils/memory_utils.h"
-#include "coreneuron/utils/profile/cuda_profile.h"
 #include "coreneuron/mpi/nrnmpi.h"
 #include "coreneuron/mpi/core/nrnmpi.hpp"
 #include "coreneuron/apps/corenrn_parameters.hpp"
@@ -36,6 +35,10 @@
 #include <malloc.h>
 #endif
 
+#ifdef CORENEURON_ENABLE_GPU
+#include "cuda_profiler_api.h"
+#endif
+
 namespace coreneuron {
 double nrn_mallinfo(void) {
     // -ve mem usage for non-supported platforms
@@ -108,7 +111,16 @@ void report_mem_usage(const char* message, bool all_ranks) {
                mem_avg);
 #ifdef CORENEURON_ENABLE_GPU
         if (corenrn_param.gpu) {
-            print_gpu_memory_usage();
+            size_t free_byte, total_byte;
+            cudaError_t cuda_status = cudaMemGetInfo(&free_byte, &total_byte);
+            if (cudaSuccess != cuda_status) {
+                std::printf("cudaMemGetInfo failed: %s\n", cudaGetErrorString(cuda_status));
+            }
+            constexpr double MiB{1. / (1024. * 1024.)};
+            std::printf(" GPU Memory (MiBs) : Used = %f, Free = %f, Total = %f\n",
+                        (total_byte - free_byte) * MiB,
+                        free_byte * MiB,
+                        total_byte * MiB);
         }
 #endif
     }