Reenable HIP support (#250)

* Add hip support

* Add security when both Cuda and hip are selected

Author: acitrain

cmake/CheckOptions.cmake

@@ -3,8 +3,14 @@ if(USE_KOKKOS AND USE_VECTOR)
     message(FATAL_ERROR "Both Vector and Kokkos array implementations selected. Please only select one.")
 endif()
 
+
+if(ENABLE_CUDA AND ENABLE_HIP)
+  message(FATAL_ERROR "Both Cudo and Hip programming model are selected. Please only select one.")
+endif()
+
 # Set default if neither is selected
 if(NOT USE_KOKKOS AND NOT USE_VECTOR)
     message(WARNING "No array implementation selected. Vector implementation chosen as default.")
     set(USE_VECTOR TRUE CACHE BOOL "Use Vector array implementation" FORCE)
 endif()
+

cmake/KokkosConfig.cmake

@@ -25,6 +25,13 @@ else()
     message(STATUS "Activating CUDA with Kokkos.")
   endif()
 
+  if(ENABLE_HIP)
+    set(Kokkos_ENABLE_HIP ON CACHE BOOL "" FORCE)
+    set(Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE OFF CACHE BOOL "" FORCE)
+    message(STATUS "Activating HIP with Kokkos.")
+  endif()
+
+
   if(CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
     message(STATUS "Activating Kokkos debug mode")
     set(Kokkos_ENABLE_DEBUG ON CACHE BOOL "" FORCE)

cmake/ProxyConfig.cmake

@@ -11,6 +11,7 @@ option(USE_VECTOR "Use vectors." OFF)
 option(USE_KOKKOS "Use KOKKOS to parallelise loops" OFF)
 option(USE_KOKKOS_TEAMS "use hierarchical parallelism in Kokkos" OFF)
 option(ENABLE_CUDA "Enable cuda compilation" OFF)
+option(ENABLE_HIP "Enable hip compilation" OFF)
 
 # Python wrapping
 option(ENABLE_PYWRAP "Enable python binding compilation with pybind11" OFF)

src/solver/fe/impl/common/include/sem_solver_impl.h

@@ -493,7 +493,8 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
   model_discretization_interface::gatherTransformData(elementNumber, mesh_local,
                                                       transformData);
 
-#ifdef __CUDACC__
+#if defined(__CUDACC__) || defined(__HIPCC__)
+
   struct CJPacked
   {
     float4 a;
@@ -554,7 +555,8 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
             float const v4 = lambda * Jp0 * Jr2 + mu * Jp2 * Jr0;
             float const v5 = lambda * Jp1 * Jr2 + mu * Jp2 * Jr1;
 
-#ifdef __CUDACC__
+#if defined(__CUDACC__) || defined(__HIPCC__)
+
             CJflat[idx].a = make_float4(v0, v1, v2, v3);
             CJflat[idx].b = make_float2(v4, v5);
 #else
@@ -570,7 +572,8 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
       },
       [&](int i, int j, float val, const int p, const int r) {
         int const idx = p * 3 + r;
-#ifdef __CUDACC__
+#if defined(__CUDACC__) || defined(__HIPCC__)
+
         float3 const u_local = make_float3(localFields[0][j], localFields[1][j],
                                            localFields[2][j]);
         float4 const a = CJflat[idx].a;
@@ -664,7 +667,8 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
   model_discretization_interface::gatherTransformData(elementNumber, mesh_local,
                                                       transformData);
 
-#ifdef __CUDACC__
+#if defined(__CUDACC__) || defined(__HIPCC__)
+
   struct CJPacked
   {
     float4 a;
@@ -740,7 +744,8 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
             float const v4 = c44 * p0r2 + c13 * p2r0;
             float const v5 = c44 * p1r2 + c13 * p2r1;
 
-#ifdef __CUDACC__
+#if defined(__CUDACC__) || defined(__HIPCC__)
+
             CJflat[idx].a = make_float4(v0, v1, v2, v3);
             CJflat[idx].b = make_float2(v4, v5);
 #else
@@ -756,7 +761,8 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
       },
       [&](int i, int j, float val, const int p, const int r) {
         int const idx = p * 3 + r;
-#ifdef __CUDACC__
+#if defined(__CUDACC__) || defined(__HIPCC__)
+
         float3 const u_local = make_float3(localFields[0][j], localFields[1][j],
                                            localFields[2][j]);
         float4 const a = CJflat[idx].a;
@@ -862,7 +868,8 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
       mesh_local.getCTensorOnElement(elementNumber, CTTI);
     }
 
-#ifdef __CUDACC__
+#if defined(__CUDACC__) || defined(__HIPCC__)
+
     struct CJPacked
     {
       float4 a;
@@ -939,7 +946,8 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
                          C13 * p1r1 + C12 * p1r2 + C34 * p2r0 + C33 * p2r1 +
                          C23 * p2r2;
 
-#ifdef __CUDACC__
+#if defined(__CUDACC__) || defined(__HIPCC__)
+
               CJflat[idx].a = make_float4(v0, v1, v2, v3);
               CJflat[idx].b = make_float2(v4, v5);
 #else
@@ -955,7 +963,8 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
         },
         [&](int i, int j, float val, const int p, const int r) {
           const int idx = p * 3 + r;
-#ifdef __CUDACC__
+#if defined(__CUDACC__) || defined(__HIPCC__)
+
           const float3 u_local = make_float3(
               localFields[0][j], localFields[1][j], localFields[2][j]);
           const float4 a = CJflat[idx].a;
@@ -1024,7 +1033,6 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
 
   if constexpr (PHYSICS == enums::physicType::kAcoustic)
   {
-    // ===== ACOUSTIC VERSION =====
     LOOPHEAD(mesh_local.getNumberOfNodes(), I)
     {
       if (mesh_local.isFreeSurface(I))
@@ -1072,7 +1080,6 @@ void SEMsolver<ORDER, INTEGRAL_TYPE, MESH_TYPE, IS_MODEL_ON_NODES,
   }
   else  // ELASTIC
   {
-    // ===== ELASTIC VERSION =====
     LOOPHEAD(mesh_local.getNumberOfNodes(), I)
     {
       if (mesh_local.isFreeSurface(I))

src/utils/include/data_type_kokkos.h

@@ -1,5 +1,6 @@
 #ifndef FUNTIDES_UTILS_INCLUDE_DATA_TYPE_KOKKOS_H_
 #define FUNTIDES_UTILS_INCLUDE_DATA_TYPE_KOKKOS_H_
+
 #include "Kokkos_Core_fwd.hpp"
 
 #ifdef ENABLE_HIP
@@ -8,22 +9,19 @@
 
 #include <Kokkos_Core.hpp>
 
-#ifdef ENABLE_CUDA
+#if defined(ENABLE_CUDA) || defined(ENABLE_HIP)
 #define MemSpace Kokkos::SharedSpace
 using Layout = Kokkos::LayoutLeft;
 #else
 #define MemSpace Kokkos::HostSpace
 using Layout = Kokkos::LayoutRight;
 #endif
-
 typedef Kokkos::View<int *, Layout, MemSpace> vectorInt;
 typedef Kokkos::View<float *, Layout, MemSpace> vectorReal;
 typedef Kokkos::View<double *, Layout, MemSpace> vectorDouble;
-
 typedef Kokkos::View<int **, Layout, MemSpace> arrayInt;
 typedef Kokkos::View<float **, Layout, MemSpace> arrayReal;
 typedef Kokkos::View<double **, Layout, MemSpace> arrayDouble;
-
 typedef Kokkos::View<int ***, Layout, MemSpace> array3DInt;
 typedef Kokkos::View<float ***, Layout, MemSpace> array3DReal;
 typedef Kokkos::View<double ***, Layout, MemSpace> array3DDouble;