accelerator/rocm: updates to the component

edgargabriel · edgargabriel · commit 86bc10a8af35 · 2022-09-29T20:30:51.000Z
this pr removes the function table from the rocm component (and hence
the dlopen functionality), as well as the lock used during initialization
and shutdown. Some minor changes are further also required to configure
and Makefile logic.

Signed-off-by: Edgar Gabriel &lt;Edgar.Gabriel@amd.com&gt;
diff --git a/config/opal_check_rocm.m4 b/config/opal_check_rocm.m4
@@ -20,11 +20,12 @@ dnl
 #
 AC_DEFUN([OPAL_CHECK_ROCM],[
 
-     OPAL_VAR_SCOPE_PUSH([opal_check_rocm_happy rocm_save_CPPFLAGS rocm_save_LDFLAGS rocm_CPPFLAGS rocm_LDFLAGS])
+     OPAL_VAR_SCOPE_PUSH([opal_check_rocm_happy rocm_save_CPPFLAGS rocm_save_LDFLAGS rocm_save_LIBS rocm_CPPFLAGS rocm_LDFLAGS rocm_LIBS])
 
      rocm_save_CPPFLAGS="$CPPFLAGS"
      rocm_save_LDFLAGS="$LDFLAGS"
-     
+     rocm_save_LIBS="$LIBS"
+
      # Get some configuration information
      AC_ARG_WITH([rocm],
         [AS_HELP_STRING([--with-rocm(=DIR)],
@@ -51,7 +52,8 @@ AC_DEFUN([OPAL_CHECK_ROCM],[
 
      LDFLAGS="$rocm_save_LDFLAGS"
      OPAL_APPEND([CPPFLAGS], [${$1_CPPFLAGS}] )
-     
+     LIBS="$rocm_save_LIBS"
+
      AS_IF([ test "$opal_check_rocm_happy" = "no" ],
            [ CPPFLAGS="$rocm_save_CPPFLAGS"])
 
@@ -70,13 +72,3 @@ AC_DEFUN([OPAL_CHECK_ROCM],[
      AM_CONDITIONAL([OPAL_rocm_support], [test "$opal_check_rocm_happy" = "yes"])
      OPAL_VAR_SCOPE_POP
 ])
-
-AC_DEFUN([OPAL_CHECK_ROCM_AFTER_OPAL_DL],[
-    # We cannot have ROCm support without OPAL DL support.  Error out
-    # if the user wants Rocm but we do not have OPAL DL support.
-     AS_IF([test $OPAL_HAVE_DL_SUPPORT -eq 0 && test "$opal_check_rocm_happy" = "yes"],
-          [AC_MSG_WARN([--with-rocm was specified, but dlopen support is disabled.])
-           AC_MSG_WARN([You must reconfigure Open MPI with dlopen ("dl") support.])
-           AC_MSG_ERROR([Cannot continue.])])
-
-])
diff --git a/configure.ac b/configure.ac
@@ -1222,18 +1222,6 @@ fi
 # checkpoint results
 AC_CACHE_SAVE
 
-##################################
-# CUDA: part two
-##################################
-
-# This is somewhat gross to have a configure check for a common MCA
-# component outside of the normal MCA checks, but this check must come
-# after the opal DL MCA checks have done.  Someday this could perhaps
-# be done better by having some kind of "run this check at the end of
-# all other MCA checks" hook...?
-
-OPAL_CHECK_ROCM_AFTER_OPAL_DL
-
 ##################################
 # MPI Extended Interfaces
 ##################################
diff --git a/opal/mca/accelerator/rocm/Makefile.am b/opal/mca/accelerator/rocm/Makefile.am
@@ -35,9 +35,11 @@ endif
 mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_accelerator_rocm_la_SOURCES = $(sources)
-mca_accelerator_rocm_la_LDFLAGS = -module -avoid-version
-mca_accelerator_rocm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
+mca_accelerator_rocm_la_LDFLAGS = -module -avoid-version $(opal_rocm_LDFLAGS)
+mca_accelerator_rocm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+	$(opal_rocm_LIBS)
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_accelerator_rocm_la_SOURCES =$(sources)
-libmca_accelerator_rocm_la_LDFLAGS = -module -avoid-version
+libmca_accelerator_rocm_la_LDFLAGS = -module -avoid-version $(opal_rocm_LDFLAGS)
+libmca_accelerator_rocm_la_LIBADD = $(opal_rocm_LIBS)
diff --git a/opal/mca/accelerator/rocm/accelerator_rocm_component.c b/opal/mca/accelerator/rocm/accelerator_rocm_component.c
@@ -23,17 +23,12 @@
 #include "opal/runtime/opal_params.h"
 #include "accelerator_rocm.h"
 
-static struct opal_dl_handle_t* hip_handle = NULL;
-opal_accelerator_rocm_hipFunctionTable_t opal_accelerator_hip_funcs={};
-
 int opal_accelerator_rocm_memcpy_async = 1;
 int opal_accelerator_rocm_verbose = 0;
 size_t opal_accelerator_rocm_memcpyD2H_limit=1024;
 size_t opal_accelerator_rocm_memcpyH2D_limit=1048576;
 
 hipStream_t opal_accelerator_rocm_MemcpyStream = NULL;
-static opal_mutex_t opal_accelerator_rocm_init_lock = OPAL_MUTEX_STATIC_INIT;
-
 
 /*
  * Public string showing the accelerator rocm component version number
@@ -46,7 +41,7 @@ const char *opal_accelerator_rocm_component_version_string
 {                                                                            \
     hipError_t error = condition;                                            \
     if (hipSuccess != error){                                                \
-        const char* msg = HIP_FUNCS.hipGetErrorString(error);                \
+        const char* msg = hipGetErrorString(error);                \
         opal_output(0, "HIP error: %d %s file: %s line: %d\n", error, msg, __FILE__, __LINE__); \
         return error;                                                        \
     }                                                                        \
@@ -56,7 +51,7 @@ const char *opal_accelerator_rocm_component_version_string
 {                                                                            \
     hipError_t error = condition;                                            \
     if (hipSuccess != error){                                                \
-        const char* msg = HIP_FUNCS.hipGetErrorString(error);                \
+        const char* msg = hipGetErrorString(error);                \
         opal_output(0, "HIP error: %d %s file: %s line: %d\n", error, msg, __FILE__, __LINE__); \
         return NULL;                                                         \
     }                                                                        \
@@ -159,221 +154,6 @@ static int accelerator_rocm_component_register(void)
     return OPAL_SUCCESS;
 }
 
-
-static int hip_dl_init(void)
-{
-    char *str;
-    void *ptr;
-    int ret  = opal_dl_open("libamdhip64.so", false, false, &hip_handle, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output(0, "Unable to open libamdhip64.so\n");
-        return OPAL_ERROR;
-    }
-
-    ret = opal_dl_lookup(hip_handle, "hipMalloc", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output(0, "Failed to find hipMalloc\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMalloc = (hipMalloc_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipFree", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipFree\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipFree = (hipFree_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipMemcpy", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipMemcpy\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMemcpy = (hipMemcpy_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipMemcpyAsync", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipMemcpyAsync\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMemcpyAsync = (hipMemcpyAsync_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipMemcpy2D", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipMemcpy2D\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMemcpy2D = (hipMemcpy2D_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipMemcpy2DAsync", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipMemcpy2DAsync\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMemcpy2DAsync = (hipMemcpy2DAsync_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipMemGetAddressRange", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipMemGetAddressRange\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMemGetAddressRange  = (hipMemGetAddressRange_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipHostRegister", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipHostRegister\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipHostRegister = (hipHostRegister_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipHostUnregister", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipHostUnregister\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipHostUnregister = (hipHostUnregister_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipStreamCreate", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipStreamCreate\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipStreamCreate = (hipStreamCreate_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipStreamDestroy", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipStreamDestroy\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipStreamDestroy = (hipStreamDestroy_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipStreamSynchronize", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipStreamSynchronize\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipStreamSynchronize = (hipStreamSynchronize_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipGetErrorString", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipGetErrorString\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipGetErrorString = (hipGetErrorString_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipPointerGetAttributes", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipPointerGetAttributes\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipPointerGetAttributes = (hipPointerGetAttributes_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipEventCreateWithFlags", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipEventCreateWithFlags\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipEventCreateWithFlags = (hipEventCreateWithFlags_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipEventDestroy", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipEventDestroy\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipEventDestroy = (hipEventDestroy_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipEventRecord", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipEventRecord\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipEventRecord = (hipEventRecord_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipEventQuery", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipEventQuery\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipEventQuery = (hipEventQuery_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipEventSynchronize", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipEventSynchronize\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipEventSynchronize = (hipEventSynchronize_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipIpcGetMemHandle", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipIpcGetMemHandle\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipIpcGetMemHandle = (hipIpcGetMemHandle_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipIpcOpenMemHandle", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipIpcOpenMemHandle\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipIpcOpenMemHandle = (hipIpcOpenMemHandle_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipIpcCloseMemHandle", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipIpcCloseMemHandle\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipIpcCloseMemHandle = (hipIpcCloseMemHandle_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipGetDevice", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipGetDevice\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipGetDevice = (hipGetDevice_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipGetDeviceCount", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipGetDeviceCount\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipGetDeviceCount = (hipGetDeviceCount_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipDeviceCanAccessPeer", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipDeviceCanAccessPeer\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipDeviceCanAccessPeer = (hipDeviceCanAccessPeer_t)ptr;
-
-    return OPAL_SUCCESS;
-}
-
-
 static opal_accelerator_base_module_t* accelerator_rocm_init(void)
 {
     hipError_t err;
@@ -382,57 +162,35 @@ static opal_accelerator_base_module_t* accelerator_rocm_init(void)
         return NULL;
     }
 
-    OPAL_THREAD_LOCK(&opal_accelerator_rocm_init_lock);
-
-    if (opal_rocm_runtime_initialized) {
-        OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
-        return NULL;
-    }
-
-    if (OPAL_SUCCESS != hip_dl_init()) {
-        opal_output(0, "Could not open libamdhip64.so. Please check your LD_LIBRARY_PATH\n");
-        OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
-        return NULL;
-    }
-
     int count=0;
-    err = HIP_FUNCS.hipGetDeviceCount(&count);
+    err = hipGetDeviceCount(&count);
     if (hipSuccess != err || 0 == count) {
         opal_output(0, "No HIP capabale device found. Disabling component.\n");
-        OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
         return NULL;
     }
 
-    err = HIP_FUNCS.hipStreamCreate(&opal_accelerator_rocm_MemcpyStream);
+    err = hipStreamCreate(&opal_accelerator_rocm_MemcpyStream);
     if (hipSuccess != err) {
         opal_output(0, "Could not create hipStream, err=%d %s\n",
-                err, HIP_FUNCS.hipGetErrorString(err));
-        OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
+                err, hipGetErrorString(err));
         return NULL;
     }
 
     opal_atomic_mb();
     opal_rocm_runtime_initialized = true;
-    OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
 
     return &opal_accelerator_rocm_module;
 }
 
 static void accelerator_rocm_finalize(opal_accelerator_base_module_t* module)
 {
-    OPAL_THREAD_LOCK(&opal_accelerator_rocm_init_lock);
     if (NULL != (void*)opal_accelerator_rocm_MemcpyStream) {
-        hipError_t err = HIP_FUNCS.hipStreamDestroy(opal_accelerator_rocm_MemcpyStream);
+        hipError_t err = hipStreamDestroy(opal_accelerator_rocm_MemcpyStream);
         if (hipSuccess != err) {
             opal_output_verbose(10, 0, "hip_dl_finalize: error while destroying the hipStream\n");
         }
         opal_accelerator_rocm_MemcpyStream = NULL;
     }
-    if (NULL != hip_handle) {
-        opal_dl_close(hip_handle);
-        hip_handle = NULL;
-    }
-    OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
 
     return;
 }
diff --git a/opal/mca/accelerator/rocm/accelerator_rocm_module.c b/opal/mca/accelerator/rocm/accelerator_rocm_module.c
diff --git a/opal/mca/accelerator/rocm/configure.m4 b/opal/mca/accelerator/rocm/configure.m4
