Merge pull request #11308 from wckzhang/delayedinitfix

accelerator/cuda: Account for possibility that cuda will never be ini…

Author: janjust

ompi/mca/pml/ob1/pml_ob1.c

@@ -197,7 +197,7 @@ int mca_pml_ob1_enable(bool enable)
                           mca_pml_ob1.free_list_inc,
                           NULL, 0, NULL, NULL, NULL);
 
-    mca_pml_ob1_accelerator_init();
+    mca_pml_ob1.accelerator_enabled = (0 == mca_pml_ob1_accelerator_init()) ? true : false;
 
     mca_pml_ob1.enabled = true;
 

ompi/mca/pml/ob1/pml_ob1.h

@@ -86,6 +86,8 @@ struct mca_pml_ob1_t {
     char* allocator_name;
     mca_allocator_base_module_t* allocator;
     unsigned int unexpected_limit;
+    /* Accelerator support initialized */
+    bool accelerator_enabled;
 };
 typedef struct mca_pml_ob1_t mca_pml_ob1_t;
 

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -363,7 +363,9 @@ int mca_pml_ob1_component_fini(void)
     OBJ_DESTRUCT(&mca_pml_ob1.lock);
     OBJ_DESTRUCT(&mca_pml_ob1.send_ranges);
 
-    mca_pml_ob1_accelerator_fini();
+    if (mca_pml_ob1.accelerator_enabled) {
+        mca_pml_ob1_accelerator_fini();
+    }
 
     if( NULL != mca_pml_ob1.allocator ) {
         (void)mca_pml_ob1.allocator->alc_finalize(mca_pml_ob1.allocator);

opal/datatype/opal_datatype_copy.c

@@ -55,9 +55,24 @@
         }                                                                                   \
     } while (0)
 
+static bool opal_datatype_is_accel(void *dest, const void *src) {
+    int dev_id;
+    uint64_t flags;
+    if (opal_accelerator.check_addr(dest, &dev_id, &flags)) {
+        return true;
+    }
+    if (opal_accelerator.check_addr(src, &dev_id, &flags)) {
+        return true;
+    }
+    return false;
+}
+
 static void *opal_datatype_accelerator_memcpy(void *dest, const void *src, size_t size)
 {
     int res;
+    if (!opal_datatype_is_accel(dest, src)) {
+        return memcpy(dest, src, size);
+    }
     res = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
                                   dest, src, size, MCA_ACCELERATOR_TRANSFER_UNSPEC);
     if (OPAL_SUCCESS != res) {
@@ -70,6 +85,9 @@ static void *opal_datatype_accelerator_memcpy(void *dest, const void *src, size_
 static void *opal_datatype_accelerator_memmove(void *dest, const void *src, size_t size)
 {
     int res;
+    if (!opal_datatype_is_accel(dest, src)) {
+        return memmove(dest, src, size);
+    }
     res = opal_accelerator.mem_move(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
                                     dest, src, size, MCA_ACCELERATOR_TRANSFER_UNSPEC);
     if (OPAL_SUCCESS != res) {

opal/mca/accelerator/cuda/accelerator_cuda.c

@@ -109,8 +109,12 @@ static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *
         *flags |= MCA_ACCELERATOR_FLAGS_UNIFIED_MEMORY;
     }
     if (CUDA_SUCCESS != result) {
-        /* Cannot identify, return an error */
-        return -1;
+        /* If cuda is not initialized, assume it is a host buffer. */
+        if (CUDA_ERROR_NOT_INITIALIZED == result) {
+            return 0;
+        } else {
+            return OPAL_ERROR;
+        }
     } else if (CU_MEMORYTYPE_HOST == mem_type) {
         /* Host memory, nothing to do here */
         return 0;
@@ -123,9 +127,12 @@ static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *
 #else /* OPAL_CUDA_GET_ATTRIBUTES */
     result = cuPointerGetAttribute(&mem_type, CU_POINTER_ATTRIBUTE_MEMORY_TYPE, dbuf);
     if (CUDA_SUCCESS != result) {
-        /* If we cannot determine it is device pointer,
-         * just assume it is not. */
-        return 0;
+        /* If cuda is not initialized, assume it is a host buffer. */
+        if (CUDA_ERROR_NOT_INITIALIZED == result) {
+            return 0;
+        } else {
+            return OPAL_ERROR;
+        }
     } else if (CU_MEMORYTYPE_HOST == mem_type) {
         /* Host memory, nothing to do here */
         return 0;