Merge pull request #12315 from edgargabriel/topic/accel-copy-type-fix

accelerator: use correct copy type if known

Author: edgargabriel

opal/datatype/opal_datatype_copy.c

@@ -58,7 +58,8 @@
 static void *opal_datatype_accelerator_memcpy(void *dest, const void *src, size_t size)
 {
     int res;
-    int dev_id;
+    int dst_type, dst_dev, src_type, src_dev;
+    int copy_type = MCA_ACCELERATOR_TRANSFER_DTOD;
     uint64_t flags;
     /* If accelerator check addr returns an error, we can only
      * assume it is a host buffer. If device buffer checking fails,
@@ -67,12 +68,19 @@ static void *opal_datatype_accelerator_memcpy(void *dest, const void *src, size_
      * and retries are also unlikely to succeed. We identify these
      * buffers as host buffers as attempting a memcpy would provide
      * a chance to succeed. */
-    if (0 >= opal_accelerator.check_addr(dest, &dev_id, &flags) &&
-        0 >= opal_accelerator.check_addr(src, &dev_id, &flags)) {
+    dst_type = opal_accelerator.check_addr(dest, &dst_dev, &flags);
+    src_type = opal_accelerator.check_addr(src, &src_dev, &flags);
+    if (0 >= dst_type && 0 >= src_type) {
         return memcpy(dest, src, size);
     }
-    res = opal_accelerator.mem_copy(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
-                                  dest, src, size, MCA_ACCELERATOR_TRANSFER_UNSPEC);
+    else if (0 >= dst_type && 0 < src_type) {
+	copy_type = MCA_ACCELERATOR_TRANSFER_DTOH;
+    }
+    else if (0 < dst_type && 0 >= dst_type) {
+	copy_type = MCA_ACCELERATOR_TRANSFER_HTOD;
+    }
+    res = opal_accelerator.mem_copy(dst_dev, src_dev,
+				    dest, src, size, copy_type);
     if (OPAL_SUCCESS != res) {
         opal_output(0, "Error in accelerator memcpy");
         abort();
@@ -83,7 +91,8 @@ static void *opal_datatype_accelerator_memcpy(void *dest, const void *src, size_
 static void *opal_datatype_accelerator_memmove(void *dest, const void *src, size_t size)
 {
     int res;
-    int dev_id;
+    int dst_type, dst_dev, src_type, src_dev;
+    int copy_type = MCA_ACCELERATOR_TRANSFER_DTOD;
     uint64_t flags;
     /* If accelerator check addr returns an error, we can only
      * assume it is a host buffer. If device buffer checking fails,
@@ -92,12 +101,19 @@ static void *opal_datatype_accelerator_memmove(void *dest, const void *src, size
      * and retries are also unlikely to succeed. We identify these
      * buffers as host buffers as attempting a memmove would provide
      * a chance to succeed. */
-    if (0 >= opal_accelerator.check_addr(dest, &dev_id, &flags) &&
-        0 >= opal_accelerator.check_addr(src, &dev_id, &flags)) {
+    dst_type = opal_accelerator.check_addr(dest, &dst_dev, &flags);
+    src_type = opal_accelerator.check_addr(src, &src_dev, &flags);
+    if (0 >= dst_type && 0 >= src_type) {
         return memmove(dest, src, size);
     }
-    res = opal_accelerator.mem_move(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
-                                    dest, src, size, MCA_ACCELERATOR_TRANSFER_UNSPEC);
+    else if (0 >= dst_type && 0 < src_type) {
+	copy_type = MCA_ACCELERATOR_TRANSFER_DTOH;
+    }
+    else if (0 < dst_type && 0 >= dst_type) {
+	copy_type = MCA_ACCELERATOR_TRANSFER_HTOD;
+    }
+    res = opal_accelerator.mem_move(dst_dev, src_dev,
+                                    dest, src, size, copy_type);
     if (OPAL_SUCCESS != res) {
         opal_output(0, "Error in accelerator memmove");
         abort();

opal/mca/accelerator/rocm/accelerator_rocm_module.c

@@ -334,7 +334,9 @@ static int mca_accelerator_rocm_memcpy(int dest_dev_id, int src_dev_id, void *de
         return OPAL_SUCCESS;
     }
 
-    if (type == MCA_ACCELERATOR_TRANSFER_DTOH && size <= opal_accelerator_rocm_memcpyD2H_limit) {
+    if ((type == MCA_ACCELERATOR_TRANSFER_DTOH ||
+	 type == MCA_ACCELERATOR_TRANSFER_UNSPEC) &&
+	size <= opal_accelerator_rocm_memcpyD2H_limit) {
         memcpy(dest, src, size);
         return OPAL_SUCCESS;
     }

opal/mca/btl/smcuda/btl_smcuda_accelerator.c

@@ -65,7 +65,7 @@ int mca_btl_smcuda_accelerator_init(void)
     }
     /* Create the events since they can be reused. */
     for (i = 0; i < accelerator_event_max; i++) {
-        rc = opal_accelerator.create_event(MCA_ACCELERATOR_NO_DEVICE_ID, &accelerator_event_ipc_array[i], true);
+        rc = opal_accelerator.create_event(MCA_ACCELERATOR_NO_DEVICE_ID, &accelerator_event_ipc_array[i], opal_accelerator_use_sync_memops ? false : true);
         if (OPAL_SUCCESS != rc) {
             opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output, "Accelerator create event failed.");
             rc = OPAL_ERROR;
@@ -215,7 +215,7 @@ int mca_btl_smcuda_memcpy(void *dst, void *src, size_t amount, char *msg,
     }
 
     result = opal_accelerator.mem_copy_async(MCA_ACCELERATOR_NO_DEVICE_ID, MCA_ACCELERATOR_NO_DEVICE_ID,
-                                             dst, src, amount, ipc_stream, MCA_ACCELERATOR_TRANSFER_UNSPEC);
+                                             dst, src, amount, ipc_stream, MCA_ACCELERATOR_TRANSFER_DTOD);
     if (OPAL_UNLIKELY(OPAL_SUCCESS != result)) {
         opal_output_verbose(1, mca_btl_smcuda_component.cuda_ipc_output, "smcuda: memcpy async failed: %d",
                             result);