Merge pull request #698 from rolfv/pr/smcuda-ipc-perf-fix

Force smcuda BTL to use CUDA IPC path for all GPU buffers where possible

Author: bosilca

opal/mca/btl/smcuda/btl_smcuda.c

@@ -885,6 +885,13 @@ int mca_btl_smcuda_sendi( struct mca_btl_base_module_t* btl,
     if (mca_common_cuda_enabled && (IPC_INIT == endpoint->ipcstate) && mca_btl_smcuda_component.use_cuda_ipc) {
         mca_btl_smcuda_send_cuda_ipc_request(btl, endpoint);
     }
+    /* We do not want to use this path when we have CUDA IPC support */
+    if ((convertor->flags & CONVERTOR_CUDA) && (IPC_ACKED == endpoint->ipcstate)) {
+        if (NULL != descriptor) {
+            *descriptor = mca_btl_smcuda_alloc(btl, endpoint, order, payload_size+header_size, flags);
+        }
+        return OPAL_ERR_RESOURCE_BUSY;
+    }
 #endif /* OPAL_CUDA_SUPPORT */
 
     /* this check should be unnecessary... turn into an assertion? */

opal/mca/btl/smcuda/btl_smcuda_component.c

@@ -190,6 +190,11 @@ static int smcuda_register(void)
     if (0 == mca_btl_smcuda.super.btl_cuda_max_send_size) {
         mca_btl_smcuda.super.btl_cuda_max_send_size = 128*1024;
     }
+    /* If user has not set the value, then set to magic number which will be converted to the minimum
+     * size needed to fit the PML header (see pml_ob1.c) */
+    if (0 == mca_btl_smcuda.super.btl_cuda_eager_limit) {
+        mca_btl_smcuda.super.btl_cuda_eager_limit = SIZE_MAX; /* magic number */
+    }
 #endif /* OPAL_CUDA_SUPPORT */
     return mca_btl_smcuda_component_verify();
 }