Check for GPU Direct RDMA and leave pinned turned off

Author: Rolf vandeVaart

opal/mca/btl/openib/btl_openib_component.c

@@ -2577,6 +2577,15 @@ btl_openib_component_init(int *num_btl_modules,
         opal_leave_pinned_pipeline = 0;
     }
 
+#if OPAL_CUDA_SUPPORT
+    if (mca_btl_openib_component.cuda_want_gdr && (0 == opal_leave_pinned)) {
+        opal_show_help("help-mpi-btl-openib.txt",
+                       "CUDA_gdr_and_nopinned", true,
+                       opal_process_info.nodename);
+        goto no_btls;
+    }
+#endif /* OPAL_CUDA_SUPPORT */
+
     index = mca_base_var_find("ompi", "btl", "openib", "max_inline_data");
     if (index >= 0) {
         if (OPAL_SUCCESS == mca_base_var_get_value(index, NULL, &source, NULL)) {

opal/mca/btl/openib/help-mpi-btl-openib.txt

@@ -692,3 +692,11 @@ You cannot have fork support and CUDA GPU Direct RDMA support on at the
 same time.  Please disable one of them.  Deactivating the openib BTL.
 
   Local host:              %s
+#
+[CUDA_gdr_and_nopinned]
+You requested to run with CUDA GPU Direct RDMA support but also with
+"leave pinned" turned off. This will result in very poor performance
+with CUDA GPU Direct RDMA. Either disable GPU Direct RDMA support or
+enable "leave pinned" support. Deactivating the openib BTL.
+
+  Local host:              %s