pml/ob1: do not cache leave_pinned

This commit fixes a bug that disabled both the RDMA pipeline and RDMA
protocols in ob1. ob1 was internally caching the values of
opal_leave_pinned and opal_leave_pinned_pipeline at init time. This is
no longer valid as opal_leave_pinned may be set by any call to a btl's
add_procs.

(cherry picked from commit 3e7ef48)

Signed-off-by: Nathan Hjelm <[email protected]>
Signed-off-by: Jeff Squyres <[email protected]>

Author: hjelmn

ompi/mca/pml/ob1/pml_ob1.h

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved
  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
- * Copyright (c) 2012-2016 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -60,9 +60,7 @@ struct mca_pml_ob1_t {
     size_t rdma_retries_limit;
     int max_rdma_per_request;
     int max_send_per_range;
-    bool leave_pinned;
     bool use_all_rdma;
-    int leave_pinned_pipeline;
 
     /* lock queue access */
     opal_mutex_t lock;

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2007-2010 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved
- * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2013-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -298,12 +298,6 @@ mca_pml_ob1_component_init( int* priority,
 
     }
 
-    /* Set this here (vs in component_open()) because
-       opal_leave_pinned* may have been set after MCA params were
-       read (e.g., by the openib btl) */
-    mca_pml_ob1.leave_pinned = (1 == opal_leave_pinned);
-    mca_pml_ob1.leave_pinned_pipeline = (int) opal_leave_pinned_pipeline;
-
     return &mca_pml_ob1.super;
 }
 

ompi/mca/pml/ob1/pml_ob1_cuda.c

@@ -13,7 +13,7 @@
  * Copyright (c) 2008      UT-Battelle, LLC. All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2012-2015 NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -25,6 +25,7 @@
 
 #include "ompi_config.h"
 #include "opal/prefetch.h"
+#include "opal/runtime/opal_params.h"
 #include "opal/mca/btl/btl.h"
 #include "opal/mca/mpool/mpool.h"
 #include "ompi/constants.h"
@@ -143,7 +144,7 @@ size_t mca_pml_ob1_rdma_cuda_btls(
     /* if we don't use leave_pinned and all BTLs that already have this memory
      * registered amount to less then half of available bandwidth - fall back to
      * pipeline protocol */
-    if(0 == num_btls_used || (!mca_pml_ob1.leave_pinned && weight_total < 0.5))
+    if(0 == num_btls_used || (!opal_leave_pinned && weight_total < 0.5))
         return 0;
 
     mca_pml_ob1_calc_weighted_length(rdma_btls, num_btls_used, size,

ompi/mca/pml/ob1/pml_ob1_rdma.c

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -27,6 +27,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/bml/bml.h"
 #include "opal/mca/mpool/mpool.h"
+#include "opal/runtime/opal_params.h"
 #include "pml_ob1.h"
 #include "pml_ob1_rdma.h"
 
@@ -79,7 +80,7 @@ size_t mca_pml_ob1_rdma_btls(
             /* do not use the RDMA protocol with this btl if 1) leave pinned is disabled,
              * 2) the btl supports put, and 3) the fragment is larger than the minimum
              * pipeline size specified by the BTL */
-            if (!mca_pml_ob1.leave_pinned && (btl->btl_flags & MCA_BTL_FLAGS_PUT) &&
+            if (!opal_leave_pinned && (btl->btl_flags & MCA_BTL_FLAGS_PUT) &&
                   size > btl->btl_min_rdma_pipeline_size) {
                 continue;
             }
@@ -102,7 +103,7 @@ size_t mca_pml_ob1_rdma_btls(
     /* if we don't use leave_pinned and all BTLs that already have this memory
      * registered amount to less then half of available bandwidth - fall back to
      * pipeline protocol */
-    if (0 == num_btls_used || (!mca_pml_ob1.leave_pinned && weight_total < 0.5))
+    if (0 == num_btls_used || (!opal_leave_pinned && weight_total < 0.5))
         return 0;
 
     mca_pml_ob1_calc_weighted_length(rdma_btls, num_btls_used, size,