Merge pull request #1162 from abouteiller/bug/v2.x/mt_lazy_addproc

bug fix: race condition when multiple threads create the same endpoint simultaneously

Author: hppritcha

ompi/mca/bml/r2/bml_r2.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
+ * Copyright (c) 2004-2016 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -160,7 +160,6 @@ static mca_bml_base_endpoint_t *mca_bml_r2_allocate_endpoint (ompi_proc_t *proc)
     mca_bml_base_btl_array_reserve(&bml_endpoint->btl_rdma,  mca_bml_r2.num_btl_modules);
     bml_endpoint->btl_max_send_size = -1;
     bml_endpoint->btl_proc = proc;
-    proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] = bml_endpoint;
 
     bml_endpoint->btl_flags_or = 0;
     return bml_endpoint;
@@ -430,6 +429,12 @@ static int mca_bml_r2_add_proc (struct ompi_proc_t *proc)
     /* compute metrics for registered btls */
     mca_bml_r2_compute_endpoint_metrics (bml_endpoint);
 
+    /* do it last, for the lazy initialization check in bml_base_get* */
+#if OPAL_ENABLE_THREAD_MULTI
+    opal_atomic_wmb();
+#endif /* OPAL_ENABLE_THREAD_MULTI */
+    proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] = bml_endpoint;
+
     return OMPI_SUCCESS;
 }
 
@@ -523,6 +528,7 @@ static int mca_bml_r2_add_procs( size_t nprocs,
 
             if (NULL == bml_endpoint) {
                 bml_endpoint = mca_bml_r2_allocate_endpoint (proc);
+                proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_BML] = bml_endpoint;
                 if (NULL == bml_endpoint) {
                     free(btl_endpoints);
                     free(new_procs);

ompi/mca/pml/ob1/pml_ob1_comm.h

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2016 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -73,10 +73,11 @@ static inline mca_pml_ob1_comm_proc_t *mca_pml_ob1_peer_lookup (struct ompi_comm
     if (OPAL_UNLIKELY(NULL == pml_comm->procs[rank])) {
         OPAL_THREAD_LOCK(&pml_comm->proc_lock);
         if (NULL == pml_comm->procs[rank]) {
-            pml_comm->procs[rank] = OBJ_NEW(mca_pml_ob1_comm_proc_t);
-            pml_comm->procs[rank]->ompi_proc = ompi_comm_peer_lookup (comm, rank);
-            OBJ_RETAIN(pml_comm->procs[rank]->ompi_proc);
+            mca_pml_ob1_comm_proc_t* proc = OBJ_NEW(mca_pml_ob1_comm_proc_t);
+            proc->ompi_proc = ompi_comm_peer_lookup (comm, rank);
+            OBJ_RETAIN(proc->ompi_proc);
             opal_atomic_wmb ();
+            pml_comm->procs[rank] = proc;
         }
         OPAL_THREAD_UNLOCK(&pml_comm->proc_lock);
     }