Merge pull request open-mpi#625 from hppritcha/topic/patch_pml_ob1_isend

Topic/patch pml ob1 isend

Author: jsquyres

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -292,6 +292,18 @@ int mca_pml_ob1_component_fini(void)
         return OMPI_SUCCESS; /* never selected.. return success.. */
     mca_pml_ob1.enabled = false;  /* not anymore */
 
+    /* return the static receive/send requests to the respective free list and
+     * let the free list handle destruction. */
+    if( NULL != mca_pml_ob1_recvreq ) {
+        opal_free_list_return (&mca_pml_base_recv_requests, (opal_free_list_item_t *) mca_pml_ob1_recvreq);
+        mca_pml_ob1_recvreq = NULL;
+    }
+
+    if( NULL != mca_pml_ob1_sendreq ) {
+        opal_free_list_return (&mca_pml_base_send_requests, (opal_free_list_item_t *) mca_pml_ob1_sendreq);
+        mca_pml_ob1_sendreq = NULL;
+    }
+
     OBJ_DESTRUCT(&mca_pml_ob1.rdma_pending);
     OBJ_DESTRUCT(&mca_pml_ob1.pckt_pending);
     OBJ_DESTRUCT(&mca_pml_ob1.recv_pending);
@@ -304,15 +316,6 @@ int mca_pml_ob1_component_fini(void)
     OBJ_DESTRUCT(&mca_pml_ob1.lock);
     OBJ_DESTRUCT(&mca_pml_ob1.send_ranges);
 
-    if( NULL != mca_pml_ob1_recvreq ) {
-        OBJ_DESTRUCT(mca_pml_ob1_recvreq);
-        mca_pml_ob1_recvreq = NULL;
-    }
-    if( NULL != mca_pml_ob1_sendreq ) {
-        OBJ_DESTRUCT(mca_pml_ob1_sendreq);
-        mca_pml_ob1_sendreq = NULL;
-    }
-
     if( NULL != mca_pml_ob1.allocator ) {
         (void)mca_pml_ob1.allocator->alc_finalize(mca_pml_ob1.allocator);
         mca_pml_ob1.allocator = NULL;

ompi/mca/pml/ob1/pml_ob1_irecv.c

@@ -3,14 +3,14 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2014 The University of Tennessee and The University
+ * Copyright (c) 2004-2015 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2014 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2007-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2010-2012 Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
@@ -28,10 +28,14 @@
 #include "pml_ob1_recvfrag.h"
 #include "ompi/peruse/peruse-internal.h"
 #include "ompi/message/message.h"
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif  /* HAVE_ALLOCA_H */
 
+/**
+ * Single usage request. As we allow recursive calls to recv
+ * (from the request completion callback), we cannot rely on
+ * using a global request. Thus, once a recv acquires ownership
+ * this global request, it should set it to NULL to prevent
+ * the reuse until the first user completes.
+ */
 mca_pml_ob1_recv_request_t *mca_pml_ob1_recvreq = NULL;
 
 int mca_pml_ob1_irecv_init(void *addr,
@@ -97,19 +101,16 @@ int mca_pml_ob1_recv(void *addr,
     mca_pml_ob1_recv_request_t *recvreq = NULL;
     int rc;
 
-#if !OPAL_ENABLE_MULTI_THREADS
+#if !OMPI_ENABLE_THREAD_MULTIPLE
     recvreq = mca_pml_ob1_recvreq;
+    mca_pml_ob1_recvreq = NULL;
     if( OPAL_UNLIKELY(NULL == recvreq) )
-#endif  /* !OPAL_ENABLE_MULTI_THREADS */
+#endif  /* !OMPI_ENABLE_THREAD_MULTIPLE */
         {
             MCA_PML_OB1_RECV_REQUEST_ALLOC(recvreq);
             if (NULL == recvreq)
                 return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
-#if !OPAL_ENABLE_MULTI_THREADS
-            mca_pml_ob1_recvreq = recvreq;
-#endif  /* !OPAL_ENABLE_MULTI_THREADS */
         }
-    OBJ_CONSTRUCT(recvreq, mca_pml_ob1_recv_request_t);
 
     MCA_PML_OB1_RECV_REQUEST_INIT(recvreq, addr, count, datatype,
                                   src, tag, comm, false);
@@ -126,8 +127,17 @@ int mca_pml_ob1_recv(void *addr,
     }
 
     rc = recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
-    MCA_PML_BASE_RECV_REQUEST_FINI(&recvreq->req_recv);
-    OBJ_DESTRUCT(recvreq);
+
+#if OMPI_ENABLE_THREAD_MULTIPLE
+    MCA_PML_OB1_RECV_REQUEST_RETURN(recvreq);
+#else
+    if( NULL != mca_pml_ob1_recvreq ) {
+        MCA_PML_OB1_RECV_REQUEST_RETURN(recvreq);
+    } else {
+        mca_pml_ob1_recv_request_fini (recvreq);
+        mca_pml_ob1_recvreq = recvreq;
+    }
+#endif
 
     return rc;
 }

ompi/mca/pml/ob1/pml_ob1_isend.c

@@ -28,10 +28,14 @@
 #include "pml_ob1_sendreq.h"
 #include "pml_ob1_recvreq.h"
 #include "ompi/peruse/peruse-internal.h"
-#if HAVE_ALLOCA_H
-#include <alloca.h>
-#endif  /* HAVE_ALLOCA_H */
 
+/**
+ * Single usage request. As we allow recursive calls (as an
+ * example from the request completion callback), we cannot rely
+ * on using a global request. Thus, once a send acquires ownership
+ * of this global request, it should set it to NULL to prevent
+ * the reuse until the first user completes.
+ */
 mca_pml_ob1_send_request_t *mca_pml_ob1_sendreq = NULL;
 
 int mca_pml_ob1_isend_init(const void *buf,
@@ -220,19 +224,16 @@ int mca_pml_ob1_send(const void *buf,
         }
     }
 
-#if !OPAL_ENABLE_MULTI_THREADS
+#if !OMPI_ENABLE_THREAD_MULTIPLE
     sendreq = mca_pml_ob1_sendreq;
+    mca_pml_ob1_sendreq = NULL;
     if( OPAL_UNLIKELY(NULL == sendreq) )
-#endif  /* !OPAL_ENABLE_MULTI_THREADS */
+#endif  /* !OMPI_ENABLE_THREAD_MULTIPLE */
         {
             MCA_PML_OB1_SEND_REQUEST_ALLOC(comm, dst, sendreq);
             if (NULL == sendreq)
                 return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
-#if !OPAL_ENABLE_MULTI_THREADS
-            mca_pml_ob1_sendreq = sendreq;
-#endif  /* !OPAL_ENABLE_MULTI_THREADS */
         }
-    OBJ_CONSTRUCT(sendreq, mca_pml_ob1_send_request_t);
     sendreq->req_send.req_base.req_proc = dst_proc;
     sendreq->rdma_frag = NULL;
 
@@ -252,9 +253,18 @@ int mca_pml_ob1_send(const void *buf,
         ompi_request_wait_completion(&sendreq->req_send.req_base.req_ompi);
 
         rc = sendreq->req_send.req_base.req_ompi.req_status.MPI_ERROR;
-        MCA_PML_BASE_SEND_REQUEST_FINI(&sendreq->req_send);
     }
-    OBJ_DESTRUCT(sendreq);
+
+#if OMPI_ENABLE_THREAD_MULTIPLE
+    MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq);
+#else
+    if( NULL != mca_pml_ob1_sendreq ) {
+        MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq);
+    } else {
+        mca_pml_ob1_send_request_fini (sendreq);
+        mca_pml_ob1_sendreq = sendreq;
+    }
+#endif
 
     return rc;
 }

ompi/mca/pml/ob1/pml_ob1_recvreq.h

@@ -128,16 +128,21 @@ do {                                                                \
         ompi_request_complete( &(recvreq->req_recv.req_base.req_ompi), true );        \
     } while (0)
 
+static inline void mca_pml_ob1_recv_request_fini (mca_pml_ob1_recv_request_t *recvreq)
+{
+    MCA_PML_BASE_RECV_REQUEST_FINI(&recvreq->req_recv);
+    if ((recvreq)->local_handle) {
+        mca_bml_base_deregister_mem (recvreq->rdma_bml, recvreq->local_handle);
+        recvreq->local_handle = NULL;
+    }
+}
+
 /*
  *  Free the PML receive request
  */
 #define MCA_PML_OB1_RECV_REQUEST_RETURN(recvreq)                        \
     {                                                                   \
-        MCA_PML_BASE_RECV_REQUEST_FINI(&(recvreq)->req_recv);           \
-        if ((recvreq)->local_handle) {                                  \
-            mca_bml_base_deregister_mem ((recvreq)->rdma_bml, (recvreq)->local_handle); \
-            (recvreq)->local_handle = NULL;                             \
-        }                                                               \
+        mca_pml_ob1_recv_request_fini (recvreq);                        \
         opal_free_list_return (&mca_pml_base_recv_requests,             \
                                (opal_free_list_item_t*)(recvreq));      \
     }

ompi/mca/pml/ob1/pml_ob1_sendreq.h

@@ -215,18 +215,23 @@ do {
                             &(sendreq->req_send.req_base), PERUSE_SEND);             \
 } while(0)
 
+static inline void mca_pml_ob1_send_request_fini (mca_pml_ob1_send_request_t *sendreq)
+{
+    /*  Let the base handle the reference counts */
+    MCA_PML_BASE_SEND_REQUEST_FINI((&(sendreq)->req_send));
+    if (sendreq->rdma_frag) {
+        MCA_PML_OB1_RDMA_FRAG_RETURN (sendreq->rdma_frag);
+        sendreq->rdma_frag = NULL;
+    }
+}
+
 /*
  * Release resources associated with a request
  */
 
 #define MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq)                        \
     do {                                                                \
-        /*  Let the base handle the reference counts */                 \
-        MCA_PML_BASE_SEND_REQUEST_FINI((&(sendreq)->req_send));         \
-        if (sendreq->rdma_frag) {                                       \
-            MCA_PML_OB1_RDMA_FRAG_RETURN (sendreq->rdma_frag);          \
-            sendreq->rdma_frag = NULL;                                  \
-        }                                                               \
+        mca_pml_ob1_send_request_fini (sendreq);                        \
         opal_free_list_return ( &mca_pml_base_send_requests,            \
                                 (opal_free_list_item_t*)sendreq);       \
     } while(0)