Merge pull request #5437 from kawashima-fj/pr/v3.0.x/pending-bsend-fix

v3.0.x: pml/ob1: Fix data corruption of MPI_BSEND

Author: hppritcha

ompi/mca/pml/ob1/pml_ob1_sendreq.c

@@ -18,6 +18,7 @@
  * Copyright (c) 2015      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -416,29 +417,31 @@ int mca_pml_ob1_send_request_start_buffered(
     des->des_cbfunc = mca_pml_ob1_rndv_completion;
     des->des_cbdata = sendreq;
 
-    /* buffer the remainder of the message */
-    rc = mca_pml_base_bsend_request_alloc((ompi_request_t*)sendreq);
-    if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) {
-        mca_bml_base_free(bml_btl, des);
-        return rc;
-    }
+    /* buffer the remainder of the message if it is not buffered yet */
+    if( OPAL_LIKELY(sendreq->req_send.req_addr == sendreq->req_send.req_base.req_addr) ) {
+        rc = mca_pml_base_bsend_request_alloc((ompi_request_t*)sendreq);
+        if( OPAL_UNLIKELY(OMPI_SUCCESS != rc) ) {
+            mca_bml_base_free(bml_btl, des);
+            return rc;
+        }
 
-    iov.iov_base = (IOVBASE_TYPE*)(((unsigned char*)sendreq->req_send.req_addr) + max_data);
-    iov.iov_len = max_data = sendreq->req_send.req_bytes_packed - max_data;
+        iov.iov_base = (IOVBASE_TYPE*)(((unsigned char*)sendreq->req_send.req_addr) + max_data);
+        iov.iov_len = max_data = sendreq->req_send.req_bytes_packed - max_data;
 
-    if((rc = opal_convertor_pack( &sendreq->req_send.req_base.req_convertor,
-                                  &iov,
-                                  &iov_count,
-                                  &max_data)) < 0) {
-        mca_bml_base_free(bml_btl, des);
-        return rc;
-    }
+        if((rc = opal_convertor_pack( &sendreq->req_send.req_base.req_convertor,
+                                      &iov,
+                                      &iov_count,
+                                      &max_data)) < 0) {
+            mca_bml_base_free(bml_btl, des);
+            return rc;
+        }
 
-    /* re-init convertor for packed data */
-    opal_convertor_prepare_for_send( &sendreq->req_send.req_base.req_convertor,
-                                     &(ompi_mpi_byte.dt.super),
-                                     sendreq->req_send.req_bytes_packed,
-                                     sendreq->req_send.req_addr );
+        /* re-init convertor for packed data */
+        opal_convertor_prepare_for_send( &sendreq->req_send.req_base.req_convertor,
+                                         &(ompi_mpi_byte.dt.super),
+                                         sendreq->req_send.req_bytes_packed,
+                                         sendreq->req_send.req_addr );
+    }
 
     /* wait for ack and completion */
     sendreq->req_state = 2;

ompi/mca/pml/ob1/pml_ob1_sendreq.h

@@ -14,6 +14,7 @@
  * Copyright (c) 2011-2012 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2011-2016 Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -468,6 +469,16 @@ mca_pml_ob1_send_request_start_seq (mca_pml_ob1_send_request_t* sendreq, mca_bml
         if( OPAL_LIKELY(OMPI_ERR_OUT_OF_RESOURCE != rc) )
             return rc;
     }
+    if(MCA_PML_BASE_SEND_BUFFERED == sendreq->req_send.req_send_mode &&
+       sendreq->req_send.req_addr == sendreq->req_send.req_base.req_addr) {
+        /* in the buffered mode, the send buffer must be saved to
+         * the attached buffer before returning it to the user */
+        int rc;
+        rc = mca_pml_base_bsend_request_start((ompi_request_t*)sendreq);
+        if(OMPI_SUCCESS != rc){
+            return rc;
+        }
+    }
     add_request_to_send_pending(sendreq, MCA_PML_OB1_SEND_PENDING_START, true);
 
     return OMPI_SUCCESS;