Merge pull request #3712 from gpaulsen/rel_v3.0.x_pr3500_pr3635

Rel v3.0.x pr3500 pr3635

Author: hppritcha

ompi/mpi/c/sendrecv_replace.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2010 The University of Tennessee and The University
+ * Copyright (c) 2004-2017 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -12,6 +12,7 @@
  * Copyright (c) 2010-2012 Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -48,10 +49,10 @@ int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype,
     int rc = MPI_SUCCESS;
 
     MEMCHECKER(
-        memchecker_datatype(datatype);
-        memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
-        memchecker_comm(comm);
-    );
+               memchecker_datatype(datatype);
+               memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);
+               memchecker_comm(comm);
+               );
 
     if ( MPI_PARAM_CHECK ) {
         rc = MPI_SUCCESS;
@@ -76,68 +77,68 @@ int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype,
 
     /* simple case */
     if ( source == MPI_PROC_NULL || dest == MPI_PROC_NULL || count == 0 ) {
-        rc = PMPI_Sendrecv(buf,count,datatype,dest,sendtag,buf,count,datatype,source,recvtag,comm,status);
+        rc = PMPI_Sendrecv(buf, count, datatype, dest, sendtag, buf, count, datatype, source, recvtag, comm, status);
 
         OPAL_CR_EXIT_LIBRARY();
         return rc;
-    } else {
-
-        opal_convertor_t convertor;
-        struct iovec iov;
-        unsigned char recv_data[2048];
-        size_t packed_size, max_data;
-        uint32_t iov_count;
-        ompi_status_public_t recv_status;
-        ompi_proc_t* proc = ompi_comm_peer_lookup(comm,source);
-        if(proc == NULL) {
-            rc = MPI_ERR_RANK;
-            OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-        }
-
-        /* initialize convertor to unpack recv buffer */
-        OBJ_CONSTRUCT(&convertor, opal_convertor_t);
-        opal_convertor_copy_and_prepare_for_recv( proc->super.proc_convertor, &(datatype->super),
-                                                  count, buf, 0, &convertor );
-
-        /* setup a buffer for recv */
-        opal_convertor_get_packed_size( &convertor, &packed_size );
-        if( packed_size > sizeof(recv_data) ) {
-            rc = PMPI_Alloc_mem(packed_size, MPI_INFO_NULL, &iov.iov_base);
-            if(OMPI_SUCCESS != rc) {
-                OMPI_ERRHANDLER_RETURN(OMPI_ERR_OUT_OF_RESOURCE, comm, MPI_ERR_BUFFER, FUNC_NAME);
-            }
-        } else {
-            iov.iov_base = (caddr_t)recv_data;
-        }
-
-        /* recv into temporary buffer */
-        rc = PMPI_Sendrecv( buf, count, datatype, dest, sendtag, iov.iov_base, packed_size,
-                           MPI_BYTE, source, recvtag, comm, &recv_status );
-        if (rc != MPI_SUCCESS) {
-            if(packed_size > sizeof(recv_data))
-                PMPI_Free_mem(iov.iov_base);
-            OBJ_DESTRUCT(&convertor);
-            OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
-        }
-
-        /* unpack into users buffer */
-        iov.iov_len = recv_status._ucount;
-        iov_count = 1;
-        max_data = recv_status._ucount;
-        opal_convertor_unpack(&convertor, &iov, &iov_count, &max_data );
+    }
 
-        /* return status to user */
-        if(status != MPI_STATUS_IGNORE) {
-            *status = recv_status;
-        }
+    /**
+     * If we look for an optimal solution, then we should receive the data into a temporary buffer
+     * and once the send completes we would unpack back into the original buffer. However, if the
+     * sender is unknown, this approach can only be implementing by receiving with the recv datatype
+     * (potentially non-contiguous) and thus the allocated memory will be larger than the size of the
+     * datatype. A simpler, but potentially less efficient approach is to work on the data we have
+     * control of, aka the sent data, and pack it into a contiguous buffer before posting the receive.
+     * Once the send completes, we free it.
+     */
+    opal_convertor_t convertor;
+    unsigned char packed_data[2048];
+    struct iovec iov = { .iov_base = packed_data, .iov_len = sizeof(packed_data) };
+    size_t packed_size, max_data;
+    uint32_t iov_count;
+    ompi_status_public_t recv_status;
+    ompi_proc_t* proc = ompi_comm_peer_lookup(comm, dest);
+    if(proc == NULL) {
+        rc = MPI_ERR_RANK;
+        OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+    }
 
-        /* release resources */
-        if(packed_size > sizeof(recv_data)) {
-            PMPI_Free_mem(iov.iov_base);
+    /* initialize convertor to unpack recv buffer */
+    OBJ_CONSTRUCT(&convertor, opal_convertor_t);
+    opal_convertor_copy_and_prepare_for_send( proc->super.proc_convertor, &(datatype->super),
+                                              count, buf, 0, &convertor );
+
+    /* setup a buffer for recv */
+    opal_convertor_get_packed_size( &convertor, &packed_size );
+    if( packed_size > sizeof(packed_data) ) {
+        rc = PMPI_Alloc_mem(packed_size, MPI_INFO_NULL, &iov.iov_base);
+        if(OMPI_SUCCESS != rc) {
+            rc = OMPI_ERR_OUT_OF_RESOURCE;
+            goto cleanup_and_return;
         }
-        OBJ_DESTRUCT(&convertor);
+        iov.iov_len = packed_size;
+    }
+    max_data = packed_size;
+    iov_count = 1;
+    rc = opal_convertor_pack(&convertor, &iov, &iov_count, &max_data);
+    
+    /* recv into temporary buffer */
+    rc = PMPI_Sendrecv( iov.iov_base, packed_size, MPI_PACKED, dest, sendtag, buf, count,
+                        datatype, source, recvtag, comm, &recv_status );
+
+ cleanup_and_return:
+    /* return status to user */
+    if(status != MPI_STATUS_IGNORE) {
+        *status = recv_status;
+    }
 
-        OPAL_CR_EXIT_LIBRARY();
-        return MPI_SUCCESS;
+    /* release resources */
+    if(packed_size > sizeof(packed_data)) {
+        PMPI_Free_mem(iov.iov_base);
     }
+    OBJ_DESTRUCT(&convertor);
+
+    OPAL_CR_EXIT_LIBRARY();
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
 }