Consistently use the request array for all modules (single array stored

in the base).
Correctly deal with persistent requests (they must be always freed when
they are stored in the request array associated with the communicator).
Always use MPI_STATUS_IGNORE for single request waiting functions.

(cherry picked from commit 88492a1)

Author: bosilca

ompi/mca/coll/base/base.h

@@ -23,7 +23,7 @@
  * These functions are normally invoked by the back-ends of:
  *
  * - The back-ends of MPI_Init() and MPI_Finalize()
- * - Communuicactor constructors (e.g., MPI_Comm_split()) and
+ * - Communicator constructors (e.g., MPI_Comm_split()) and
  *   destructors (e.g., MPI_Comm_free())
  * - The laminfo command
  */

ompi/mca/coll/base/coll_base_alltoall.c

@@ -652,9 +652,9 @@ int ompi_coll_base_alltoall_intra_basic_linear(const void *sbuf, int scount,
     if( MPI_SUCCESS != err ) {
         OPAL_OUTPUT( (ompi_coll_base_framework.framework_output,"%s:%4d\tError occurred %d, rank %2d",
                       __FILE__, line, err, rank) );
-        /* Free the reqs */
-        ompi_coll_base_free_reqs(req, nreqs);
     }
+    /* Free the reqs in all cases as they are persistent requests */
+    ompi_coll_base_free_reqs(req, nreqs);
 
     /* All done */
     return err;

ompi/mca/coll/base/coll_base_alltoallv.c

@@ -288,9 +288,8 @@ ompi_coll_base_alltoallv_intra_basic_linear(const void *sbuf, const int *scounts
     err = ompi_request_wait_all(nreqs, data->mcct_reqs,
                                 MPI_STATUSES_IGNORE);
  err_hndl:
-    if( MPI_SUCCESS != err ) {  /* Free the requests. */
-        ompi_coll_base_free_reqs(data->mcct_reqs, nreqs);
-    }
+    /* Free the requests in all cases as they are persistent */
+    ompi_coll_base_free_reqs(data->mcct_reqs, nreqs);
 
     return err;
 }

ompi/mca/coll/base/coll_base_bcast.c

@@ -143,7 +143,7 @@ ompi_coll_base_bcast_intra_generic( void* buffer,
 
             /* wait for and forward the previous segment to children */
             err = ompi_request_wait( &recv_reqs[req_index ^ 0x1],
-                                     MPI_STATUSES_IGNORE );
+                                     MPI_STATUS_IGNORE );
             if (err != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
 
             for( i = 0; i < tree->tree_nextsize; i++ ) {
@@ -175,7 +175,7 @@ ompi_coll_base_bcast_intra_generic( void* buffer,
         }
 
         /* Process the last segment */
-        err = ompi_request_wait( &recv_reqs[req_index], MPI_STATUSES_IGNORE );
+        err = ompi_request_wait( &recv_reqs[req_index], MPI_STATUS_IGNORE );
         if (err != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
         sendcount = original_count - (ptrdiff_t)(num_segments - 1) * count_by_segment;
         for( i = 0; i < tree->tree_nextsize; i++ ) {
@@ -240,8 +240,11 @@ ompi_coll_base_bcast_intra_generic( void* buffer,
  error_hndl:
     OPAL_OUTPUT( (ompi_coll_base_framework.framework_output,"%s:%4d\tError occurred %d, rank %2d",
                   __FILE__, line, err, rank) );
-    if( (MPI_SUCCESS != err) && (NULL != send_reqs) ) {
-        ompi_coll_base_free_reqs( send_reqs, tree->tree_nextsize);
+    if( MPI_SUCCESS != err ) {
+        ompi_coll_base_free_reqs( recv_reqs, 2);
+        if( NULL != send_reqs ) {
+            ompi_coll_base_free_reqs( send_reqs, tree->tree_nextsize);
+        }
     }
 
     return err;
@@ -378,7 +381,6 @@ ompi_coll_base_bcast_intra_split_bintree ( void* buffer,
     ptrdiff_t type_extent, lb;
     ompi_request_t *base_req, *new_req;
     ompi_coll_tree_t *tree;
-    mca_coll_base_comm_t *data = module->base_data;
 
     size = ompi_comm_size(comm);
     rank = ompi_comm_rank(comm);
@@ -391,7 +393,7 @@ ompi_coll_base_bcast_intra_split_bintree ( void* buffer,
 
     /* setup the binary tree topology. */
     COLL_BASE_UPDATE_BINTREE( comm, module, root );
-    tree = data->cached_bintree;
+    tree = module->base_data->cached_bintree;
 
     err = ompi_datatype_type_size( datatype, &type_size );
 
@@ -501,8 +503,8 @@ ompi_coll_base_bcast_intra_split_bintree ( void* buffer,
                                       comm, &new_req));
             if (err != MPI_SUCCESS) { line = __LINE__; goto error_hndl; }
 
-            /* wait for and forward current segment */
-            err = ompi_request_wait_all( 1, &base_req, MPI_STATUSES_IGNORE );
+            /* wait for and forward the previous segment */
+            err = ompi_request_wait( &base_req, MPI_STATUS_IGNORE );
             for( i = 0; i < tree->tree_nextsize; i++ ) {  /* send data to children (segcount[lr]) */
                 err = MCA_PML_CALL(send( tmpbuf[lr], segcount[lr], datatype,
                                          tree->tree_next[i], MCA_COLL_BASE_TAG_BCAST,
@@ -517,7 +519,7 @@ ompi_coll_base_bcast_intra_split_bintree ( void* buffer,
         } /* end of for segindex */
 
         /* wait for the last segment and forward current segment */
-        err = ompi_request_wait_all( 1, &base_req, MPI_STATUSES_IGNORE );
+        err = ompi_request_wait( &base_req, MPI_STATUS_IGNORE );
         for( i = 0; i < tree->tree_nextsize; i++ ) {  /* send data to children */
             err = MCA_PML_CALL(send(tmpbuf[lr], sendcount[lr], datatype,
                                     tree->tree_next[i], MCA_COLL_BASE_TAG_BCAST,
@@ -633,10 +635,8 @@ ompi_coll_base_bcast_intra_basic_linear(void *buff, int count,
                                         mca_coll_base_module_t *module)
 {
     int i, size, rank, err;
-    mca_coll_base_comm_t *data = module->base_data;
     ompi_request_t **preq, **reqs;
 
-
     size = ompi_comm_size(comm);
     rank = ompi_comm_rank(comm);
 
@@ -651,24 +651,20 @@ ompi_coll_base_bcast_intra_basic_linear(void *buff, int count,
     }
 
     /* Root sends data to all others. */
-    preq = reqs = coll_base_comm_get_reqs(data, size-1);
+    preq = reqs = coll_base_comm_get_reqs(module->base_data, size-1);
     for (i = 0; i < size; ++i) {
         if (i == rank) {
             continue;
         }
 
-        err = MCA_PML_CALL(isend_init(buff, count, datatype, i,
-                                      MCA_COLL_BASE_TAG_BCAST,
-                                      MCA_PML_BASE_SEND_STANDARD,
-                                      comm, preq++));
+        err = MCA_PML_CALL(isend(buff, count, datatype, i,
+                                 MCA_COLL_BASE_TAG_BCAST,
+                                 MCA_PML_BASE_SEND_STANDARD,
+                                 comm, preq++));
         if (MPI_SUCCESS != err) { goto err_hndl; }
     }
     --i;
 
-    /* Start your engines.  This will never return an error. */
-
-    MCA_PML_CALL(start(i, reqs));
-
     /* Wait for them all.  If there's an error, note that we don't
      * care what the error was -- just that there *was* an error.  The
      * PML will finish all requests, even if one or more of them fail.

ompi/mca/coll/base/coll_base_frame.c

@@ -68,15 +68,7 @@ OBJ_CLASS_INSTANCE(mca_coll_base_module_t, opal_object_t,
 static void
 coll_base_comm_construct(mca_coll_base_comm_t *data)
 {
-    data->mcct_reqs = NULL;
-    data->mcct_num_reqs = 0;
-    data->cached_ntree = NULL;
-    data->cached_bintree = NULL;
-    data->cached_bmtree = NULL;
-    data->cached_in_order_bmtree = NULL;
-    data->cached_chain = NULL;
-    data->cached_pipeline = NULL;
-    data->cached_in_order_bintree = NULL;
+    memset ((char *) data + sizeof (data->super), 0, sizeof (*data) - sizeof (data->super));
 }
 
 static void

ompi/mca/coll/basic/coll_basic.h

@@ -285,29 +285,12 @@ BEGIN_C_DECLS
 
 struct mca_coll_basic_module_t {
     mca_coll_base_module_t super;
-
-    ompi_request_t **mccb_reqs;
-    int mccb_num_reqs;
 };
 typedef struct mca_coll_basic_module_t mca_coll_basic_module_t;
-OBJ_CLASS_DECLARATION(mca_coll_basic_module_t);
-
-/* Utility functions */
-
-static inline void mca_coll_basic_free_reqs(ompi_request_t ** reqs, int count)
-{
-    int i;
-    for (i = 0; i < count; ++i)
-        if( MPI_REQUEST_NULL != reqs[i] ) {
-            ompi_request_free(&reqs[i]);
-        }
-}
-
-/**
- * Return the array of requests on the data. If the array was not initialized
- * or if it's size was too small, allocate it to fit the requested size.
- */
-ompi_request_t** mca_coll_basic_get_reqs(mca_coll_basic_module_t* data, int nreqs);
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_basic_module_t);
+
+typedef mca_coll_base_comm_t mca_coll_basic_comm_t;
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_basic_comm_t);
 
 END_C_DECLS
 

ompi/mca/coll/basic/coll_basic_allgather.c

@@ -51,7 +51,6 @@ mca_coll_basic_allgather_inter(const void *sbuf, int scount,
     char *tmpbuf = NULL, *ptmp;
     ptrdiff_t rlb, slb, rextent, sextent, incr;
     ompi_request_t *req;
-    mca_coll_basic_module_t *basic_module = (mca_coll_basic_module_t*) module;
     ompi_request_t **reqs = NULL;
 
     rank = ompi_comm_rank(comm);
@@ -80,7 +79,7 @@ mca_coll_basic_allgather_inter(const void *sbuf, int scount,
         if (OMPI_SUCCESS != err) { line = __LINE__; goto exit; }
 
         /* Get a requests arrays of the right size */
-        reqs = mca_coll_basic_get_reqs(basic_module, rsize + 1);
+        reqs = coll_base_comm_get_reqs(module->base_data, rsize + 1);
         if( NULL == reqs ) { line = __LINE__; goto exit; }
 
         /* Do a send-recv between the two root procs. to avoid deadlock */
@@ -156,7 +155,7 @@ mca_coll_basic_allgather_inter(const void *sbuf, int scount,
     if( MPI_SUCCESS != err ) {
         OPAL_OUTPUT( (ompi_coll_base_framework.framework_output,"%s:%4d\tError occurred %d, rank %2d",
                       __FILE__, line, err, rank) );
-        if( NULL != reqs ) mca_coll_basic_free_reqs(reqs, rsize+1);
+        if( NULL != reqs ) ompi_coll_base_free_reqs(reqs, rsize+1);
     }
     if (NULL != tmpbuf) {
         free(tmpbuf);

ompi/mca/coll/basic/coll_basic_allreduce.c

@@ -85,7 +85,6 @@ mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, int count,
     ptrdiff_t true_lb, true_extent;
     char *tmpbuf = NULL, *pml_buffer = NULL;
     ompi_request_t *req[2];
-    mca_coll_basic_module_t *basic_module = (mca_coll_basic_module_t*) module;
     ompi_request_t **reqs = NULL;
 
     rank = ompi_comm_rank(comm);
@@ -114,7 +113,7 @@ mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, int count,
         if (NULL == tmpbuf) { err = OMPI_ERR_OUT_OF_RESOURCE; line = __LINE__; goto exit; }
         pml_buffer = tmpbuf - true_lb;
 
-        reqs = mca_coll_basic_get_reqs(basic_module, rsize - 1);
+        reqs = coll_base_comm_get_reqs(module->base_data, rsize - 1);
         if( NULL == reqs ) { err = OMPI_ERR_OUT_OF_RESOURCE; line = __LINE__; goto exit; }
 
         /* Do a send-recv between the two root procs. to avoid deadlock */
@@ -201,7 +200,7 @@ mca_coll_basic_allreduce_inter(const void *sbuf, void *rbuf, int count,
     if( MPI_SUCCESS != err ) {
         OPAL_OUTPUT((ompi_coll_base_framework.framework_output,"%s:%4d\tError occurred %d, rank %2d", __FILE__,
                      line, err, rank));
-        mca_coll_basic_free_reqs(reqs, rsize - 1);
+        ompi_coll_base_free_reqs(reqs, rsize - 1);
     }
     if (NULL != tmpbuf) {
         free(tmpbuf);

ompi/mca/coll/basic/coll_basic_alltoall.c

@@ -77,7 +77,7 @@ mca_coll_basic_alltoall_inter(const void *sbuf, int scount,
 
     /* Initiate all send/recv to/from others. */
     nreqs = size * 2;
-    req = rreq = mca_coll_basic_get_reqs( (mca_coll_basic_module_t*) module, nreqs);
+    req = rreq = coll_base_comm_get_reqs( module->base_data, nreqs);
     sreq = rreq + size;
 
     prcv = (char *) rbuf;
@@ -88,7 +88,7 @@ mca_coll_basic_alltoall_inter(const void *sbuf, int scount,
         err = MCA_PML_CALL(irecv(prcv + (i * rcvinc), rcount, rdtype, i,
                                  MCA_COLL_BASE_TAG_ALLTOALL, comm, rreq));
         if (OMPI_SUCCESS != err) {
-            mca_coll_basic_free_reqs(req, nreqs);
+            ompi_coll_base_free_reqs(req, nreqs);
             return err;
         }
     }
@@ -99,7 +99,7 @@ mca_coll_basic_alltoall_inter(const void *sbuf, int scount,
                                  MCA_COLL_BASE_TAG_ALLTOALL,
                                  MCA_PML_BASE_SEND_STANDARD, comm, sreq));
         if (OMPI_SUCCESS != err) {
-            mca_coll_basic_free_reqs(req, nreqs);
+            ompi_coll_base_free_reqs(req, nreqs);
             return err;
         }
     }
@@ -112,7 +112,7 @@ mca_coll_basic_alltoall_inter(const void *sbuf, int scount,
      * the error after we free everything. */
     err = ompi_request_wait_all(nreqs, req, MPI_STATUSES_IGNORE);
     if (OMPI_SUCCESS != err) {
-        mca_coll_basic_free_reqs(req, nreqs);
+        ompi_coll_base_free_reqs(req, nreqs);
     }
 
     /* All done */

ompi/mca/coll/basic/coll_basic_alltoallv.c

@@ -68,7 +68,7 @@ mca_coll_basic_alltoallv_inter(const void *sbuf, const int *scounts, const int *
 
     /* Initiate all send/recv to/from others. */
     nreqs = rsize * 2;
-    preq = mca_coll_basic_get_reqs((mca_coll_basic_module_t*) module, nreqs);
+    preq = coll_base_comm_get_reqs(module->base_data, nreqs);
 
     /* Post all receives first  */
     /* A simple optimization: do not send and recv msgs of length zero */
@@ -79,7 +79,7 @@ mca_coll_basic_alltoallv_inter(const void *sbuf, const int *scounts, const int *
                                      i, MCA_COLL_BASE_TAG_ALLTOALLV, comm,
                                      &preq[i]));
             if (MPI_SUCCESS != err) {
-                mca_coll_basic_free_reqs(preq, i);
+                ompi_coll_base_free_reqs(preq, i);
                 return err;
             }
         }
@@ -94,15 +94,15 @@ mca_coll_basic_alltoallv_inter(const void *sbuf, const int *scounts, const int *
                                      MCA_PML_BASE_SEND_STANDARD, comm,
                                      &preq[rsize + i]));
             if (MPI_SUCCESS != err) {
-                mca_coll_basic_free_reqs(preq, rsize + i);
+                ompi_coll_base_free_reqs(preq, rsize + i);
                 return err;
             }
         }
     }
 
     err = ompi_request_wait_all(nreqs, preq, MPI_STATUSES_IGNORE);
     if (MPI_SUCCESS != err) {
-        mca_coll_basic_free_reqs(preq, nreqs);
+        ompi_coll_base_free_reqs(preq, nreqs);
     }
 
     /* All done */