Fix MPI_Waitany and MPI_Waitsome

(request handling related)

(cherry picked from open-mpi/ompi@5417155)

Signed-off-by: Nathan Hjelm <[email protected]>

Author: artpol84

ompi/request/req_wait.c

@@ -15,6 +15,7 @@
  * Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2016      Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2016      Mellanox Technologies. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -116,7 +117,8 @@ int ompi_request_default_wait_any(size_t count,
         if (MPI_STATUS_IGNORE != status) {
             *status = ompi_status_empty;
         }
-        WAIT_SYNC_RELEASE(&sync);
+        /* No signal-in-flight can be in this case */
+        WAIT_SYNC_RELEASE_NOWAIT(&sync);
         return rc;
     }
 
@@ -140,6 +142,15 @@ int ompi_request_default_wait_any(size_t count,
             *index = i;
         }
     }
+    
+    if( *index == completed ){
+        /* Only one request has triggered. There was no
+         * in-flight completions.
+         * Drop the signalled flag so we won't block
+         * in WAIT_SYNC_RELEASE 
+         */
+        WAIT_SYNC_SIGNALLED(&sync);
+    }        
 
     request = requests[*index];
     assert( REQUEST_COMPLETE(request) );
@@ -348,7 +359,8 @@ int ompi_request_default_wait_some(size_t count,
     ompi_request_t **rptr = NULL;
     ompi_request_t *request = NULL;
     ompi_wait_sync_t sync;
-
+    size_t sync_sets = 0, sync_unsets = 0;
+    
     WAIT_SYNC_INIT(&sync, 1);
 
     *outcount = 0;
@@ -373,10 +385,12 @@ int ompi_request_default_wait_some(size_t count,
             num_requests_done++;
         }
     }
+    sync_sets = count - num_requests_null_inactive - num_requests_done;
 
     if(num_requests_null_inactive == count) {
         *outcount = MPI_UNDEFINED;
-        WAIT_SYNC_RELEASE(&sync);
+        /* nobody will signall us */
+        WAIT_SYNC_RELEASE_NOWAIT(&sync);
         return rc;
     }
 
@@ -407,6 +421,14 @@ int ompi_request_default_wait_some(size_t count,
             num_requests_done++;
         }
     }
+    sync_unsets = count - num_requests_null_inactive - num_requests_done;
+
+    if( sync_sets == sync_unsets ){
+        /* nobody knows about us,
+         * set signa-in-progress flag to false
+         */
+        WAIT_SYNC_SIGNALLED(&sync);
+    }
 
     WAIT_SYNC_RELEASE(&sync);
 

opal/threads/wait_sync.h

@@ -45,21 +45,32 @@ typedef struct ompi_wait_sync_t {
  * the critical path. */
 #define WAIT_SYNC_RELEASE(sync)                       \
     if (opal_using_threads()) {                       \
-        while ((sync)->signaling) {                  \
+        while ((sync)->signaling) {                   \
             continue;                                 \
         }                                             \
         pthread_cond_destroy(&(sync)->condition);     \
         pthread_mutex_destroy(&(sync)->lock);         \
     }
 
+#define WAIT_SYNC_RELEASE_NOWAIT(sync)                \
+    if (opal_using_threads()) {                       \
+        pthread_cond_destroy(&(sync)->condition);     \
+        pthread_mutex_destroy(&(sync)->lock);         \
+    }
+
+
 #define WAIT_SYNC_SIGNAL(sync)                        \
     if (opal_using_threads()) {                       \
         pthread_mutex_lock(&(sync->lock));            \
         pthread_cond_signal(&sync->condition);        \
         pthread_mutex_unlock(&(sync->lock));          \
-        sync->signaling = false;                     \
+        sync->signaling = false;                      \
     }
 
+#define WAIT_SYNC_SIGNALLED(sync){                    \
+        (sync)->signaling = false;                    \
+}        
+
 OPAL_DECLSPEC int sync_wait_mt(ompi_wait_sync_t *sync);
 static inline int sync_wait_st (ompi_wait_sync_t *sync)
 {