opal/sync: Fix assert during multi-threaded progress invocation

PR #5241 provided an MCA variable to allow multi-threaded opal_progress.
However, it allowed to update the linked list even when multiple threads was
allowed to call opal_progress. This caused a scenario when a more recent thread
could complete it's progress and fail the assert(sync ==
wait_sync_list).

Allowing to update the linked list only for the case when the number of threads
exceeds the threshold fixes the problem.

Signed-off-by: Aravind Gopalakrishnan <[email protected]>

Author: aravindksg

opal/threads/wait_sync.c

@@ -47,17 +47,19 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync)
     }
 
     /* Insert sync on the list of pending synchronization constructs */
-    OPAL_THREAD_LOCK(&wait_sync_lock);
-    if( NULL == wait_sync_list ) {
-        sync->next = sync->prev = sync;
-        wait_sync_list = sync;
-    } else {
-        sync->prev = wait_sync_list->prev;
-        sync->prev->next = sync;
-        sync->next = wait_sync_list;
-        wait_sync_list->prev = sync;
+    if (num_thread_in_progress >= opal_max_thread_in_progress) {
+        OPAL_THREAD_LOCK(&wait_sync_lock);
+        if( NULL == wait_sync_list ) {
+            sync->next = sync->prev = sync;
+            wait_sync_list = sync;
+        } else {
+            sync->prev = wait_sync_list->prev;
+            sync->prev->next = sync;
+            sync->next = wait_sync_list;
+            wait_sync_list->prev = sync;
+        }
+        OPAL_THREAD_UNLOCK(&wait_sync_lock);
     }
-    OPAL_THREAD_UNLOCK(&wait_sync_lock);
 
     /**
      * If we are not responsible for progresing, go silent until something worth noticing happen:
@@ -89,20 +91,23 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync)
     }
     OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, -1);
 
-    assert(sync == wait_sync_list);
+    if (NULL != wait_sync_list) {
+        assert(sync == wait_sync_list);
+    }
 
  i_am_done:
     /* My sync is now complete. Trim the list: remove self, wake next */
-    OPAL_THREAD_LOCK(&wait_sync_lock);
-    sync->prev->next = sync->next;
-    sync->next->prev = sync->prev;
-    /* In case I am the progress manager, pass the duties on */
-    if( sync == wait_sync_list ) {
-        wait_sync_list = (sync == sync->next) ? NULL : sync->next;
-        if( NULL != wait_sync_list )
-            WAIT_SYNC_PASS_OWNERSHIP(wait_sync_list);
+    if (num_thread_in_progress >= opal_max_thread_in_progress) {
+        OPAL_THREAD_LOCK(&wait_sync_lock);
+        sync->prev->next = sync->next;
+        sync->next->prev = sync->prev;
+        /* In case I am the progress manager, pass the duties on */
+        if( sync == wait_sync_list ) {
+            wait_sync_list = (sync == sync->next) ? NULL : sync->next;
+            if( NULL != wait_sync_list )
+                WAIT_SYNC_PASS_OWNERSHIP(wait_sync_list);
+        }
+        OPAL_THREAD_UNLOCK(&wait_sync_lock);
     }
-    OPAL_THREAD_UNLOCK(&wait_sync_lock);
-
     return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR;
 }