osc/pt2pt: fix possible race in peer locking

It is possible for another thread to process a lock ack before the
peer is set as locked. In this case either setting the locked or the
eager active flag might clobber the other thread. To address this the
flags have been made volatile and are set atomically. Since there is
no a opal_atomic_or or opal_atomic_and function just use cmpset for
now.

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

ompi/mca/osc/pt2pt/osc_pt2pt.h

@@ -121,7 +121,7 @@ struct ompi_osc_pt2pt_peer_t {
     int32_t passive_incoming_frag_count;
 
     /** peer flags */
-    int32_t flags;
+    volatile int32_t flags;
 };
 typedef struct ompi_osc_pt2pt_peer_t ompi_osc_pt2pt_peer_t;
 
@@ -144,11 +144,15 @@ static inline bool ompi_osc_pt2pt_peer_eager_active (ompi_osc_pt2pt_peer_t *peer
 
 static inline void ompi_osc_pt2pt_peer_set_flag (ompi_osc_pt2pt_peer_t *peer, int32_t flag, bool value)
 {
-    if (value) {
-        peer->flags |= flag;
-    } else {
-        peer->flags &= ~flag;
-    }
+    int32_t peer_flags, new_flags;
+    do {
+        peer_flags = peer->flags;
+        if (value) {
+            new_flags = peer_flags | flag;
+        } else {
+            new_flags = peer_flags & ~flag;
+        }
+    } while (!OPAL_ATOMIC_CMPSET_32 (&peer->flags, peer_flags, new_flags));
 }
 
 static inline void ompi_osc_pt2pt_peer_set_locked (ompi_osc_pt2pt_peer_t *peer, bool value)

ompi/mca/osc/pt2pt/osc_pt2pt_sync.h

@@ -74,10 +74,10 @@ struct ompi_osc_pt2pt_sync_t {
     int num_peers;
 
     /** number of synchronization messages expected */
-    int32_t sync_expected;
+    volatile int32_t sync_expected;
 
     /** eager sends are active to all peers in this access epoch */
-    bool eager_send_active;
+    volatile bool eager_send_active;
 
     /** communication has started on this epoch */
     bool epoch_active;
@@ -175,7 +175,7 @@ static inline void ompi_osc_pt2pt_sync_expected (ompi_osc_pt2pt_sync_t *sync)
 static inline void ompi_osc_pt2pt_sync_reset (ompi_osc_pt2pt_sync_t *sync)
 {
     sync->type = OMPI_OSC_PT2PT_SYNC_TYPE_NONE;
-    sync->eager_send_active = 0;
+    sync->eager_send_active = false;
     sync->epoch_active = 0;
     sync->peer_list.peers = NULL;
     sync->sync.pscw.group = NULL;