Complete C11 atomic wrapper implementation - working and preventing direct access

Co-authored-by: devreal <[email protected]>

Author: Copilot

opal/class/opal_lifo.h

@@ -83,7 +83,7 @@ opal_read_counted_pointer(volatile opal_counted_pointer_t *volatile addr,
      * specific order */
     value->data.counter = addr->data.counter;
     opal_atomic_rmb();
-    opal_atomic_store_ptr_relaxed(&value->data.item, opal_atomic_load_ptr_relaxed(&addr->data.item));
+    opal_atomic_store_ptr_volatile_relaxed(&value->data.item, opal_atomic_load_ptr_volatile_relaxed(&addr->data.item));
 }
 
 #endif
@@ -140,7 +140,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic(opal_lifo_t *lifo, opal_li
         opal_atomic_wmb();
 
         /* to protect against ABA issues it is sufficient to only update the counter in pop */
-        if (opal_atomic_compare_exchange_strong_ptr(&lifo->opal_lifo_head.data.item,
+        if (opal_atomic_compare_exchange_strong_ptr_volatile(&lifo->opal_lifo_head.data.item,
                                                     (intptr_t *) &next, (intptr_t) item)) {
             return next;
         }
@@ -159,7 +159,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo)
     opal_read_counted_pointer(&lifo->opal_lifo_head, &old_head);
 
     do {
-        item = (opal_list_item_t *) opal_atomic_load_ptr_relaxed(&old_head.data.item);
+        item = (opal_list_item_t *) opal_atomic_load_ptr_volatile_relaxed(&old_head.data.item);
         if (item == &lifo->opal_lifo_ghost) {
             return NULL;
         }
@@ -189,7 +189,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic(opal_lifo_t *lifo, opal_li
     do {
         item->opal_list_next = next;
         opal_atomic_wmb();
-        if (opal_atomic_compare_exchange_strong_ptr(&lifo->opal_lifo_head.data.item,
+        if (opal_atomic_compare_exchange_strong_ptr_volatile(&lifo->opal_lifo_head.data.item,
                                                     (intptr_t *) &next, (intptr_t) item)) {
             opal_atomic_wmb();
             /* now safe to pop this item */
@@ -252,7 +252,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo)
 
         head = item;
         /* try to swap out the head pointer */
-        if (opal_atomic_compare_exchange_strong_ptr(&lifo->opal_lifo_head.data.item,
+        if (opal_atomic_compare_exchange_strong_ptr_volatile(&lifo->opal_lifo_head.data.item,
                                                     (intptr_t *) &head,
                                                     (intptr_t) item->opal_list_next)) {
             break;

opal/include/opal/sys/atomic_stdc.h

@@ -106,6 +106,11 @@ static inline bool opal_atomic_compare_exchange_strong_ptr(opal_atomic_intptr_t
     return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_relaxed, memory_order_relaxed);
 }
 
+static inline bool opal_atomic_compare_exchange_strong_ptr_volatile(volatile opal_atomic_intptr_t *addr, intptr_t *compare, intptr_t value)
+{
+    return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_relaxed, memory_order_relaxed);
+}
+
 static inline bool opal_atomic_compare_exchange_strong_acq_ptr(opal_atomic_intptr_t *addr, intptr_t *compare, intptr_t value)
 {
     return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_acquire, memory_order_relaxed);

opal/mca/btl/sm/btl_sm_component.c

@@ -549,7 +549,7 @@ static void mca_btl_sm_progress_endpoints(void)
 
 static int mca_btl_sm_component_progress(void)
 {
-    static opal_atomic_int32_t lock = 0;
+    static opal_atomic_int32_t lock = { .value = 0 };
     int count = 0;
 
     if (opal_using_threads()) {
@@ -565,14 +565,14 @@ static int mca_btl_sm_component_progress(void)
 
     mca_btl_sm_progress_endpoints();
 
-    if (SM_FIFO_FREE == mca_btl_sm_component.my_fifo->fifo_head) {
-        lock = 0;
+    if (SM_FIFO_FREE == opal_atomic_load_ptr_relaxed(&mca_btl_sm_component.my_fifo->fifo_head)) {
+        opal_atomic_store_32_relaxed(&lock, 0);
         return count;
     }
 
     count += mca_btl_sm_poll_fifo();
     opal_atomic_mb();
-    lock = 0;
+    opal_atomic_store_32_relaxed(&lock, 0);
 
     return count;
 }

opal/mca/btl/sm/btl_sm_fifo.h

@@ -112,8 +112,8 @@ static inline void sm_fifo_init(sm_fifo_t *fifo)
      * ideal but oh well. See #5814 */
     /* fifo->fifo_head = fifo->fifo_tail = SM_FIFO_FREE; */
     opal_atomic_store_ptr_relaxed(&fifo->fifo_head, SM_FIFO_FREE);
-    fifo->fifo_tail = SM_FIFO_FREE;
-    fifo->fbox_available = mca_btl_sm_component.fbox_max;
+    opal_atomic_store_ptr_relaxed(&fifo->fifo_tail, SM_FIFO_FREE);
+    opal_atomic_store_32_relaxed(&fifo->fbox_available, mca_btl_sm_component.fbox_max);
     mca_btl_sm_component.my_fifo = fifo;
 }
 

opal/mca/mpool/hugepage/mpool_hugepage_component.c

@@ -131,7 +131,7 @@ static int mca_mpool_hugepage_register(void)
                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_LOCAL, &mca_mpool_hugepage_page_size);
 
-    mca_mpool_hugepage_component.bytes_allocated = 0;
+    opal_atomic_store_size_t_relaxed(&mca_mpool_hugepage_component.bytes_allocated, 0);
     (void) mca_base_component_pvar_register(&mca_mpool_hugepage_component.super.mpool_version,
                                             "bytes_allocated",
                                             "Number of bytes currently allocated in the mpool "