diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index 787e1e10249..54ea385cf08 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -125,7 +125,7 @@ const char ompi_version_string[] = OMPI_IDENT_STRING; * Global variables and symbols for the MPI layer */ -opal_atomic_int32_t ompi_mpi_state = OMPI_MPI_STATE_NOT_INITIALIZED; +opal_atomic_int32_t ompi_mpi_state = { .value = OMPI_MPI_STATE_NOT_INITIALIZED }; volatile bool ompi_rte_initialized = false; bool ompi_mpi_thread_multiple = false; @@ -371,7 +371,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided, // silently return successfully once the initializing // thread has completed. if (reinit_ok) { - while (ompi_mpi_state < OMPI_MPI_STATE_INIT_COMPLETED) { + while (opal_atomic_load_32_relaxed(&ompi_mpi_state) < OMPI_MPI_STATE_INIT_COMPLETED) { usleep(1); } return MPI_SUCCESS; diff --git a/opal/class/opal_lifo.h b/opal/class/opal_lifo.h index 72cc531e3b8..adbad3389f2 100644 --- a/opal/class/opal_lifo.h +++ b/opal/class/opal_lifo.h @@ -70,7 +70,7 @@ static inline bool opal_update_counted_pointer(volatile opal_counted_pointer_t * opal_counted_pointer_t *old, opal_list_item_t *item) { opal_counted_pointer_t new_p; - new_p.data.item = (intptr_t) item; + opal_atomic_store_ptr_volatile_relaxed(&new_p.data.item, (intptr_t) item); new_p.data.counter = old->data.counter + 1; return opal_atomic_compare_exchange_strong_128(&addr->atomic_value, &old->value, new_p.value); } @@ -83,7 +83,7 @@ opal_read_counted_pointer(volatile opal_counted_pointer_t *volatile addr, * specific order */ value->data.counter = addr->data.counter; opal_atomic_rmb(); - value->data.item = addr->data.item; + opal_atomic_store_ptr_volatile_relaxed(&value->data.item, opal_atomic_load_ptr_volatile_relaxed(&addr->data.item)); } #endif @@ -122,7 +122,7 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_lifo_t); */ static inline bool opal_lifo_is_empty(opal_lifo_t *lifo) { - return (opal_list_item_t *) lifo->opal_lifo_head.data.item == &lifo->opal_lifo_ghost; + return (opal_list_item_t *) opal_atomic_load_ptr_volatile_relaxed(&lifo->opal_lifo_head.data.item) == &lifo->opal_lifo_ghost; } #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && !OPAL_HAVE_ATOMIC_LLSC_PTR @@ -133,14 +133,14 @@ static inline bool opal_lifo_is_empty(opal_lifo_t *lifo) */ static inline opal_list_item_t *opal_lifo_push_atomic(opal_lifo_t *lifo, opal_list_item_t *item) { - opal_list_item_t *next = (opal_list_item_t *) lifo->opal_lifo_head.data.item; + opal_list_item_t *next = (opal_list_item_t *) opal_atomic_load_ptr_volatile_relaxed(&lifo->opal_lifo_head.data.item); do { item->opal_list_next = next; opal_atomic_wmb(); /* to protect against ABA issues it is sufficient to only update the counter in pop */ - if (opal_atomic_compare_exchange_strong_ptr(&lifo->opal_lifo_head.data.item, + if (opal_atomic_compare_exchange_strong_ptr_volatile(&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) { return next; } @@ -159,7 +159,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo) opal_read_counted_pointer(&lifo->opal_lifo_head, &old_head); do { - item = (opal_list_item_t *) old_head.data.item; + item = (opal_list_item_t *) opal_atomic_load_ptr_volatile_relaxed(&old_head.data.item); if (item == &lifo->opal_lifo_ghost) { return NULL; } @@ -181,7 +181,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo) */ static inline opal_list_item_t *opal_lifo_push_atomic(opal_lifo_t *lifo, opal_list_item_t *item) { - opal_list_item_t *next = (opal_list_item_t *) lifo->opal_lifo_head.data.item; + opal_list_item_t *next = (opal_list_item_t *) opal_atomic_load_ptr_volatile_relaxed(&lifo->opal_lifo_head.data.item); /* item free acts as a mini lock to avoid ABA problems */ item->item_free = 1; @@ -189,7 +189,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic(opal_lifo_t *lifo, opal_li do { item->opal_list_next = next; opal_atomic_wmb(); - if (opal_atomic_compare_exchange_strong_ptr(&lifo->opal_lifo_head.data.item, + if (opal_atomic_compare_exchange_strong_ptr_volatile(&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) { opal_atomic_wmb(); /* now safe to pop this item */ @@ -218,13 +218,13 @@ static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo) attempt = 0; } - opal_atomic_ll_ptr(&lifo->opal_lifo_head.data.item, item); + opal_atomic_ll_ptr(&lifo->opal_lifo_head.data.item.value, item); if (&lifo->opal_lifo_ghost == item) { return NULL; } next = (opal_list_item_t *) item->opal_list_next; - opal_atomic_sc_ptr(&lifo->opal_lifo_head.data.item, next, ret); + opal_atomic_sc_ptr(&lifo->opal_lifo_head.data.item.value, next, ret); } while (!ret); opal_atomic_wmb(); @@ -242,7 +242,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo) { opal_list_item_t *item, *head, *ghost = &lifo->opal_lifo_ghost; - while ((item = (opal_list_item_t *) lifo->opal_lifo_head.data.item) != ghost) { + while ((item = (opal_list_item_t *) opal_atomic_load_ptr_volatile_relaxed(&lifo->opal_lifo_head.data.item)) != ghost) { /* ensure it is safe to pop the head */ if (opal_atomic_swap_32((opal_atomic_int32_t *) &item->item_free, 1)) { continue; @@ -252,7 +252,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo) head = item; /* try to swap out the head pointer */ - if (opal_atomic_compare_exchange_strong_ptr(&lifo->opal_lifo_head.data.item, + if (opal_atomic_compare_exchange_strong_ptr_volatile(&lifo->opal_lifo_head.data.item, (intptr_t *) &head, (intptr_t) item->opal_list_next)) { break; @@ -282,17 +282,17 @@ static inline opal_list_item_t *opal_lifo_pop_atomic(opal_lifo_t *lifo) /* single-threaded versions of the lifo functions */ static inline opal_list_item_t *opal_lifo_push_st(opal_lifo_t *lifo, opal_list_item_t *item) { - item->opal_list_next = (opal_list_item_t *) lifo->opal_lifo_head.data.item; + item->opal_list_next = (opal_list_item_t *) opal_atomic_load_ptr_volatile_relaxed(&lifo->opal_lifo_head.data.item); item->item_free = 0; - lifo->opal_lifo_head.data.item = (intptr_t) item; + opal_atomic_store_ptr_volatile_relaxed(&lifo->opal_lifo_head.data.item, (intptr_t) item); return (opal_list_item_t *) item->opal_list_next; } static inline opal_list_item_t *opal_lifo_pop_st(opal_lifo_t *lifo) { opal_list_item_t *item; - item = (opal_list_item_t *) lifo->opal_lifo_head.data.item; - lifo->opal_lifo_head.data.item = (intptr_t) item->opal_list_next; + item = (opal_list_item_t *) opal_atomic_load_ptr_volatile_relaxed(&lifo->opal_lifo_head.data.item); + opal_atomic_store_ptr_volatile_relaxed(&lifo->opal_lifo_head.data.item, (intptr_t) item->opal_list_next); if (item == &lifo->opal_lifo_ghost) { return NULL; } diff --git a/opal/class/opal_list.h b/opal/class/opal_list.h index a829b6e328a..a549d3fc00e 100644 --- a/opal/class/opal_list.h +++ b/opal/class/opal_list.h @@ -168,7 +168,7 @@ typedef struct opal_list_t opal_list_t; #define OPAL_LIST_DESTRUCT(list) \ do { \ opal_list_item_t *it; \ - if (1 == ((opal_object_t *) (list))->obj_reference_count) { \ + if (1 == opal_atomic_load_32_relaxed(&((opal_object_t *) (list))->obj_reference_count)) { \ while (NULL != (it = opal_list_remove_first(list))) { \ OBJ_RELEASE(it); \ } \ @@ -179,7 +179,7 @@ typedef struct opal_list_t opal_list_t; #define OPAL_LIST_RELEASE(list) \ do { \ opal_list_item_t *it; \ - if (1 == ((opal_object_t *) (list))->obj_reference_count) { \ + if (1 == opal_atomic_load_32_relaxed(&((opal_object_t *) (list))->obj_reference_count)) { \ while (NULL != (it = opal_list_remove_first(list))) { \ OBJ_RELEASE(it); \ } \ diff --git a/opal/class/opal_object.h b/opal/class/opal_object.h index 40cd9e5adbe..f8c1b8ae898 100644 --- a/opal/class/opal_object.h +++ b/opal/class/opal_object.h @@ -172,12 +172,12 @@ extern int opal_class_init_epoch; # define OPAL_OBJ_STATIC_INIT(BASE_CLASS) \ { \ .obj_magic_id = OPAL_OBJ_MAGIC_ID, .obj_class = OBJ_CLASS(BASE_CLASS), \ - .obj_reference_count = 1, .cls_init_file_name = __FILE__, .cls_init_lineno = __LINE__, \ + .obj_reference_count = { .value = 1 }, .cls_init_file_name = __FILE__, .cls_init_lineno = __LINE__, \ } #else # define OPAL_OBJ_STATIC_INIT(BASE_CLASS) \ { \ - .obj_class = OBJ_CLASS(BASE_CLASS), .obj_reference_count = 1, \ + .obj_class = OBJ_CLASS(BASE_CLASS), .obj_reference_count = { .value = 1 }, \ } #endif @@ -275,7 +275,7 @@ static inline opal_object_t *opal_obj_new_debug(opal_class_t *type, const char * assert(NULL != ((opal_object_t *) (object))->obj_class); \ assert(OPAL_OBJ_MAGIC_ID == ((opal_object_t *) (object))->obj_magic_id); \ opal_obj_update((opal_object_t *) (object), 1); \ - assert(((opal_object_t *) (object))->obj_reference_count >= 0); \ + assert(opal_atomic_load_32_relaxed(&((opal_object_t *) (object))->obj_reference_count) >= 0); \ } while (0) #else # define OBJ_RETAIN(object) opal_obj_update((opal_object_t *) (object), 1); @@ -377,7 +377,7 @@ static inline opal_object_t *opal_obj_new_debug(opal_class_t *type, const char * opal_class_initialize((type)); \ } \ ((opal_object_t *) (object))->obj_class = (type); \ - ((opal_object_t *) (object))->obj_reference_count = 1; \ + opal_atomic_store_32_relaxed(&((opal_object_t *) (object))->obj_reference_count, 1); \ opal_obj_run_constructors((opal_object_t *) (object)); \ OBJ_REMEMBER_FILE_AND_LINENO(object, __FILE__, __LINE__); \ } while (0) @@ -499,7 +499,7 @@ static inline opal_object_t *opal_obj_new(opal_class_t *cls) } if (NULL != object) { object->obj_class = cls; - object->obj_reference_count = 1; + opal_atomic_store_32_relaxed(&object->obj_reference_count, 1); opal_obj_run_constructors(object); } return object; diff --git a/opal/datatype/opal_datatype_destroy.c b/opal/datatype/opal_datatype_destroy.c index 3381df622bf..7517fb1746b 100644 --- a/opal/datatype/opal_datatype_destroy.c +++ b/opal/datatype/opal_datatype_destroy.c @@ -27,7 +27,7 @@ int32_t opal_datatype_destroy(opal_datatype_t **dt) { opal_datatype_t *pData = *dt; - if ((pData->flags & OPAL_DATATYPE_FLAG_PREDEFINED) && (pData->super.obj_reference_count <= 1)) { + if ((pData->flags & OPAL_DATATYPE_FLAG_PREDEFINED) && (opal_atomic_load_32_relaxed(&pData->super.obj_reference_count) <= 1)) { return OPAL_ERROR; } diff --git a/opal/include/opal/sys/atomic_impl_math.h b/opal/include/opal/sys/atomic_impl_math.h index 7f48e50a23b..028f9b9fbd1 100644 --- a/opal/include/opal/sys/atomic_impl_math.h +++ b/opal/include/opal/sys/atomic_impl_math.h @@ -32,7 +32,7 @@ { \ type oldval; \ do { \ - oldval = *addr; \ + oldval = opal_atomic_load_##bits##_relaxed(addr); \ } while (!opal_atomic_compare_exchange_strong_##bits(addr, &oldval, \ oldval operation value)); \ \ @@ -43,9 +43,9 @@ { \ type oldval, newval; \ do { \ - oldval = *addr; \ + oldval = opal_atomic_load_##bits##_relaxed(addr); \ newval = oldval operation value; \ - } while (!opal_atomic_compare_exchange_strong_##bits(addr, &oldval, newval); \ + } while (!opal_atomic_compare_exchange_strong_##bits(addr, &oldval, newval)); \ \ return newval; \ } diff --git a/opal/include/opal/sys/atomic_impl_minmax_math.h b/opal/include/opal/sys/atomic_impl_minmax_math.h index 7ec5920ad46..5a396b7844b 100644 --- a/opal/include/opal/sys/atomic_impl_minmax_math.h +++ b/opal/include/opal/sys/atomic_impl_minmax_math.h @@ -31,7 +31,7 @@ static inline int32_t opal_atomic_fetch_min_32(opal_atomic_int32_t *addr, int32_t value) { - int32_t old = *addr; + int32_t old = opal_atomic_load_32_relaxed(addr); do { if (old <= value) { break; @@ -49,7 +49,7 @@ static inline int32_t opal_atomic_min_fetch_32(opal_atomic_int32_t *addr, int32_ static inline int32_t opal_atomic_fetch_max_32(opal_atomic_int32_t *addr, int32_t value) { - int32_t old = *addr; + int32_t old = opal_atomic_load_32_relaxed(addr); do { if (old >= value) { break; @@ -67,7 +67,7 @@ static inline int32_t opal_atomic_max_fetch_32(opal_atomic_int32_t *addr, int32_ static inline int64_t opal_atomic_fetch_min_64(opal_atomic_int64_t *addr, int64_t value) { - int64_t old = *addr; + int64_t old = opal_atomic_load_64_relaxed(addr); do { if (old <= value) { break; @@ -79,7 +79,7 @@ static inline int64_t opal_atomic_fetch_min_64(opal_atomic_int64_t *addr, int64_ static inline int64_t opal_atomic_fetch_max_64(opal_atomic_int64_t *addr, int64_t value) { - int64_t old = *addr; + int64_t old = opal_atomic_load_64_relaxed(addr); do { if (old >= value) { break; diff --git a/opal/include/opal/sys/atomic_stdc.h b/opal/include/opal/sys/atomic_stdc.h index 5eca86c9edb..5327b94560a 100644 --- a/opal/include/opal/sys/atomic_stdc.h +++ b/opal/include/opal/sys/atomic_stdc.h @@ -71,40 +71,61 @@ static inline void opal_atomic_rmb(void) * *********************************************************************/ -# define opal_atomic_compare_exchange_strong_32(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_relaxed, \ - memory_order_relaxed) -# define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_acquire, \ - memory_order_relaxed) -# define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_release, \ - memory_order_relaxed) - -# define opal_atomic_compare_exchange_strong_64(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_relaxed, \ - memory_order_relaxed) -# define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_acquire, \ - memory_order_relaxed) -# define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_release, \ - memory_order_relaxed) - -# define opal_atomic_compare_exchange_strong_ptr(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_relaxed, \ - memory_order_relaxed) -# define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_acquire, \ - memory_order_relaxed) -# define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_release, \ - memory_order_relaxed) +static inline bool opal_atomic_compare_exchange_strong_32(opal_atomic_int32_t *addr, int32_t *compare, int32_t value) +{ + return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_relaxed, memory_order_relaxed); +} + +static inline bool opal_atomic_compare_exchange_strong_acq_32(opal_atomic_int32_t *addr, int32_t *compare, int32_t value) +{ + return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_acquire, memory_order_relaxed); +} + +static inline bool opal_atomic_compare_exchange_strong_rel_32(opal_atomic_int32_t *addr, int32_t *compare, int32_t value) +{ + return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_release, memory_order_relaxed); +} + +static inline bool opal_atomic_compare_exchange_strong_64(opal_atomic_int64_t *addr, int64_t *compare, int64_t value) +{ + return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_relaxed, memory_order_relaxed); +} + +static inline bool opal_atomic_compare_exchange_strong_acq_64(opal_atomic_int64_t *addr, int64_t *compare, int64_t value) +{ + return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_acquire, memory_order_relaxed); +} + +static inline bool opal_atomic_compare_exchange_strong_rel_64(opal_atomic_int64_t *addr, int64_t *compare, int64_t value) +{ + return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_release, memory_order_relaxed); +} + +static inline bool opal_atomic_compare_exchange_strong_ptr(opal_atomic_intptr_t *addr, intptr_t *compare, intptr_t value) +{ + return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_relaxed, memory_order_relaxed); +} + +static inline bool opal_atomic_compare_exchange_strong_ptr_volatile(volatile opal_atomic_intptr_t *addr, intptr_t *compare, intptr_t value) +{ + return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_relaxed, memory_order_relaxed); +} + +static inline bool opal_atomic_compare_exchange_strong_acq_ptr(opal_atomic_intptr_t *addr, intptr_t *compare, intptr_t value) +{ + return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_acquire, memory_order_relaxed); +} + +static inline bool opal_atomic_compare_exchange_strong_rel_ptr(opal_atomic_intptr_t *addr, intptr_t *compare, intptr_t value) +{ + return atomic_compare_exchange_strong_explicit(&(addr->value), compare, value, memory_order_release, memory_order_relaxed); +} # if OPAL_HAVE_C11_CSWAP_INT128 /* the C11 atomic compare-exchange is lock free so use it */ -# define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong +# define opal_atomic_compare_exchange_strong_128(addr, oldval, newval) \ + atomic_compare_exchange_strong(&((addr)->value), oldval, newval) # define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 @@ -137,12 +158,20 @@ opal_atomic_compare_exchange_strong_128(opal_atomic_int128_t *addr, opal_int128_ * *********************************************************************/ -# define opal_atomic_swap_32(addr, value) \ - atomic_exchange_explicit((_Atomic unsigned int *) addr, value, memory_order_relaxed) -# define opal_atomic_swap_64(addr, value) \ - atomic_exchange_explicit((_Atomic unsigned long *) addr, value, memory_order_relaxed) -# define opal_atomic_swap_ptr(addr, value) \ - atomic_exchange_explicit((_Atomic unsigned long *) addr, value, memory_order_relaxed) +static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t value) +{ + return atomic_exchange_explicit(&(addr->value), value, memory_order_relaxed); +} + +static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t value) +{ + return atomic_exchange_explicit(&(addr->value), value, memory_order_relaxed); +} + +static inline intptr_t opal_atomic_swap_ptr(opal_atomic_intptr_t *addr, intptr_t value) +{ + return atomic_exchange_explicit(&(addr->value), value, memory_order_relaxed); +} /********************************************************************** @@ -184,12 +213,12 @@ static inline void opal_atomic_unlock(opal_atomic_lock_t *lock) # define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator) \ static inline type opal_atomic_fetch_##op##_##bits(opal_atomic_##type *addr, type value) \ { \ - return atomic_fetch_##op##_explicit(addr, value, memory_order_relaxed); \ + return atomic_fetch_##op##_explicit(&((addr)->value), value, memory_order_relaxed); \ } \ \ static inline type opal_atomic_##op##_fetch_##bits(opal_atomic_##type *addr, type value) \ { \ - return atomic_fetch_##op##_explicit(addr, value, memory_order_relaxed) operator value; \ + return atomic_fetch_##op##_explicit(&((addr)->value), value, memory_order_relaxed) operator value; \ } OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +) @@ -208,7 +237,7 @@ OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +) OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -) # define opal_atomic_add(addr, value) \ - (void) atomic_fetch_add_explicit(addr, value, memory_order_relaxed) + (void) atomic_fetch_add_explicit(&((addr)->value), value, memory_order_relaxed) #include "opal/sys/atomic_impl_minmax_math.h" diff --git a/opal/include/opal/sys/x86_64/atomic.h b/opal/include/opal/sys/x86_64/atomic.h index f16133bda25..b1c820777fc 100644 --- a/opal/include/opal/sys/x86_64/atomic.h +++ b/opal/include/opal/sys/x86_64/atomic.h @@ -168,7 +168,7 @@ static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t new static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *v, int i) { int ret = i; - __asm__ __volatile__(SMPLOCK "xaddl %1,%0" : "+m"(*v), "+r"(ret) : : "memory", "cc"); + __asm__ __volatile__(SMPLOCK "xaddl %1,%0" : "+m"(v->value), "+r"(ret) : : "memory", "cc"); return ret; } @@ -180,7 +180,7 @@ static inline int32_t opal_atomic_add_fetch_32(opal_atomic_int32_t *v, int i) static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *v, int64_t i) { int64_t ret = i; - __asm__ __volatile__(SMPLOCK "xaddq %1,%0" : "+m"(*v), "+r"(ret) : : "memory", "cc"); + __asm__ __volatile__(SMPLOCK "xaddq %1,%0" : "+m"(v->value), "+r"(ret) : : "memory", "cc"); return ret; } @@ -192,7 +192,7 @@ static inline int64_t opal_atomic_add_fetch_64(opal_atomic_int64_t *v, int64_t i static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *v, int i) { int ret = -i; - __asm__ __volatile__(SMPLOCK "xaddl %1,%0" : "+m"(*v), "+r"(ret) : : "memory", "cc"); + __asm__ __volatile__(SMPLOCK "xaddl %1,%0" : "+m"(v->value), "+r"(ret) : : "memory", "cc"); return ret; } @@ -204,7 +204,7 @@ static inline int32_t opal_atomic_sub_fetch_32(opal_atomic_int32_t *v, int i) static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *v, int64_t i) { int64_t ret = -i; - __asm__ __volatile__(SMPLOCK "xaddq %1,%0" : "+m"(*v), "+r"(ret) : : "memory", "cc"); + __asm__ __volatile__(SMPLOCK "xaddq %1,%0" : "+m"(v->value), "+r"(ret) : : "memory", "cc"); return ret; } @@ -218,7 +218,7 @@ static inline int64_t opal_atomic_sub_fetch_64(opal_atomic_int64_t *v, int64_t i { \ type oldval; \ do { \ - oldval = *addr; \ + oldval = opal_atomic_load_##bits##_relaxed(addr); \ } while (!opal_atomic_compare_exchange_strong_##bits(addr, &oldval, \ oldval operation value)); \ \ @@ -229,7 +229,7 @@ static inline int64_t opal_atomic_sub_fetch_64(opal_atomic_int64_t *v, int64_t i { \ type oldval, newval; \ do { \ - oldval = *addr; \ + oldval = opal_atomic_load_##bits##_relaxed(addr); \ newval = oldval operation value; \ } while (!opal_atomic_compare_exchange_strong_##bits(addr, &oldval, newval)); \ \ diff --git a/opal/include/opal_stdatomic.h b/opal/include/opal_stdatomic.h index f7dd8353d3b..eb671ce4202 100644 --- a/opal/include/opal_stdatomic.h +++ b/opal/include/opal_stdatomic.h @@ -42,18 +42,20 @@ enum { OPAL_ATOMIC_LOCK_UNLOCKED = 0, # include -typedef atomic_int opal_atomic_int_t; -typedef atomic_long opal_atomic_long_t; +/* Wrap _Atomic types in structs to prevent direct access and enforce + * use of accessor functions with memory ordering control */ +typedef struct { atomic_int value; } opal_atomic_int_t; +typedef struct { atomic_long value; } opal_atomic_long_t; -typedef _Atomic int32_t opal_atomic_int32_t; -typedef _Atomic uint32_t opal_atomic_uint32_t; -typedef _Atomic int64_t opal_atomic_int64_t; -typedef _Atomic uint64_t opal_atomic_uint64_t; +typedef struct { _Atomic int32_t value; } opal_atomic_int32_t; +typedef struct { _Atomic uint32_t value; } opal_atomic_uint32_t; +typedef struct { _Atomic int64_t value; } opal_atomic_int64_t; +typedef struct { _Atomic uint64_t value; } opal_atomic_uint64_t; -typedef _Atomic size_t opal_atomic_size_t; -typedef _Atomic ssize_t opal_atomic_ssize_t; -typedef _Atomic intptr_t opal_atomic_intptr_t; -typedef _Atomic uintptr_t opal_atomic_uintptr_t; +typedef struct { _Atomic size_t value; } opal_atomic_size_t; +typedef struct { _Atomic ssize_t value; } opal_atomic_ssize_t; +typedef struct { _Atomic intptr_t value; } opal_atomic_intptr_t; +typedef struct { _Atomic uintptr_t value; } opal_atomic_uintptr_t; typedef atomic_flag opal_atomic_lock_t; @@ -62,13 +64,189 @@ typedef atomic_flag opal_atomic_lock_t; # define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT +/* Load and store accessor functions with memory ordering control. + * Default to relaxed ordering to avoid sequential consistency overhead. */ + +/* Load functions with memory ordering */ +static inline int opal_atomic_load_int(const opal_atomic_int_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +static inline long opal_atomic_load_long(const opal_atomic_long_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +static inline int32_t opal_atomic_load_32(const opal_atomic_int32_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +static inline uint32_t opal_atomic_load_uint32(const opal_atomic_uint32_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +static inline int64_t opal_atomic_load_64(const opal_atomic_int64_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +static inline uint64_t opal_atomic_load_uint64(const opal_atomic_uint64_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +static inline size_t opal_atomic_load_size_t(const opal_atomic_size_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +static inline ssize_t opal_atomic_load_ssize_t(const opal_atomic_ssize_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +static inline intptr_t opal_atomic_load_ptr(const opal_atomic_intptr_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +static inline intptr_t opal_atomic_load_ptr_volatile(const volatile opal_atomic_intptr_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +static inline uintptr_t opal_atomic_load_uptr(const opal_atomic_uintptr_t *addr, memory_order order) +{ + return atomic_load_explicit(&addr->value, order); +} + +/* Store functions with memory ordering */ +static inline void opal_atomic_store_int(opal_atomic_int_t *addr, int value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +static inline void opal_atomic_store_long(opal_atomic_long_t *addr, long value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +static inline void opal_atomic_store_32(opal_atomic_int32_t *addr, int32_t value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +static inline void opal_atomic_store_uint32(opal_atomic_uint32_t *addr, uint32_t value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +static inline void opal_atomic_store_64(opal_atomic_int64_t *addr, int64_t value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +static inline void opal_atomic_store_uint64(opal_atomic_uint64_t *addr, uint64_t value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +static inline void opal_atomic_store_size_t(opal_atomic_size_t *addr, size_t value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +static inline void opal_atomic_store_ssize_t(opal_atomic_ssize_t *addr, ssize_t value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +static inline void opal_atomic_store_ptr(opal_atomic_intptr_t *addr, intptr_t value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +static inline void opal_atomic_store_ptr_volatile(volatile opal_atomic_intptr_t *addr, intptr_t value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +static inline void opal_atomic_store_uptr(opal_atomic_uintptr_t *addr, uintptr_t value, memory_order order) +{ + atomic_store_explicit(&addr->value, value, order); +} + +/* Convenience functions with default relaxed ordering */ +static inline int32_t opal_atomic_load_32_relaxed(const opal_atomic_int32_t *addr) +{ + return opal_atomic_load_32(addr, memory_order_relaxed); +} + +static inline int64_t opal_atomic_load_64_relaxed(const opal_atomic_int64_t *addr) +{ + return opal_atomic_load_64(addr, memory_order_relaxed); +} + +static inline intptr_t opal_atomic_load_ptr_relaxed(const opal_atomic_intptr_t *addr) +{ + return opal_atomic_load_ptr(addr, memory_order_relaxed); +} + +static inline intptr_t opal_atomic_load_ptr_volatile_relaxed(const volatile opal_atomic_intptr_t *addr) +{ + return opal_atomic_load_ptr_volatile(addr, memory_order_relaxed); +} + +static inline void opal_atomic_store_32_relaxed(opal_atomic_int32_t *addr, int32_t value) +{ + opal_atomic_store_32(addr, value, memory_order_relaxed); +} + +static inline void opal_atomic_store_64_relaxed(opal_atomic_int64_t *addr, int64_t value) +{ + opal_atomic_store_64(addr, value, memory_order_relaxed); +} + +static inline void opal_atomic_store_ptr_relaxed(opal_atomic_intptr_t *addr, intptr_t value) +{ + opal_atomic_store_ptr(addr, value, memory_order_relaxed); +} + +static inline void opal_atomic_store_ptr_volatile_relaxed(volatile opal_atomic_intptr_t *addr, intptr_t value) +{ + opal_atomic_store_ptr_volatile(addr, value, memory_order_relaxed); +} + +static inline size_t opal_atomic_load_size_t_relaxed(const opal_atomic_size_t *addr) +{ + return opal_atomic_load_size_t(addr, memory_order_relaxed); +} + +static inline void opal_atomic_store_size_t_relaxed(opal_atomic_size_t *addr, size_t value) +{ + opal_atomic_store_size_t(addr, value, memory_order_relaxed); +} + +static inline uint32_t opal_atomic_load_uint32_relaxed(const opal_atomic_uint32_t *addr) +{ + return opal_atomic_load_uint32(addr, memory_order_relaxed); +} + +static inline void opal_atomic_store_uint32_relaxed(opal_atomic_uint32_t *addr, uint32_t value) +{ + opal_atomic_store_uint32(addr, value, memory_order_relaxed); +} + # endif /* OPAL_USE_C11_ATOMICS == 0 */ # if HAVE_OPAL_INT128_T # if OPAL_USE_C11_ATOMICS && OPAL_HAVE_C11_CSWAP_INT128 -typedef _Atomic opal_int128_t opal_atomic_int128_t; +typedef struct { _Atomic opal_int128_t value; } opal_atomic_int128_t; # else diff --git a/opal/mca/btl/base/btl_base_am_rdma.c b/opal/mca/btl/base/btl_base_am_rdma.c index 2d8dd41f764..c9d8ca69238 100644 --- a/opal/mca/btl/base/btl_base_am_rdma.c +++ b/opal/mca/btl/base/btl_base_am_rdma.c @@ -134,8 +134,8 @@ typedef struct am_rdma_context_t am_rdma_context_t; static void am_rdma_context_init(am_rdma_context_t *context) { - context->sent = 0; - context->acknowledged = 0; + opal_atomic_store_64_relaxed(&context->sent, 0); + opal_atomic_store_64_relaxed(&context->acknowledged, 0); context->descriptor = NULL; } diff --git a/opal/mca/btl/sm/btl_sm_component.c b/opal/mca/btl/sm/btl_sm_component.c index 21e1bcf2a6c..de89ec46a17 100644 --- a/opal/mca/btl/sm/btl_sm_component.c +++ b/opal/mca/btl/sm/btl_sm_component.c @@ -549,7 +549,7 @@ static void mca_btl_sm_progress_endpoints(void) static int mca_btl_sm_component_progress(void) { - static opal_atomic_int32_t lock = 0; + static opal_atomic_int32_t lock = { .value = 0 }; int count = 0; if (opal_using_threads()) { @@ -565,14 +565,14 @@ static int mca_btl_sm_component_progress(void) mca_btl_sm_progress_endpoints(); - if (SM_FIFO_FREE == mca_btl_sm_component.my_fifo->fifo_head) { - lock = 0; + if (SM_FIFO_FREE == opal_atomic_load_ptr_relaxed(&mca_btl_sm_component.my_fifo->fifo_head)) { + opal_atomic_store_32_relaxed(&lock, 0); return count; } count += mca_btl_sm_poll_fifo(); opal_atomic_mb(); - lock = 0; + opal_atomic_store_32_relaxed(&lock, 0); return count; } diff --git a/opal/mca/btl/sm/btl_sm_fifo.h b/opal/mca/btl/sm/btl_sm_fifo.h index 36f497a5dd9..e85b0c6bc55 100644 --- a/opal/mca/btl/sm/btl_sm_fifo.h +++ b/opal/mca/btl/sm/btl_sm_fifo.h @@ -73,18 +73,18 @@ static inline mca_btl_sm_hdr_t *sm_fifo_read(sm_fifo_t *fifo, struct mca_btl_bas mca_btl_sm_hdr_t *hdr; fifo_value_t value; - if (SM_FIFO_FREE == fifo->fifo_head) { + if (SM_FIFO_FREE == opal_atomic_load_ptr_relaxed(&fifo->fifo_head)) { return NULL; } opal_atomic_rmb(); - value = fifo->fifo_head; + value = opal_atomic_load_ptr_relaxed(&fifo->fifo_head); *ep = &mca_btl_sm_component.endpoints[value >> MCA_BTL_SM_OFFSET_BITS]; hdr = (mca_btl_sm_hdr_t *) relative2virtual(value); - fifo->fifo_head = SM_FIFO_FREE; + opal_atomic_store_ptr_relaxed(&fifo->fifo_head, SM_FIFO_FREE); assert(hdr->next != value); @@ -96,10 +96,10 @@ static inline mca_btl_sm_hdr_t *sm_fifo_read(sm_fifo_t *fifo, struct mca_btl_bas opal_atomic_rmb(); } - fifo->fifo_head = hdr->next; + opal_atomic_store_ptr_relaxed(&fifo->fifo_head, hdr->next); } } else { - fifo->fifo_head = hdr->next; + opal_atomic_store_ptr_relaxed(&fifo->fifo_head, hdr->next); } opal_atomic_wmb(); @@ -111,9 +111,9 @@ static inline void sm_fifo_init(sm_fifo_t *fifo) /* due to a compiler bug in Oracle C 5.15 the following line was broken into two. Not * ideal but oh well. See #5814 */ /* fifo->fifo_head = fifo->fifo_tail = SM_FIFO_FREE; */ - fifo->fifo_head = SM_FIFO_FREE; - fifo->fifo_tail = SM_FIFO_FREE; - fifo->fbox_available = mca_btl_sm_component.fbox_max; + opal_atomic_store_ptr_relaxed(&fifo->fifo_head, SM_FIFO_FREE); + opal_atomic_store_ptr_relaxed(&fifo->fifo_tail, SM_FIFO_FREE); + opal_atomic_store_32_relaxed(&fifo->fbox_available, mca_btl_sm_component.fbox_max); mca_btl_sm_component.my_fifo = fifo; } @@ -131,7 +131,7 @@ static inline void sm_fifo_write(sm_fifo_t *fifo, fifo_value_t value) mca_btl_sm_hdr_t *hdr = (mca_btl_sm_hdr_t *) relative2virtual(prev); hdr->next = value; } else { - fifo->fifo_head = value; + opal_atomic_store_ptr_relaxed(&fifo->fifo_head, value); } opal_atomic_wmb(); diff --git a/opal/mca/common/sm/common_sm.c b/opal/mca/common/sm/common_sm.c index 0650cccea68..3e5a11305d5 100644 --- a/opal/mca/common/sm/common_sm.c +++ b/opal/mca/common/sm/common_sm.c @@ -110,8 +110,8 @@ static mca_common_sm_module_t *attach_and_init(opal_shmem_ds_t *shmem_bufp, size /* initialize some segment information */ size_t mem_offset = map->module_data_addr - (unsigned char *) map->module_seg; opal_atomic_lock_init(&map->module_seg->seg_lock, OPAL_ATOMIC_LOCK_UNLOCKED); - map->module_seg->seg_inited = 0; - map->module_seg->seg_num_procs_inited = 0; + opal_atomic_store_32_relaxed(&map->module_seg->seg_inited, 0); + opal_atomic_store_size_t_relaxed(&map->module_seg->seg_num_procs_inited, 0); map->module_seg->seg_offset = mem_offset; map->module_seg->seg_size = size - mem_offset; opal_atomic_wmb(); diff --git a/opal/mca/mpool/hugepage/mpool_hugepage_component.c b/opal/mca/mpool/hugepage/mpool_hugepage_component.c index 41d2d70c4e9..4f5572cf211 100644 --- a/opal/mca/mpool/hugepage/mpool_hugepage_component.c +++ b/opal/mca/mpool/hugepage/mpool_hugepage_component.c @@ -131,7 +131,7 @@ static int mca_mpool_hugepage_register(void) MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_mpool_hugepage_page_size); - mca_mpool_hugepage_component.bytes_allocated = 0; + opal_atomic_store_size_t_relaxed(&mca_mpool_hugepage_component.bytes_allocated, 0); (void) mca_base_component_pvar_register(&mca_mpool_hugepage_component.super.mpool_version, "bytes_allocated", "Number of bytes currently allocated in the mpool " diff --git a/opal/mca/rcache/base/rcache_base_frame.c b/opal/mca/rcache/base/rcache_base_frame.c index 4bd035ead85..7971e310cae 100644 --- a/opal/mca/rcache/base/rcache_base_frame.c +++ b/opal/mca/rcache/base/rcache_base_frame.c @@ -57,8 +57,8 @@ static void mca_rcache_base_registration_constructor(mca_rcache_base_registratio reg->rcache = NULL; reg->base = NULL; reg->bound = NULL; - reg->ref_count = 0; - reg->flags = 0; + opal_atomic_store_32_relaxed(®->ref_count, 0); + opal_atomic_store_uint32_relaxed(®->flags, 0); } OBJ_CLASS_INSTANCE(mca_rcache_base_registration_t, opal_free_list_item_t, diff --git a/opal/mca/rcache/base/rcache_base_vma_tree.c b/opal/mca/rcache/base/rcache_base_vma_tree.c index 2dc5e0d70e8..31ecaadcbd7 100644 --- a/opal/mca/rcache/base/rcache_base_vma_tree.c +++ b/opal/mca/rcache/base/rcache_base_vma_tree.c @@ -144,7 +144,7 @@ static int mca_rcache_base_tree_dump_range_helper(uint64_t low, uint64_t high, v mca_rcache_base_registration_t *reg = (mca_rcache_base_registration_t *) data; opal_output(0, " reg: base=%p, bound=%p, ref_count=%d, flags=0x%x", (void *) reg->base, - (void *) reg->bound, reg->ref_count, reg->flags); + (void *) reg->bound, opal_atomic_load_32_relaxed(®->ref_count), opal_atomic_load_uint32_relaxed(®->flags)); return OPAL_SUCCESS; } diff --git a/opal/mca/threads/base/wait_sync.c b/opal/mca/threads/base/wait_sync.c index 2458a9614be..7e932c807fa 100644 --- a/opal/mca/threads/base/wait_sync.c +++ b/opal/mca/threads/base/wait_sync.c @@ -47,7 +47,7 @@ void opal_threads_base_wait_sync_global_wakeup_mt(int status) opal_mutex_unlock(&wait_sync_lock); } -static opal_atomic_int32_t num_thread_in_progress = 0; +static opal_atomic_int32_t num_thread_in_progress = { .value = 0 }; #define WAIT_SYNC_PASS_OWNERSHIP(who) \ do { \ @@ -62,7 +62,7 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync) * race condition around the release of the synchronization using the * signaling field. */ - if (sync->count <= 0) { + if (opal_atomic_load_32_relaxed(&sync->count) <= 0) { return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR; } @@ -72,7 +72,7 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync) /* Now that we hold the lock make sure another thread has not already * call cond_signal. */ - if (sync->count <= 0) { + if (opal_atomic_load_32_relaxed(&sync->count) <= 0) { opal_thread_internal_mutex_unlock(&sync->lock); return (0 == sync->status) ? OPAL_SUCCESS : OPAL_ERROR; } @@ -97,7 +97,7 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync) * - our sync has been triggered. */ check_status: - if (sync != opal_threads_base_wait_sync_list && num_thread_in_progress >= opal_max_thread_in_progress) { + if (sync != opal_threads_base_wait_sync_list && opal_atomic_load_32_relaxed(&num_thread_in_progress) >= opal_max_thread_in_progress) { opal_thread_internal_cond_wait(&sync->condition, &sync->lock); /** @@ -106,7 +106,7 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync) * promoted as the progress manager. */ - if (sync->count <= 0) { /* Completed? */ + if (opal_atomic_load_32_relaxed(&sync->count) <= 0) { /* Completed? */ opal_thread_internal_mutex_unlock(&sync->lock); goto i_am_done; } @@ -116,7 +116,7 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync) opal_thread_internal_mutex_unlock(&sync->lock); OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, 1); - while (sync->count > 0) { /* progress till completion */ + while (opal_atomic_load_32_relaxed(&sync->count) > 0) { /* progress till completion */ /* don't progress with the sync lock locked or you'll deadlock */ opal_progress(); } diff --git a/opal/mca/threads/thread_usage.h b/opal/mca/threads/thread_usage.h index 4e2fd75a7e1..4f17d3e89d1 100644 --- a/opal/mca/threads/thread_usage.h +++ b/opal/mca/threads/thread_usage.h @@ -101,8 +101,8 @@ static inline bool opal_set_using_threads(bool have) return opal_atomic_##name##_fetch_##suffix(addr, delta); \ } \ \ - *addr = *addr operator delta; \ - return *addr; \ + opal_atomic_store_##suffix##_relaxed(addr, opal_atomic_load_##suffix##_relaxed(addr) operator delta); \ + return opal_atomic_load_##suffix##_relaxed(addr); \ } \ \ static inline type opal_thread_fetch_##name##_##suffix(opal_atomic_##type *addr, type delta) \ @@ -111,8 +111,8 @@ static inline bool opal_set_using_threads(bool have) return opal_atomic_fetch_##name##_##suffix(addr, delta); \ } \ \ - type old = *addr; \ - *addr = old operator delta; \ + type old = opal_atomic_load_##suffix##_relaxed(addr); \ + opal_atomic_store_##suffix##_relaxed(addr, old operator delta); \ return old; \ } @@ -125,12 +125,12 @@ static inline bool opal_set_using_threads(bool have) (addr_type) value); \ } \ \ - if ((type) *addr == *compare) { \ - ((type *) addr)[0] = value; \ + if ((type) opal_atomic_load_##suffix##_relaxed(addr) == *compare) { \ + opal_atomic_store_##suffix##_relaxed(addr, value); \ return true; \ } \ \ - *compare = ((type *) addr)[0]; \ + *compare = opal_atomic_load_##suffix##_relaxed(addr); \ \ return false; \ } @@ -142,8 +142,8 @@ static inline bool opal_set_using_threads(bool have) return (type) opal_atomic_swap_##suffix(ptr, (addr_type) newvalue); \ } \ \ - type old = ((type *) ptr)[0]; \ - ((type *) ptr)[0] = newvalue; \ + type old = opal_atomic_load_##suffix##_relaxed(ptr); \ + opal_atomic_store_##suffix##_relaxed(ptr, newvalue); \ \ return old; \ } diff --git a/opal/mca/threads/wait_sync.h b/opal/mca/threads/wait_sync.h index c77724bacbb..2dcb9cb7daa 100644 --- a/opal/mca/threads/wait_sync.h +++ b/opal/mca/threads/wait_sync.h @@ -100,7 +100,7 @@ static inline int sync_wait_st(ompi_wait_sync_t *sync) assert(NULL == sync->next); opal_threads_base_wait_sync_list = sync; - while (sync->count > 0) { + while (opal_atomic_load_32_relaxed(&sync->count) > 0) { opal_progress(); } opal_threads_base_wait_sync_list = NULL;