@@ -95,11 +95,25 @@ Apple targets have historically being problematic, xcode 4.6 would miscompile th
9595
9696#include <stdatomic.h>
9797
98+ #if defined(HOST_ARM64 )
99+ // C11 atomics on ARM64 offers a weaker version of sequential consistent, not expected by mono atomics operations.
100+ // C11 seq_cst on ARM64 corresponds to acquire/release semantics, but mono expects these functions to emit a full memory
101+ // barrier preventing any kind of reordering around the atomic operation. GCC atomics on ARM64 had similar limitations,
102+ // see comments on GCC atomics below and mono injected full memory barriers around GCC atomic functions to mitigate this.
103+ // Since mono GCC atomics implementation ended up even stronger (full memory barrier before/after), the C11 atomics
104+ // implementation is still a little weaker, but should correspond to the exact same semantics as implemented by JIT
105+ // compiler for sequential consistent atomic load/store/add/exchange/cas op codes on ARM64.
106+ #define C11_MEMORY_ORDER_SEQ_CST () atomic_thread_fence (memory_order_seq_cst)
107+ #else
108+ #define C11_MEMORY_ORDER_SEQ_CST ()
109+ #endif
110+
98111static inline guint8
99112mono_atomic_cas_u8 (volatile guint8 * dest , guint8 exch , guint8 comp )
100113{
101114 g_static_assert (sizeof (atomic_char ) == sizeof (* dest ) && ATOMIC_CHAR_LOCK_FREE == 2 );
102115 (void )atomic_compare_exchange_strong ((volatile atomic_char * )dest , (char * )& comp , exch );
116+ C11_MEMORY_ORDER_SEQ_CST ();
103117 return comp ;
104118}
105119
@@ -108,6 +122,7 @@ mono_atomic_cas_u16 (volatile guint16 *dest, guint16 exch, guint16 comp)
108122{
109123 g_static_assert (sizeof (atomic_short ) == sizeof (* dest ) && ATOMIC_SHORT_LOCK_FREE == 2 );
110124 (void )atomic_compare_exchange_strong ((volatile atomic_short * )dest , (short * )& comp , exch );
125+ C11_MEMORY_ORDER_SEQ_CST ();
111126 return comp ;
112127}
113128
@@ -116,6 +131,7 @@ mono_atomic_cas_i32 (volatile gint32 *dest, gint32 exch, gint32 comp)
116131{
117132 g_static_assert (sizeof (atomic_int ) == sizeof (* dest ) && ATOMIC_INT_LOCK_FREE == 2 );
118133 (void )atomic_compare_exchange_strong ((volatile atomic_int * )dest , & comp , exch );
134+ C11_MEMORY_ORDER_SEQ_CST ();
119135 return comp ;
120136}
121137
@@ -125,21 +141,22 @@ mono_atomic_cas_i64 (volatile gint64 *dest, gint64 exch, gint64 comp)
125141#if SIZEOF_LONG == 8
126142 g_static_assert (sizeof (atomic_long ) == sizeof (* dest ) && ATOMIC_LONG_LOCK_FREE == 2 );
127143 (void )atomic_compare_exchange_strong ((volatile atomic_long * )dest , (long * )& comp , exch );
128- return comp ;
129144#elif SIZEOF_LONG_LONG == 8
130145 g_static_assert (sizeof (atomic_llong ) == sizeof (* dest ) && ATOMIC_LLONG_LOCK_FREE == 2 );
131146 (void )atomic_compare_exchange_strong ((volatile atomic_llong * )dest , (long long * )& comp , exch );
132- return comp ;
133147#else
134148#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
135149#endif
150+ C11_MEMORY_ORDER_SEQ_CST ();
151+ return comp ;
136152}
137153
138154static inline gpointer
139155mono_atomic_cas_ptr (volatile gpointer * dest , gpointer exch , gpointer comp )
140156{
141157 g_static_assert (ATOMIC_POINTER_LOCK_FREE == 2 );
142158 (void )atomic_compare_exchange_strong ((volatile _Atomic (gpointer ) * )dest , & comp , exch );
159+ C11_MEMORY_ORDER_SEQ_CST ();
143160 return comp ;
144161}
145162
@@ -191,125 +208,153 @@ static inline guint8
191208mono_atomic_xchg_u8 (volatile guint8 * dest , guint8 exch )
192209{
193210 g_static_assert (sizeof (atomic_char ) == sizeof (* dest ) && ATOMIC_CHAR_LOCK_FREE == 2 );
194- return atomic_exchange ((volatile atomic_char * )dest , exch );
211+ guint8 old = atomic_exchange ((volatile atomic_char * )dest , exch );
212+ C11_MEMORY_ORDER_SEQ_CST ();
213+ return old ;
195214}
196215
197216static inline guint16
198217mono_atomic_xchg_u16 (volatile guint16 * dest , guint16 exch )
199218{
200219 g_static_assert (sizeof (atomic_short ) == sizeof (* dest ) && ATOMIC_SHORT_LOCK_FREE == 2 );
201- return atomic_exchange ((volatile atomic_short * )dest , exch );
220+ guint16 old = atomic_exchange ((volatile atomic_short * )dest , exch );
221+ C11_MEMORY_ORDER_SEQ_CST ();
222+ return old ;
202223}
203224
204225static inline gint32
205226mono_atomic_xchg_i32 (volatile gint32 * dest , gint32 exch )
206227{
207228 g_static_assert (sizeof (atomic_int ) == sizeof (* dest ) && ATOMIC_INT_LOCK_FREE == 2 );
208- return atomic_exchange ((volatile atomic_int * )dest , exch );
229+ gint32 old = atomic_exchange ((volatile atomic_int * )dest , exch );
230+ C11_MEMORY_ORDER_SEQ_CST ();
231+ return old ;
209232}
210233
211234static inline gint64
212235mono_atomic_xchg_i64 (volatile gint64 * dest , gint64 exch )
213236{
214237#if SIZEOF_LONG == 8
215238 g_static_assert (sizeof (atomic_long ) == sizeof (* dest ) && ATOMIC_LONG_LOCK_FREE == 2 );
216- return atomic_exchange ((volatile atomic_long * )dest , exch );
239+ gint64 old = atomic_exchange ((volatile atomic_long * )dest , exch );
217240#elif SIZEOF_LONG_LONG == 8
218241 g_static_assert (sizeof (atomic_llong ) == sizeof (* dest ) && ATOMIC_LLONG_LOCK_FREE == 2 );
219- return atomic_exchange ((volatile atomic_llong * )dest , exch );
242+ gint64 old = atomic_exchange ((volatile atomic_llong * )dest , exch );
220243#else
221244#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
222245#endif
246+ C11_MEMORY_ORDER_SEQ_CST ();
247+ return old ;
223248}
224249
225250static inline gpointer
226251mono_atomic_xchg_ptr (volatile gpointer * dest , gpointer exch )
227252{
228253 g_static_assert (ATOMIC_POINTER_LOCK_FREE == 2 );
229- return atomic_exchange ((volatile _Atomic (gpointer ) * )dest , exch );
254+ gpointer old = atomic_exchange ((volatile _Atomic (gpointer ) * )dest , exch );
255+ C11_MEMORY_ORDER_SEQ_CST ();
256+ return old ;
230257}
231258
232259static inline gint32
233260mono_atomic_fetch_add_i32 (volatile gint32 * dest , gint32 add )
234261{
235262 g_static_assert (sizeof (atomic_int ) == sizeof (* dest ) && ATOMIC_INT_LOCK_FREE == 2 );
236- return atomic_fetch_add ((volatile atomic_int * )dest , add );
263+ gint32 old = atomic_fetch_add ((volatile atomic_int * )dest , add );
264+ C11_MEMORY_ORDER_SEQ_CST ();
265+ return old ;
237266}
238267
239268static inline gint64
240269mono_atomic_fetch_add_i64 (volatile gint64 * dest , gint64 add )
241270{
242271#if SIZEOF_LONG == 8
243272 g_static_assert (sizeof (atomic_long ) == sizeof (* dest ) && ATOMIC_LONG_LOCK_FREE == 2 );
244- return atomic_fetch_add ((volatile atomic_long * )dest , add );
273+ gint64 old = atomic_fetch_add ((volatile atomic_long * )dest , add );
245274#elif SIZEOF_LONG_LONG == 8
246275 g_static_assert (sizeof (atomic_llong ) == sizeof (* dest ) && ATOMIC_LLONG_LOCK_FREE == 2 );
247- return atomic_fetch_add ((volatile atomic_llong * )dest , add );
276+ gint64 old = atomic_fetch_add ((volatile atomic_llong * )dest , add );
248277#else
249278#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
250279#endif
280+ C11_MEMORY_ORDER_SEQ_CST ();
281+ return old ;
251282}
252283
253284static inline gint8
254285mono_atomic_load_i8 (volatile gint8 * src )
255286{
256287 g_static_assert (sizeof (atomic_char ) == sizeof (* src ) && ATOMIC_CHAR_LOCK_FREE == 2 );
257- return atomic_load ((volatile atomic_char * )src );
288+ C11_MEMORY_ORDER_SEQ_CST ();
289+ gint8 val = atomic_load ((volatile atomic_char * )src );
290+ return val ;
258291}
259292
260293static inline gint16
261294mono_atomic_load_i16 (volatile gint16 * src )
262295{
263296 g_static_assert (sizeof (atomic_short ) == sizeof (* src ) && ATOMIC_SHORT_LOCK_FREE == 2 );
264- return atomic_load ((volatile atomic_short * )src );
297+ C11_MEMORY_ORDER_SEQ_CST ();
298+ gint16 val = atomic_load ((volatile atomic_short * )src );
299+ return val ;
265300}
266301
267302static inline gint32 mono_atomic_load_i32 (volatile gint32 * src )
268303{
269304 g_static_assert (sizeof (atomic_int ) == sizeof (* src ) && ATOMIC_INT_LOCK_FREE == 2 );
270- return atomic_load ((volatile atomic_int * )src );
305+ C11_MEMORY_ORDER_SEQ_CST ();
306+ gint32 val = atomic_load ((volatile atomic_int * )src );
307+ return val ;
271308}
272309
273310static inline gint64
274311mono_atomic_load_i64 (volatile gint64 * src )
275312{
276313#if SIZEOF_LONG == 8
277314 g_static_assert (sizeof (atomic_long ) == sizeof (* src ) && ATOMIC_LONG_LOCK_FREE == 2 );
278- return atomic_load ((volatile atomic_long * )src );
315+ C11_MEMORY_ORDER_SEQ_CST ();
316+ gint64 val = atomic_load ((volatile atomic_long * )src );
279317#elif SIZEOF_LONG_LONG == 8
280318 g_static_assert (sizeof (atomic_llong ) == sizeof (* src ) && ATOMIC_LLONG_LOCK_FREE == 2 );
281- return atomic_load ((volatile atomic_llong * )src );
319+ C11_MEMORY_ORDER_SEQ_CST ();
320+ gint64 val = atomic_load ((volatile atomic_llong * )src );
282321#else
283322#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
284323#endif
324+ return val ;
285325}
286326
287327static inline gpointer
288328mono_atomic_load_ptr (volatile gpointer * src )
289329{
290330 g_static_assert (ATOMIC_POINTER_LOCK_FREE == 2 );
291- return atomic_load ((volatile _Atomic (gpointer ) * )src );
331+ C11_MEMORY_ORDER_SEQ_CST ();
332+ gpointer val = atomic_load ((volatile _Atomic (gpointer ) * )src );
333+ return val ;
292334}
293335
294336static inline void
295337mono_atomic_store_i8 (volatile gint8 * dst , gint8 val )
296338{
297339 g_static_assert (sizeof (atomic_char ) == sizeof (* dst ) && ATOMIC_CHAR_LOCK_FREE == 2 );
298340 atomic_store ((volatile atomic_char * )dst , val );
341+ C11_MEMORY_ORDER_SEQ_CST ();
299342}
300343
301344static inline void
302345mono_atomic_store_i16 (volatile gint16 * dst , gint16 val )
303346{
304347 g_static_assert (sizeof (atomic_short ) == sizeof (* dst ) && ATOMIC_SHORT_LOCK_FREE == 2 );
305348 atomic_store ((volatile atomic_short * )dst , val );
349+ C11_MEMORY_ORDER_SEQ_CST ();
306350}
307351
308352static inline void
309353mono_atomic_store_i32 (volatile gint32 * dst , gint32 val )
310354{
311355 g_static_assert (sizeof (atomic_int ) == sizeof (* dst ) && ATOMIC_INT_LOCK_FREE == 2 );
312356 atomic_store ((atomic_int * )dst , val );
357+ C11_MEMORY_ORDER_SEQ_CST ();
313358}
314359
315360static inline void
@@ -324,13 +369,15 @@ mono_atomic_store_i64 (volatile gint64 *dst, gint64 val)
324369#else
325370#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
326371#endif
372+ C11_MEMORY_ORDER_SEQ_CST ();
327373}
328374
329375static inline void
330376mono_atomic_store_ptr (volatile gpointer * dst , gpointer val )
331377{
332378 g_static_assert (ATOMIC_POINTER_LOCK_FREE == 2 );
333379 atomic_store ((volatile _Atomic (gpointer ) * )dst , val );
380+ C11_MEMORY_ORDER_SEQ_CST ();
334381}
335382
336383#elif defined(MONO_USE_WIN32_ATOMIC )
0 commit comments