Skip to content

Commit 06e49b9

Browse files
bwbarrettbwbarrett
committed
Refactor atomic math interface
The atomic math interface has become required based on usage, so embrace the requirement and remove all the gorp to make the interface optional. Split out the compatibility implementation into headers that can be included as required by implementations, saving us from a mess of #defines in atomic_impl.h. Reorganize the implementation headers so that the code is in the same order across implementations, for readability. Signed-off-by: Brian Barrett <[email protected]>
1 parent e9d7bb9 commit 06e49b9

File tree

11 files changed

+478
-731
lines changed

11 files changed

+478
-731
lines changed

‎opal/include/opal/sys/Makefile.am

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,10 @@ headers += \
3131
opal/sys/architecture.h \
3232
opal/sys/atomic.h \
3333
opal/sys/atomic_stdc.h \
34-
opal/sys/atomic_impl.h \
34+
opal/sys/atomic_impl_minmax_math.h \
3535
opal/sys/atomic_impl_ptr_cswap.h \
3636
opal/sys/atomic_impl_ptr_swap.h \
37+
opal/sys/atomic_impl_size_t_math.h \
3738
opal/sys/atomic_impl_spinlock.h \
3839
opal/sys/timer.h \
3940
opal/sys/cma.h

‎opal/include/opal/sys/arm64/atomic.h

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,6 @@
3030
#ifndef OPAL_SYS_ARCH_ATOMIC_H
3131
#define OPAL_SYS_ARCH_ATOMIC_H 1
3232

33-
#define OPAL_HAVE_ATOMIC_ADD_32 1
34-
#define OPAL_HAVE_ATOMIC_AND_32 1
35-
#define OPAL_HAVE_ATOMIC_OR_32 1
36-
#define OPAL_HAVE_ATOMIC_XOR_32 1
37-
#define OPAL_HAVE_ATOMIC_SUB_32 1
38-
#define OPAL_HAVE_ATOMIC_ADD_64 1
39-
#define OPAL_HAVE_ATOMIC_AND_64 1
40-
#define OPAL_HAVE_ATOMIC_OR_64 1
41-
#define OPAL_HAVE_ATOMIC_XOR_64 1
42-
#define OPAL_HAVE_ATOMIC_SUB_64 1
43-
4433

4534
/**********************************************************************
4635
*
@@ -286,17 +275,38 @@ static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t new
286275
: "cc", "memory"); \
287276
\
288277
return old; \
278+
} \
279+
static inline type opal_atomic_##name##_fetch_##bits(opal_atomic_##type *addr, \
280+
type value) \
281+
{ \
282+
type newval, old; \
283+
int32_t tmp; \
284+
\
285+
__asm__ __volatile__("1: ldxr %" reg "1, [%3] \n" \
286+
" " inst " %" reg "0, %" reg "1, %" reg "4 \n" \
287+
" stxr %w2, %" reg "0, [%3] \n" \
288+
" cbnz %w2, 1b \n" \
289+
: "=&r"(newval), "=&r"(old), "=&r"(tmp) \
290+
: "r"(addr), "r"(value) \
291+
: "cc", "memory"); \
292+
\
293+
return newval; \
289294
}
290295

291296
OPAL_ASM_MAKE_ATOMIC(int32_t, 32, add, "add", "w")
292297
OPAL_ASM_MAKE_ATOMIC(int32_t, 32, and, "and", "w")
293298
OPAL_ASM_MAKE_ATOMIC(int32_t, 32, or, "orr", "w")
294299
OPAL_ASM_MAKE_ATOMIC(int32_t, 32, xor, "eor", "w")
295300
OPAL_ASM_MAKE_ATOMIC(int32_t, 32, sub, "sub", "w")
301+
296302
OPAL_ASM_MAKE_ATOMIC(int64_t, 64, add, "add", "")
297303
OPAL_ASM_MAKE_ATOMIC(int64_t, 64, and, "and", "")
298304
OPAL_ASM_MAKE_ATOMIC(int64_t, 64, or, "orr", "")
299305
OPAL_ASM_MAKE_ATOMIC(int64_t, 64, xor, "eor", "")
300306
OPAL_ASM_MAKE_ATOMIC(int64_t, 64, sub, "sub", "")
301307

308+
#include "opal/sys/atomic_impl_minmax_math.h"
309+
#include "opal/sys/atomic_impl_size_t_math.h"
310+
311+
302312
#endif /* ! OPAL_SYS_ARCH_ATOMIC_H */

‎opal/include/opal/sys/atomic.h

Lines changed: 11 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -65,23 +65,6 @@
6565

6666
BEGIN_C_DECLS
6767

68-
/**********************************************************************
69-
*
70-
* Set or unset these macros in the architecture-specific atomic.h
71-
* files if we need to specify them as inline or non-inline
72-
*
73-
*********************************************************************/
74-
#define OPAL_HAVE_INLINE_ATOMIC_ADD_32 1
75-
#define OPAL_HAVE_INLINE_ATOMIC_AND_32 1
76-
#define OPAL_HAVE_INLINE_ATOMIC_OR_32 1
77-
#define OPAL_HAVE_INLINE_ATOMIC_XOR_32 1
78-
#define OPAL_HAVE_INLINE_ATOMIC_SUB_32 1
79-
#define OPAL_HAVE_INLINE_ATOMIC_ADD_64 1
80-
#define OPAL_HAVE_INLINE_ATOMIC_AND_64 1
81-
#define OPAL_HAVE_INLINE_ATOMIC_OR_64 1
82-
#define OPAL_HAVE_INLINE_ATOMIC_XOR_64 1
83-
#define OPAL_HAVE_INLINE_ATOMIC_SUB_64 1
84-
8568
/**********************************************************************
8669
*
8770
* Load the appropriate architecture files and set some reasonable
@@ -396,7 +379,6 @@ static inline int32_t opal_atomic_fetch_min_32(opal_atomic_int32_t *addr, int32_
396379
static inline int32_t opal_atomic_max_fetch_32(opal_atomic_int32_t *addr, int32_t value);
397380
static inline int32_t opal_atomic_fetch_max_32(opal_atomic_int32_t *addr, int32_t value);
398381

399-
400382
static inline int64_t opal_atomic_add_fetch_64(opal_atomic_int64_t *addr, int64_t delta);
401383
static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta);
402384
static inline int64_t opal_atomic_and_fetch_64(opal_atomic_int64_t *addr, int64_t value);
@@ -411,122 +393,22 @@ static inline int64_t opal_atomic_fetch_min_64(opal_atomic_int64_t *addr, int64_
411393
static inline int64_t opal_atomic_max_fetch_64(opal_atomic_int64_t *addr, int64_t value);
412394
static inline int64_t opal_atomic_fetch_max_64(opal_atomic_int64_t *addr, int64_t value);
413395

396+
static inline size_t opal_atomic_add_fetch_size_t(opal_atomic_size_t *addr, size_t delta);
397+
static inline size_t opal_atomic_fetch_add_size_t(opal_atomic_size_t *addr, size_t delta);
414398

415-
/* provide a size_t add/subtract. When in debug mode, make it an
416-
* inline function so that we don't have any casts in the
417-
* interface and can catch type errors. When not in debug mode,
418-
* just make it a macro, so that there's no performance penalty
419-
*/
420-
# if defined(DOXYGEN) || OPAL_ENABLE_DEBUG
421-
static inline size_t opal_atomic_add_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
422-
{
423-
# if SIZEOF_SIZE_T == 4
424-
return (size_t) opal_atomic_add_fetch_32((int32_t *) addr, delta);
425-
# elif SIZEOF_SIZE_T == 8
426-
return (size_t) opal_atomic_add_fetch_64((int64_t *) addr, delta);
427-
# else
428-
# error "Unknown size_t size"
429-
# endif
430-
}
431-
432-
static inline size_t opal_atomic_fetch_add_size_t(opal_atomic_size_t *addr, size_t delta)
433-
{
434-
# if SIZEOF_SIZE_T == 4
435-
return (size_t) opal_atomic_fetch_add_32((int32_t *) addr, delta);
436-
# elif SIZEOF_SIZE_T == 8
437-
return (size_t) opal_atomic_fetch_add_64((int64_t *) addr, delta);
438-
# else
439-
# error "Unknown size_t size"
440-
# endif
441-
}
442-
443-
static inline size_t opal_atomic_sub_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
444-
{
445-
# if SIZEOF_SIZE_T == 4
446-
return (size_t) opal_atomic_sub_fetch_32((int32_t *) addr, delta);
447-
# elif SIZEOF_SIZE_T == 8
448-
return (size_t) opal_atomic_sub_fetch_64((int64_t *) addr, delta);
449-
# else
450-
# error "Unknown size_t size"
451-
# endif
452-
}
453-
454-
static inline size_t opal_atomic_fetch_sub_size_t(opal_atomic_size_t *addr, size_t delta)
455-
{
456-
# if SIZEOF_SIZE_T == 4
457-
return (size_t) opal_atomic_fetch_sub_32((int32_t *) addr, delta);
458-
# elif SIZEOF_SIZE_T == 8
459-
return (size_t) opal_atomic_fetch_sub_64((int64_t *) addr, delta);
460-
# else
461-
# error "Unknown size_t size"
462-
# endif
463-
}
464-
465-
# else
466-
# if SIZEOF_SIZE_T == 4
467-
# define opal_atomic_add_fetch_size_t(addr, delta) \
468-
((size_t) opal_atomic_add_fetch_32((opal_atomic_int32_t *) addr, delta))
469-
# define opal_atomic_fetch_add_size_t(addr, delta) \
470-
((size_t) opal_atomic_fetch_add_32((opal_atomic_int32_t *) addr, delta))
471-
# define opal_atomic_sub_fetch_size_t(addr, delta) \
472-
((size_t) opal_atomic_sub_fetch_32((opal_atomic_int32_t *) addr, delta))
473-
# define opal_atomic_fetch_sub_size_t(addr, delta) \
474-
((size_t) opal_atomic_fetch_sub_32((opal_atomic_int32_t *) addr, delta))
475-
# elif SIZEOF_SIZE_T == 8
476-
# define opal_atomic_add_fetch_size_t(addr, delta) \
477-
((size_t) opal_atomic_add_fetch_64((opal_atomic_int64_t *) addr, delta))
478-
# define opal_atomic_fetch_add_size_t(addr, delta) \
479-
((size_t) opal_atomic_fetch_add_64((opal_atomic_int64_t *) addr, delta))
480-
# define opal_atomic_sub_fetch_size_t(addr, delta) \
481-
((size_t) opal_atomic_sub_fetch_64((opal_atomic_int64_t *) addr, delta))
482-
# define opal_atomic_fetch_sub_size_t(addr, delta) \
483-
((size_t) opal_atomic_fetch_sub_64((opal_atomic_int64_t *) addr, delta))
484-
# else
485-
# error "Unknown size_t size"
486-
# endif
487-
# endif
488-
489-
490-
static inline void opal_atomic_add_xx(opal_atomic_intptr_t *addr, int32_t value, size_t length);
491-
static inline void opal_atomic_sub_xx(opal_atomic_intptr_t *addr, int32_t value, size_t length);
492-
493-
static inline intptr_t opal_atomic_add_fetch_ptr(opal_atomic_intptr_t *addr, void *delta);
494-
static inline intptr_t opal_atomic_fetch_add_ptr(opal_atomic_intptr_t *addr, void *delta);
495-
static inline intptr_t opal_atomic_sub_fetch_ptr(opal_atomic_intptr_t *addr, void *delta);
496-
static inline intptr_t opal_atomic_fetch_sub_ptr(opal_atomic_intptr_t *addr, void *delta);
497-
399+
#ifdef DOXYGEN /* because this isn't a proper C prototype */
498400
/**
499-
* Atomically increment the content depending on the type. This
500-
* macro detect at compile time the type of the first argument
501-
* and choose the correct function to be called.
401+
* Atomically add delta to addr, type independent
502402
*
503-
* \note This macro should only be used for integer types.
403+
* @param addr Address of value to update
404+
* @param delta Value by which to change the value in addr
504405
*
505-
* @param addr Address of <TYPE>
506-
* @param delta Value to add (converted to <TYPE>).
406+
* Generally implemented as a macro (except for when implemented as a
407+
* compiler built-in), this function provides a type-independent math
408+
* operator.
507409
*/
508-
# define opal_atomic_add(ADDR, VALUE) \
509-
opal_atomic_add_xx((opal_atomic_intptr_t *) (ADDR), (int32_t)(VALUE), sizeof(*(ADDR)))
510-
511-
/**
512-
* Atomically decrement the content depending on the type. This
513-
* macro detect at compile time the type of the first argument
514-
* and choose the correct function to be called.
515-
*
516-
* \note This macro should only be used for integer types.
517-
*
518-
* @param addr Address of <TYPE>
519-
* @param delta Value to substract (converted to <TYPE>).
520-
*/
521-
# define opal_atomic_sub(ADDR, VALUE) \
522-
opal_atomic_sub_xx((opal_atomic_intptr_t *) (ADDR), (int32_t)(VALUE), sizeof(*(ADDR)))
523-
524-
525-
/*
526-
* Include inline implementations of everything not defined directly
527-
* in assembly
528-
*/
529-
# include "opal/sys/atomic_impl.h"
410+
static inline void opal_atomic_add(type *addr, type delta);
411+
#endif
530412

531413
#endif /* !OPAL_C_HAVE__ATOMIC */
532414

0 commit comments

Comments
 (0)