Skip to content

Commit 14c5d15

Browse files
bwbarrettbwbarrett
committed
Refactor LL/SC interface
Unlike other interfaces, the LL/SC interface must be optional, as many platforms do not provide LL/SC support. Document the interface a bit better, and move the PowerPC LL/SC interface into its own header file so that it can be used with C11/GCC builtin atomics, similar to the ARM platform. Signed-off-by: Brian Barrett <[email protected]>
1 parent 06e49b9 commit 14c5d15

File tree

8 files changed

+237
-101
lines changed

8 files changed

+237
-101
lines changed

‎opal/include/opal/sys/Makefile.am

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ headers += \
3333
opal/sys/atomic_stdc.h \
3434
opal/sys/atomic_impl_minmax_math.h \
3535
opal/sys/atomic_impl_ptr_cswap.h \
36+
opal/sys/atomic_impl_ptr_llsc.h \
3637
opal/sys/atomic_impl_ptr_swap.h \
3738
opal/sys/atomic_impl_size_t_math.h \
3839
opal/sys/atomic_impl_spinlock.h \

‎opal/include/opal/sys/arm64/atomic_llsc.h

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,16 @@
2525
* $HEADER$
2626
*/
2727

28-
#if !defined(OPAL_SYS_ARCH_ATOMIC_LLSC_H)
28+
#ifndef OPAL_SYS_ARCH_ATOMIC_LLSC_H
29+
#define OPAL_SYS_ARCH_ATOMIC_LLSC_H 1
2930

30-
# define OPAL_SYS_ARCH_ATOMIC_LLSC_H
31+
/*
32+
* this file is included even when C11 or GCC built-in atomics are
33+
* used, which is why we must check for gcc inline assembly support.
34+
*/
3135

3236
# if OPAL_C_GCC_INLINE_ASSEMBLY
3337

34-
# undef OPAL_HAVE_ATOMIC_LLSC_32
35-
# undef OPAL_HAVE_ATOMIC_LLSC_64
36-
3738
# define OPAL_HAVE_ATOMIC_LLSC_32 1
3839
# define OPAL_HAVE_ATOMIC_LLSC_64 1
3940

@@ -79,6 +80,8 @@
7980
ret = (_ret == 0); \
8081
} while (0)
8182

83+
#include "opal/sys/atomic_impl_ptr_llsc.h"
84+
8285
# endif /* OPAL_C_GCC_INLINE_ASSEMBLY */
8386

8487
#endif /* ! OPAL_SYS_ARCH_ATOMIC_LLSC_H */

‎opal/include/opal/sys/atomic.h

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -412,49 +412,52 @@ static inline void opal_atomic_add(type *addr, type delta);
412412

413413
#endif /* !OPAL_C_HAVE__ATOMIC */
414414

415-
/****** load-linked, store-conditional atomic implementations ******/
416415

417-
/* C11 atomics do not expose the low-level load-linked, store-conditional
418-
* instructions. Open MPI can use these instructions to implement a more
419-
* efficient version of the lock-free lifo and fifo. On Apple Silicon the
420-
* LL/SC fifo and lifo are ~ 2-20x faster than the CAS128 implementation. */
421-
#if OPAL_ASSEMBLY_ARCH == OPAL_ARM64
422-
# include "opal/sys/arm64/atomic_llsc.h"
423-
#endif
416+
/**********************************************************************
417+
*
418+
* Load-linked, Store Conditional
419+
*
420+
* Optional. Check OPAL_HAVE_ATOMIC_LLSC_32,
421+
* OPAL_HAVE_ATOMIC_LLSC_64, or OPAL_HAVE_ATOMIC_LLSC_PTR before
422+
* using. Implemented as macros due to function call behaviors;
423+
* prototyped here as C++-style fuctions for readability.
424+
*
425+
* C11 and GCC built-in atomics don't provide native LL/SC support, so
426+
* if there is an architectual implementation, we use it even if
427+
* we are using the C11 or GCC built-in atomics.
428+
*
429+
*********************************************************************/
424430

425-
#if !defined(OPAL_HAVE_ATOMIC_LLSC_32)
426-
# define OPAL_HAVE_ATOMIC_LLSC_32 0
427-
#endif
431+
#ifdef DOXYGEN
428432

429-
#if !defined(OPAL_HAVE_ATOMIC_LLSC_64)
430-
# define OPAL_HAVE_ATOMIC_LLSC_64 0
431-
#endif
432-
433-
#if (OPAL_HAVE_ATOMIC_LLSC_32 || OPAL_HAVE_ATOMIC_LLSC_64)
433+
static inline void opal_atomic_ll_32(opal_atomic_int32_t *addr, int32_t &ret);
434434

435-
# if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_LLSC_32
435+
static inline void opal_atomic_sc_32(opal_atomic_int32_t *addr, int32_t newval, int &ret);
436436

437-
# define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((opal_atomic_int32_t *) (addr), ret)
438-
# define opal_atomic_sc_ptr(addr, value, ret) \
439-
opal_atomic_sc_32((opal_atomic_int32_t *) (addr), (intptr_t)(value), ret)
437+
static inline void opal_atomic_ll_64(opal_atomic_int64_t *addr, int64_t &ret);
440438

441-
# define OPAL_HAVE_ATOMIC_LLSC_PTR 1
439+
static inline void opal_atomic_sc_64(opal_atomic_int64_t *addr, int64_t newval, int &ret);
442440

443-
# elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64
441+
static inline void opal_atomic_ll_ptr(opal_atomic_intptr_t *addr, intptr_t &ret);
444442

445-
# define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((opal_atomic_int64_t *) (addr), ret)
446-
# define opal_atomic_sc_ptr(addr, value, ret) \
447-
opal_atomic_sc_64((opal_atomic_int64_t *) (addr), (intptr_t)(value), ret)
443+
static inline void opal_atomic_sc_ptr(opal_atomic_intptr_t *addr, intptr_t newval, int &ret);
448444

449-
# define OPAL_HAVE_ATOMIC_LLSC_PTR 1
445+
#endif
450446

451-
# endif
447+
#if OPAL_ASSEMBLY_ARCH == OPAL_ARM64
448+
# include "opal/sys/arm64/atomic_llsc.h"
449+
#elif OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64
450+
# include "opal/sys/powerpc/atomic_llsc.h"
451+
#endif
452452

453-
#else
453+
#if !defined(OPAL_HAVE_ATOMIC_LLSC_32)
454+
# define OPAL_HAVE_ATOMIC_LLSC_32 0
455+
#endif
454456

455-
# define OPAL_HAVE_ATOMIC_LLSC_PTR 0
457+
#if !defined(OPAL_HAVE_ATOMIC_LLSC_64)
458+
# define OPAL_HAVE_ATOMIC_LLSC_64 0
459+
#endif
456460

457-
#endif /* (OPAL_HAVE_ATOMIC_LLSC_32 || OPAL_HAVE_ATOMIC_LLSC_64)*/
458461

459462
END_C_DECLS
460463

‎opal/include/opal/sys/atomic_impl_ptr_llsc.h

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
3+
* University Research and Technology
4+
* Corporation. All rights reserved.
5+
* Copyright (c) 2004-2006 The University of Tennessee and The University
6+
* of Tennessee Research Foundation. All rights
7+
* reserved.
8+
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9+
* University of Stuttgart. All rights reserved.
10+
* Copyright (c) 2004-2005 The Regents of the University of California.
11+
* All rights reserved.
12+
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
13+
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
14+
* Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
15+
* reserved.
16+
* Copyright (c) 2017 Research Organization for Information Science
17+
* and Technology (RIST). All rights reserved.
18+
* Copyright (c) 2020-2021 Google, LLC. All rights reserved.
19+
* Copyright (c) 2022 Amazon.com, Inc. or its affiliates.
20+
* All Rights reserved.
21+
* $COPYRIGHT$
22+
*
23+
* Additional copyrights may follow
24+
*
25+
* $HEADER$
26+
*
27+
* Pointer-sized wrapper for LL/SC calls, wrappers around size-defined
28+
* calls. Note that these must be macros, as LL/SC may not work
29+
* across function calls.
30+
*/
31+
32+
33+
#ifndef ATOMIC_IMPL_PTR_LLSC_H
34+
#define ATOMIC_IMPL_PTR_LLSC_H 1
35+
36+
#if SIZEOF_VOID_P == 4 && defined(OPAL_HAVE_ATOMIC_LLSC_32) && OPAL_HAVE_ATOMIC_LLSC_32
37+
38+
# define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((opal_atomic_int32_t *) (addr), ret)
39+
# define opal_atomic_sc_ptr(addr, value, ret) \
40+
opal_atomic_sc_32((opal_atomic_int32_t *) (addr), (intptr_t)(value), ret)
41+
42+
# define OPAL_HAVE_ATOMIC_LLSC_PTR 1
43+
44+
#elif SIZEOF_VOID_P == 8 && defined(OPAL_HAVE_ATOMIC_LLSC_64) && OPAL_HAVE_ATOMIC_LLSC_64
45+
46+
# define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((opal_atomic_int64_t *) (addr), ret)
47+
# define opal_atomic_sc_ptr(addr, value, ret) \
48+
opal_atomic_sc_64((opal_atomic_int64_t *) (addr), (intptr_t)(value), ret)
49+
50+
# define OPAL_HAVE_ATOMIC_LLSC_PTR 1
51+
52+
#endif /* SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64 */
53+
54+
#endif /* ATOMIC_IMPL_PTR_LLSC_H */

‎opal/include/opal/sys/powerpc/Makefile.am

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,6 @@
2020

2121
headers += \
2222
opal/sys/powerpc/atomic.h \
23+
opal/sys/powerpc/atomic_helper.h \
24+
opal/sys/powerpc/atomic_llsc.h \
2325
opal/sys/powerpc/timer.h

‎opal/include/opal/sys/powerpc/atomic.h

Lines changed: 1 addition & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -30,23 +30,7 @@
3030
* On powerpc ...
3131
*/
3232

33-
#if defined(__xlC__) || defined(__IBMC__) || defined(__IBMCPP__) || defined(__ibmxl__)
34-
/* work-around bizzare xlc bug in which it sign-extends
35-
a pointer to a 32-bit signed integer */
36-
# define OPAL_ASM_ADDR(a) ((uintptr_t) a)
37-
#else
38-
# define OPAL_ASM_ADDR(a) (a)
39-
#endif
40-
41-
#if defined(__PGI)
42-
/* work-around for bug in PGI 16.5-16.7 where the compiler fails to
43-
* correctly emit load instructions for 64-bit operands. without this
44-
* it will emit lwz instead of ld to load the 64-bit operand. */
45-
# define OPAL_ASM_VALUE64(x) (void *) (intptr_t)(x)
46-
#else
47-
# define OPAL_ASM_VALUE64(x) x
48-
#endif
49-
33+
#include "opal/sys/powerpc/atomic_helper.h"
5034

5135
/**********************************************************************
5236
*
@@ -291,53 +275,4 @@ OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf)
291275
#include "opal/sys/atomic_impl_minmax_math.h"
292276
#include "opal/sys/atomic_impl_size_t_math.h"
293277

294-
/* NTH: the LL/SC support is done through macros due to issues with non-optimized builds. The reason
295-
* is that even with an always_inline attribute the compiler may still emit instructions to store
296-
* then load the arguments to/from the stack. This sequence may cause the ll reservation to be
297-
* cancelled. */
298-
#define opal_atomic_ll_32(addr, ret) \
299-
do { \
300-
opal_atomic_int32_t *_addr = (addr); \
301-
__asm__ __volatile__("lwarx %0, 0, %1 \n\t" : "=&r"(ret) : "r"(_addr)); \
302-
} while (0)
303-
304-
#define opal_atomic_sc_32(addr, value, ret) \
305-
do { \
306-
opal_atomic_int32_t *_addr = (addr); \
307-
int32_t _ret, _foo, _newval = (int32_t) value; \
308-
\
309-
__asm__ __volatile__(" stwcx. %4, 0, %3 \n\t" \
310-
" li %0,0 \n\t" \
311-
" bne- 1f \n\t" \
312-
" ori %0,%0,1 \n\t" \
313-
"1:" \
314-
: "=r"(_ret), "=m"(*_addr), "=r"(_foo) \
315-
: "r"(_addr), "r"(_newval) \
316-
: "cc", "memory"); \
317-
ret = _ret; \
318-
} while (0)
319-
320-
#define opal_atomic_ll_64(addr, ret) \
321-
do { \
322-
opal_atomic_int64_t *_addr = (addr); \
323-
__asm__ __volatile__("ldarx %0, 0, %1 \n\t" : "=&r"(ret) : "r"(_addr)); \
324-
} while (0)
325-
326-
#define opal_atomic_sc_64(addr, value, ret) \
327-
do { \
328-
opal_atomic_int64_t *_addr = (addr); \
329-
int64_t _newval = (int64_t) value; \
330-
int32_t _ret; \
331-
\
332-
__asm__ __volatile__(" stdcx. %2, 0, %1 \n\t" \
333-
" li %0,0 \n\t" \
334-
" bne- 1f \n\t" \
335-
" ori %0,%0,1 \n\t" \
336-
"1:" \
337-
: "=r"(_ret) \
338-
: "r"(_addr), "r"(OPAL_ASM_VALUE64(_newval)) \
339-
: "cc", "memory"); \
340-
ret = _ret; \
341-
} while (0)
342-
343278
#endif /* ! OPAL_SYS_ARCH_ATOMIC_H */

‎opal/include/opal/sys/powerpc/atomic_helper.h

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
> * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4+
* University Research and Technology
5+
* Corporation. All rights reserved.
6+
* Copyright (c) 2004-2005 The University of Tennessee and The University
7+
* of Tennessee Research Foundation. All rights
8+
* reserved.
9+
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10+
* University of Stuttgart. All rights reserved.
11+
* Copyright (c) 2004-2005 The Regents of the University of California.
12+
* All rights reserved.
13+
* Copyright (c) 2010-2021 IBM Corporation. All rights reserved.
14+
* Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
15+
* reserved.
16+
* Copyright (c) 2021 Google, LLC. All rights reserved.
17+
* $COPYRIGHT$
18+
*
19+
* Additional copyrights may follow
20+
*
21+
* $HEADER$
22+
*/
23+
24+
#ifndef OPAL_SYS_ARCH_ATOMIC_HELPER_H
25+
#define OPAL_SYS_ARCH_ATOMIC_HELPER_H 1
26+
27+
#if defined(__xlC__) || defined(__IBMC__) || defined(__IBMCPP__) || defined(__ibmxl__)
28+
/* work-around bizzare xlc bug in which it sign-extends
29+
a pointer to a 32-bit signed integer */
30+
# define OPAL_ASM_ADDR(a) ((uintptr_t) a)
31+
#else
32+
# define OPAL_ASM_ADDR(a) (a)
33+
#endif
34+
35+
#if defined(__PGI)
36+
/* work-around for bug in PGI 16.5-16.7 where the compiler fails to
37+
* correctly emit load instructions for 64-bit operands. without this
38+
* it will emit lwz instead of ld to load the 64-bit operand. */
39+
# define OPAL_ASM_VALUE64(x) (void *) (intptr_t)(x)
40+
#else
41+
# define OPAL_ASM_VALUE64(x) x
42+
#endif
43+
44+
#endif /* OPAL_SYS_ARCH_ATOMIC_HELPER_H */

0 commit comments

Comments
 (0)