diff --git a/opal/class/opal_fifo.h b/opal/class/opal_fifo.h index ad9cbdbcbb4..41e68b6f2cc 100644 --- a/opal/class/opal_fifo.h +++ b/opal/class/opal_fifo.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2007 Voltaire All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. - * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights * reseved. * $COPYRIGHT$ * @@ -215,28 +215,33 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo, */ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) { - opal_list_item_t *item, *next; - #if OPAL_HAVE_ATOMIC_LLSC_PTR + register opal_list_item_t *ghost = &fifo->opal_fifo_ghost; + register opal_list_item_t *item, *next; + register opal_list_item_t volatile * volatile * head = &fifo->opal_fifo_head.data.item; + int ret; + /* use load-linked store-conditional to avoid ABA issues */ do { - item = opal_atomic_ll_ptr (&fifo->opal_fifo_head.data.item); - if (&fifo->opal_fifo_ghost == item) { - if (&fifo->opal_fifo_ghost == fifo->opal_fifo_tail.data.item) { + opal_atomic_ll_ptr(head, item); + next = (opal_list_item_t *) item->opal_list_next; + if (ghost != item) { + opal_atomic_sc_ptr(head, next, ret); + if (ret) { + break; + } + } else { + if (ghost == fifo->opal_fifo_tail.data.item) { return NULL; } /* fifo does not appear empty. wait for the fifo to be made * consistent by conflicting thread. */ - continue; - } - - next = (opal_list_item_t *) item->opal_list_next; - if (opal_atomic_sc_ptr (&fifo->opal_fifo_head.data.item, next)) { - break; } } while (1); #else + opal_list_item_t *item, *next; + /* protect against ABA issues by "locking" the head */ do { if (opal_atomic_cmpset_32 ((int32_t *) &fifo->opal_fifo_head.data.counter, 0, 1)) { diff --git a/opal/class/opal_lifo.h b/opal/class/opal_lifo.h index af3dedd9272..7302c18d23c 100644 --- a/opal/class/opal_lifo.h +++ b/opal/class/opal_lifo.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2007 Voltaire All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. - * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights * reseved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -204,8 +204,10 @@ static inline void _opal_lifo_release_cpu (void) */ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo) { - opal_list_item_t *item, *next; - int attempt = 0; + register opal_list_item_t volatile * volatile * head = &lifo->opal_lifo_head.data.item; + register opal_list_item_t *ghost = &lifo->opal_lifo_ghost; + register opal_list_item_t *item, *next; + int attempt = 0, ret; do { if (++attempt == 5) { @@ -215,13 +217,14 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo) attempt = 0; } - item = (opal_list_item_t *) opal_atomic_ll_ptr (&lifo->opal_lifo_head.data.item); - if (&lifo->opal_lifo_ghost == item) { + opal_atomic_ll_ptr(head, item); + next = (opal_list_item_t *) item->opal_list_next; + if (ghost == item) { return NULL; } - next = (opal_list_item_t *) item->opal_list_next; - } while (!opal_atomic_sc_ptr (&lifo->opal_lifo_head.data.item, next)); + opal_atomic_sc_ptr(head, next, ret); + } while (!ret); opal_atomic_wmb (); diff --git a/opal/include/opal/sys/arm64/atomic.h b/opal/include/opal/sys/arm64/atomic.h index 2f7f7d32aac..f637c2f52d8 100644 --- a/opal/include/opal/sys/arm64/atomic.h +++ b/opal/include/opal/sys/arm64/atomic.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved. - * Copyright (c) 2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -150,28 +150,19 @@ static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr, return (ret == oldval); } -static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr) -{ - int32_t ret; - - __asm__ __volatile__ ("ldaxr %w0, [%1] \n" - : "=&r" (ret) - : "r" (addr)); - - return ret; -} - -static inline int opal_atomic_sc_32 (volatile int32_t *addr, int32_t newval) -{ - int ret; - - __asm__ __volatile__ ("stlxr %w0, %w2, [%1] \n" - : "=&r" (ret) - : "r" (addr), "r" (newval) - : "cc", "memory"); +#define opal_atomic_ll_32(addr, value) \ + __asm__ __volatile__ ("ldaxr %w0, [%1] \n" \ + : "=&r" ((int32_t) (value)) \ + : "r" ((volatile int32_t *) (addr))) - return ret == 0; -} +#define opal_atomic_sc_32(addr, newval, ret) \ + do { \ + __asm__ __volatile__ ("stlxr %w0, %w2, [%1] \n" \ + : "=&r" ((int) (ret)) \ + : "r" ((volatile int32_t *) (addr)), "r" ((int32_t) (newval)) \ + : "cc", "memory"); \ + (ret) = ((ret) == 0); \ + } while (0) static inline int opal_atomic_cmpset_64(volatile int64_t *addr, int64_t oldval, int64_t newval) @@ -251,28 +242,19 @@ static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr, return (ret == oldval); } -static inline int64_t opal_atomic_ll_64 (volatile int64_t *addr) -{ - int64_t ret; - - __asm__ __volatile__ ("ldaxr %0, [%1] \n" - : "=&r" (ret) - : "r" (addr)); - - return ret; -} - -static inline int opal_atomic_sc_64 (volatile int64_t *addr, int64_t newval) -{ - int ret; - - __asm__ __volatile__ ("stlxr %w0, %2, [%1] \n" - : "=&r" (ret) - : "r" (addr), "r" (newval) - : "cc", "memory"); - - return ret == 0; -} +#define opal_atomic_ll_64(addr, value) \ + __asm__ __volatile__ ("ldaxr %0, [%1] \n" \ + : "=&r" ((int64_t) (value)) \ + : "r" ((volatile int64_t *) (addr))) + +#define opal_atomic_sc_64(addr, newval, ret) \ + do { \ + __asm__ __volatile__ ("stlxr %w0, %2, [%1] \n" \ + : "=&r" ((int) (ret)) \ + : "r" ((volatile int64_t *) (addr)), "r" ((int64_t) (newval)) \ + : "cc", "memory"); \ + (ret) = ((ret) == 0); \ + } while (0) #define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \ static inline type opal_atomic_ ## name ## _ ## bits (volatile type *addr, type value) \ diff --git a/opal/include/opal/sys/atomic_impl.h b/opal/include/opal/sys/atomic_impl.h index 16b03b485f3..a19e4ade725 100644 --- a/opal/include/opal/sys/atomic_impl.h +++ b/opal/include/opal/sys/atomic_impl.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -278,15 +278,15 @@ static inline int opal_atomic_cmpset_rel_ptr(volatile void* addr, #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_LLSC_32 -#define opal_atomic_ll_ptr(addr) (void *) opal_atomic_ll_32((int32_t *) addr) -#define opal_atomic_sc_ptr(addr, newval) opal_atomic_sc_32((int32_t *) addr, (int32_t) newval) +#define opal_atomic_ll_ptr opal_atomic_ll_32 +#define opal_atomic_sc_ptr opal_atomic_sc_32 #define OPAL_HAVE_ATOMIC_LLSC_PTR 1 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64 -#define opal_atomic_ll_ptr(addr) (void *) opal_atomic_ll_64((int64_t *) addr) -#define opal_atomic_sc_ptr(addr, newval) opal_atomic_sc_64((int64_t *) addr, (int64_t) newval) +#define opal_atomic_ll_ptr opal_atomic_ll_64 +#define opal_atomic_sc_ptr opal_atomic_sc_64 #define OPAL_HAVE_ATOMIC_LLSC_PTR 1 diff --git a/opal/include/opal/sys/powerpc/atomic.h b/opal/include/opal/sys/powerpc/atomic.h index 05c155eddbf..d62c09ac78a 100644 --- a/opal/include/opal/sys/powerpc/atomic.h +++ b/opal/include/opal/sys/powerpc/atomic.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2017 IBM Corporation. All rights reserved. - * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -135,7 +135,7 @@ void opal_atomic_isync(void) * it will emit lwz instead of ld to load the 64-bit operand. */ #define OPAL_ASM_VALUE64(x) (void *)(intptr_t) (x) #else -#define OPAL_ASM_VALUE64(x) x +#define OPAL_ASM_VALUE64(x) (int64_t) (x) #endif @@ -158,31 +158,21 @@ static inline int opal_atomic_cmpset_32(volatile int32_t *addr, return (ret == oldval); } -static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr) -{ - int32_t ret; - - __asm__ __volatile__ ("lwarx %0, 0, %1 \n\t" - : "=&r" (ret) - : "r" (addr) - ); - return ret; -} - -static inline int opal_atomic_sc_32 (volatile int32_t *addr, int32_t newval) -{ - int32_t ret, foo; - - __asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t" - " li %0,0 \n\t" - " bne- 1f \n\t" - " ori %0,%0,1 \n\t" - "1:" - : "=r" (ret), "=m" (*addr), "=r" (foo) - : "r" (addr), "r" (newval) - : "cc", "memory"); - return ret; -} +#define opal_atomic_ll_32(addr, value) \ + __asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \ + : "=&r" ((int32_t) (value)) \ + : "r" ((volatile int32_t *) (addr))) + +#define opal_atomic_sc_32(addr, newval, ret) \ + __asm__ __volatile__ (" stwcx. %2, 0, %1 \n\t" \ + " li %0,0 \n\t" \ + " bne- 1f \n\t" \ + " ori %0,%0,1 \n\t" \ + "1:" \ + : "=r" ((int32_t) ret) \ + : "r" ((volatile int32_t *) (addr)), \ + "r" ((int32_t) newval) \ + : "cc", "memory") /* these two functions aren't inlined in the non-gcc case because then there would be two function calls (since neither cmpset_32 nor @@ -280,31 +270,21 @@ static inline int opal_atomic_cmpset_64(volatile int64_t *addr, return (ret == oldval); } -static inline int64_t opal_atomic_ll_64(volatile int64_t *addr) -{ - int64_t ret; - - __asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" - : "=&r" (ret) - : "r" (addr) - ); - return ret; -} - -static inline int opal_atomic_sc_64(volatile int64_t *addr, int64_t newval) -{ - int32_t ret; - - __asm__ __volatile__ (" stdcx. %2, 0, %1 \n\t" - " li %0,0 \n\t" - " bne- 1f \n\t" - " ori %0,%0,1 \n\t" - "1:" - : "=r" (ret) - : "r" (addr), "r" (OPAL_ASM_VALUE64(newval)) - : "cc", "memory"); - return ret; -} +#define opal_atomic_ll_64(addr, value) \ + __asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \ + : "=&r" (OPAL_ASM_VALUE64(value)) \ + : "r" ((volatile int64_t *) (addr))) + +#define opal_atomic_sc_64(addr, newval, ret) \ + __asm__ __volatile__ (" stdcx. %2, 0, %1 \n\t" \ + " li %0,0 \n\t" \ + " bne- 1f \n\t" \ + " ori %0,%0,1 \n\t" \ + "1:" \ + : "=r" ((ret)) \ + : "r" ((volatile int64_t *) (addr)), \ + "r" (OPAL_ASM_VALUE64(newval)) \ + : "cc", "memory") /* these two functions aren't inlined in the non-gcc case because then there would be two function calls (since neither cmpset_64 nor