Make spinlocks mandatory

With the change to make compare and swap mandatory, we can always
implement spinlocks, so make them mandatory and simplify the interface.

Signed-off-by: Brian Barrett <[email protected]>

Author: bwbarrett

opal/include/opal/sys/Makefile.am

@@ -34,6 +34,7 @@ headers += \
 	opal/sys/atomic_impl.h \
 	opal/sys/atomic_impl_ptr_cswap.h \
 	opal/sys/atomic_impl_ptr_swap.h \
+	opal/sys/atomic_impl_spinlock.h \
 	opal/sys/timer.h \
         opal/sys/cma.h
 

opal/include/opal/sys/arm64/atomic.h

@@ -255,6 +255,15 @@ static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t new
 #include "opal/sys/atomic_impl_ptr_swap.h"
 
 
+/**********************************************************************
+ *
+ * Atomic spinlocks
+ *
+ *********************************************************************/
+
+#include "opal/sys/atomic_impl_spinlock.h"
+
+
 /**********************************************************************
  *
  * Atomic math operations

opal/include/opal/sys/atomic.h

@@ -41,8 +41,6 @@
  * The following #defines will be true / false based on
  * assembly support:
  *
- *  - \c OPAL_HAVE_ATOMIC_SPINLOCKS atomic spinlocks
- *
  * Note that for the Atomic math, atomic add/sub may be implemented as
  * C code using opal_atomic_compare_exchange.  The appearance of atomic
  * operation will be upheld in these cases.
@@ -66,27 +64,6 @@
 
 
 BEGIN_C_DECLS
-/**********************************************************************
- *
- * Data structures for atomic ops
- *
- *********************************************************************/
-/**
- * Volatile lock object (with optional padding).
- *
- * \note The internals of the lock are included here, but should be
- * considered private.  The implementation currently in use may choose
- * to use an int or unsigned char as the lock value - the user is not
- * informed either way.
- */
-struct opal_atomic_lock_t {
-    union {
-        opal_atomic_int32_t lock;          /**< The lock address (an integer) */
-        volatile unsigned char sparc_lock; /**< The lock address on sparc */
-        char padding[sizeof(int)];         /**< Array for optional padding */
-    } u;
-};
-typedef struct opal_atomic_lock_t opal_atomic_lock_t;
 
 /**********************************************************************
  *
@@ -105,16 +82,6 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
 #define OPAL_HAVE_INLINE_ATOMIC_XOR_64              1
 #define OPAL_HAVE_INLINE_ATOMIC_SUB_64              1
 
-/**
- * Enumeration of lock states
- */
-enum { OPAL_ATOMIC_LOCK_UNLOCKED = 0, OPAL_ATOMIC_LOCK_LOCKED = 1 };
-
-#    define OPAL_ATOMIC_LOCK_INIT                     \
-        {                                             \
-            .u = {.lock = OPAL_ATOMIC_LOCK_UNLOCKED } \
-        }
-
 /**********************************************************************
  *
  * Load the appropriate architecture files and set some reasonable
@@ -374,73 +341,39 @@ static inline intptr_t opal_atomic_swap_ptr(opal_atomic_intptr_t *addr, intptr_t
 
 /**********************************************************************
  *
- * Atomic spinlocks - always inlined, if have atomic compare-and-swap
+ * Atomic spinlocks
  *
  *********************************************************************/
-
-#    if !defined(OPAL_HAVE_ATOMIC_SPINLOCKS) && !defined(DOXYGEN)
-/* 0 is more like "pending" - we'll fix up at the end after all
-   the static inline functions are declared */
-#        define OPAL_HAVE_ATOMIC_SPINLOCKS 0
-#    endif
-
-#    if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS \
-        || (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
-
 /**
  * Initialize a lock to value
  *
  * @param lock         Address of the lock
  * @param value        Initial value to set lock to
  */
-#        if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
-static inline
-#        endif
-    void
-    opal_atomic_lock_init(opal_atomic_lock_t *lock, int32_t value);
+static inline void opal_atomic_lock_init(opal_atomic_lock_t *lock, int32_t value);
 
 /**
  * Try to acquire a lock.
  *
  * @param lock          Address of the lock.
  * @return              0 if the lock was acquired, 1 otherwise.
  */
-#        if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
-static inline
-#        endif
-    int
-    opal_atomic_trylock(opal_atomic_lock_t *lock);
+static inline int opal_atomic_trylock(opal_atomic_lock_t *lock);
 
 /**
  * Acquire a lock by spinning.
  *
  * @param lock          Address of the lock.
  */
-#        if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
-static inline
-#        endif
-    void
-    opal_atomic_lock(opal_atomic_lock_t *lock);
+static inline void opal_atomic_lock(opal_atomic_lock_t *lock);
 
 /**
  * Release a lock.
  *
  * @param lock          Address of the lock.
  */
-#        if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
-static inline
-#        endif
-    void
-    opal_atomic_unlock(opal_atomic_lock_t *lock);
-
-#        if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
-#            undef OPAL_HAVE_ATOMIC_SPINLOCKS
-#            define OPAL_HAVE_ATOMIC_SPINLOCKS \
-                (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
-#            define OPAL_NEED_INLINE_ATOMIC_SPINLOCKS 1
-#        endif
+static inline void opal_atomic_unlock(opal_atomic_lock_t *lock);
 
-#    endif /* OPAL_HAVE_ATOMIC_SPINLOCKS */
 
 /**********************************************************************
  *

opal/include/opal/sys/atomic_impl.h

@@ -321,44 +321,3 @@ static inline intptr_t opal_atomic_sub_fetch_ptr(opal_atomic_intptr_t *addr, voi
     return 0;
 #    endif
 }
-
-
-/**********************************************************************
- *
- * Atomic spinlocks
- *
- *********************************************************************/
-#ifdef OPAL_NEED_INLINE_ATOMIC_SPINLOCKS
-
-/*
- * Lock initialization function. It set the lock to UNLOCKED.
- */
-static inline void opal_atomic_lock_init(opal_atomic_lock_t *lock, int32_t value)
-{
-    lock->u.lock = value;
-}
-
-static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
-{
-    int32_t unlocked = OPAL_ATOMIC_LOCK_UNLOCKED;
-    bool ret = opal_atomic_compare_exchange_strong_acq_32(&lock->u.lock, &unlocked,
-                                                          OPAL_ATOMIC_LOCK_LOCKED);
-    return (ret == false) ? 1 : 0;
-}
-
-static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
-{
-    while (opal_atomic_trylock(lock)) {
-        while (lock->u.lock == OPAL_ATOMIC_LOCK_LOCKED) {
-            /* spin */;
-        }
-    }
-}
-
-static inline void opal_atomic_unlock(opal_atomic_lock_t *lock)
-{
-    opal_atomic_wmb();
-    lock->u.lock = OPAL_ATOMIC_LOCK_UNLOCKED;
-}
-
-#endif /* OPAL_HAVE_ATOMIC_SPINLOCKS */

opal/include/opal/sys/atomic_impl_spinlock.h

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+ *                         All Rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ *
+ * Compare-and-swap based implementation of the atomic interface
+ */
+
+#ifndef ATOMIC_SPINLOCK_IMPL_H
+#define ATOMIC_SPINLOCK_IMPL_H
+
+static inline void opal_atomic_lock_init(opal_atomic_lock_t *lock, int32_t value)
+{
+    *lock = value;
+    opal_atomic_wmb();
+}
+
+static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
+{
+    int32_t unlocked = OPAL_ATOMIC_LOCK_UNLOCKED;
+    bool ret = opal_atomic_compare_exchange_strong_acq_32(lock, &unlocked,
+                                                          OPAL_ATOMIC_LOCK_LOCKED);
+    return (ret == false) ? 1 : 0;
+}
+
+static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
+{
+    while (opal_atomic_trylock(lock)) {
+        while (*lock == OPAL_ATOMIC_LOCK_LOCKED) {
+            /* spin */;
+        }
+    }
+}
+
+static inline void opal_atomic_unlock(opal_atomic_lock_t *lock)
+{
+    opal_atomic_wmb();
+    *lock = OPAL_ATOMIC_LOCK_UNLOCKED;
+}
+
+#endif /* #ifndef ATOMIC_SPINLOCK_IMPL_H */

opal/include/opal/sys/atomic_stdc.h

@@ -50,8 +50,6 @@
 #    define OPAL_HAVE_ATOMIC_MIN_64 1
 #    define OPAL_HAVE_ATOMIC_MAX_64 1
 
-#    define OPAL_HAVE_ATOMIC_SPINLOCKS 1
-
 
 /**********************************************************************
  *
@@ -162,6 +160,36 @@ opal_atomic_compare_exchange_strong_128(opal_atomic_int128_t *addr, opal_int128_
         atomic_exchange_explicit((_Atomic unsigned long *) addr, value, memory_order_relaxed)
 
 
+/**********************************************************************
+ *
+ * Atomic spinlocks
+ *
+ *********************************************************************/
+/*
+ * Lock initialization function. It set the lock to UNLOCKED.
+ */
+static inline void opal_atomic_lock_init(opal_atomic_lock_t *lock, bool value)
+{
+    atomic_flag_clear_explicit(lock, memory_order_relaxed);
+}
+
+static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
+{
+    return (int) atomic_flag_test_and_set(lock);
+}
+
+static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
+{
+    while (opal_atomic_trylock(lock)) {
+    }
+}
+
+static inline void opal_atomic_unlock(opal_atomic_lock_t *lock)
+{
+    atomic_flag_clear(lock);
+}
+
+
 #    define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator)                             \
         static inline type opal_atomic_fetch_##op##_##bits(opal_atomic_##type *addr, type value)   \
         {                                                                                          \
@@ -173,6 +201,7 @@ opal_atomic_compare_exchange_strong_128(opal_atomic_int128_t *addr, opal_int128_
             return atomic_fetch_##op##_explicit(addr, value, memory_order_relaxed) operator value; \
         }
 
+
 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +)
 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +)
 OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +)
@@ -265,35 +294,4 @@ static inline int64_t opal_atomic_max_fetch_64(opal_atomic_int64_t *addr, int64_
     return old >= value ? old : value;
 }
 
-#    define OPAL_ATOMIC_LOCK_UNLOCKED false
-#    define OPAL_ATOMIC_LOCK_LOCKED   true
-
-#    define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT
-
-typedef atomic_flag opal_atomic_lock_t;
-
-/*
- * Lock initialization function. It set the lock to UNLOCKED.
- */
-static inline void opal_atomic_lock_init(opal_atomic_lock_t *lock, bool value)
-{
-    atomic_flag_clear_explicit(lock, memory_order_relaxed);
-}
-
-static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
-{
-    return (int) atomic_flag_test_and_set(lock);
-}
-
-static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
-{
-    while (opal_atomic_trylock(lock)) {
-    }
-}
-
-static inline void opal_atomic_unlock(opal_atomic_lock_t *lock)
-{
-    atomic_flag_clear(lock);
-}
-
 #endif /* !defined(OPAL_ATOMIC_STDC_H) */