review feedback

Author: huixie90

libcxx/include/__atomic/atomic_sync.h

@@ -9,14 +9,12 @@
 #ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H
 #define _LIBCPP___ATOMIC_ATOMIC_SYNC_H
 
-#include <__algorithm/ranges_find.h>
 #include <__atomic/contention_t.h>
 #include <__atomic/memory_order.h>
 #include <__atomic/to_gcc_order.h>
 #include <__chrono/duration.h>
 #include <__config>
 #include <__memory/addressof.h>
-#include <__ranges/access.h>
 #include <__thread/poll_with_backoff.h>
 #include <__type_traits/conjunction.h>
 #include <__type_traits/decay.h>
@@ -78,70 +76,77 @@ _LIBCPP_EXPORTED_FROM_ABI void
 __libcpp_atomic_wait(__cxx_atomic_contention_t const volatile*, __cxx_contention_t) _NOEXCEPT;
 
 // new dylib interface
+
+// return the global contention state's current value for the address
 _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI __cxx_contention_t
-__libcpp_atomic_monitor_global(void const volatile* __address) _NOEXCEPT;
+__atomic_monitor_global(void const volatile* __address) _NOEXCEPT;
 
+// wait on the global contention state to be changed from the given value for the address
 _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
-__libcpp_atomic_wait_global_table(void const volatile* __address, __cxx_contention_t __monitor_value) _NOEXCEPT;
+__atomic_wait_global_table(void const volatile* __address, __cxx_contention_t __monitor_value) _NOEXCEPT;
 
+// notify one waiter waiting on the global contention state for the address
 _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
-__libcpp_atomic_notify_one_global_table(void const volatile*) _NOEXCEPT;
+__atomic_notify_one_global_table(void const volatile*) _NOEXCEPT;
+
+// notify all waiters waiting on the global contention state for the address
 _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
-__libcpp_atomic_notify_all_global_table(void const volatile*) _NOEXCEPT;
+__atomic_notify_all_global_table(void const volatile*) _NOEXCEPT;
 
+// wait on the address directly with the native platform wait
 template <std::size_t _Size>
 _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
-__libcpp_atomic_wait_native(void const volatile* __address, void const* __old_value) _NOEXCEPT;
+__atomic_wait_native(void const volatile* __address, void const* __old_value) _NOEXCEPT;
+
+// notify one waiter waiting on the address directly with the native platform wait
 template <std::size_t _Size>
-_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
-__libcpp_atomic_notify_one_native(const volatile void*) _NOEXCEPT;
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one_native(const volatile void*) _NOEXCEPT;
 
+// notify all waiters waiting on the address directly with the native platform wait
 template <std::size_t _Size>
-_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void
-__libcpp_atomic_notify_all_native(const volatile void*) _NOEXCEPT;
+_LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(const volatile void*) _NOEXCEPT;
+
+#    ifdef __linux__
+#      define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(4)
+#    elif defined(__APPLE__)
+#      define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY)                                                               \
+        _APPLY(4)                                                                                                      \
+        _APPLY(8)
+#    elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
+#      define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8)
+#    else
+#      define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(sizeof(__cxx_contention_t))
+#    endif // __linux__
 
 // concepts defines the types are supported natively by the platform's wait
 
 #    if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
 
-#      ifdef __linux__
-
-#        define _LIBCPP_ATOMIC_WAIT_SIZES_LIST(_APPLY) _APPLY(4)
-
-#      elif defined(__APPLE__)
-
-#        define _LIBCPP_ATOMIC_WAIT_SIZES_LIST(_APPLY)                                                                 \
-          _APPLY(4)                                                                                                    \
-          _APPLY(8)
-
-#      elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
-
-#        define _LIBCPP_ATOMIC_WAIT_SIZES_LIST(_APPLY) _APPLY(8)
-
-#      else
-
-#        define _LIBCPP_ATOMIC_WAIT_SIZES_LIST(_APPLY) _APPLY(sizeof(__cxx_contention_t))
-
-#      endif // __linux__
-
-inline constexpr std::size_t __supported_native_wait_sizes[] = {
-#      define _IDENTITY(_SIZE) _SIZE,
-    _LIBCPP_ATOMIC_WAIT_SIZES_LIST(_IDENTITY)
-#      undef _IDENTITY
-};
+_LIBCPP_HIDE_FROM_ABI constexpr bool __has_native_atomic_wait_impl(size_t __size) {
+  switch (__size) {
+#      define _LIBCPP_MAKE_CASE(n)                                                                                     \
+      case n:                                                                                                          \
+        return true;
+    _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_LIBCPP_MAKE_CASE)
+  default:
+    return false;
+#      undef _LIBCPP_MAKE_CASE
+  };
+}
 
 template <class _Tp>
-concept __atomic_wait_native_type =
-    has_unique_object_representations_v<_Tp> &&
-    std::ranges::find(__supported_native_wait_sizes, sizeof(_Tp)) != ranges::end(__supported_native_wait_sizes);
+concept __has_native_atomic_wait =
+    has_unique_object_representations_v<_Tp> && __has_native_atomic_wait_impl(sizeof(_Tp));
 
 #    else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
 
 template <class _Tp>
-concept __atomic_wait_native_type = is_same_v<_Tp, __cxx_contention_t>;
+concept __has_native_atomic_wait = is_same_v<_Tp, __cxx_contention_t>;
 
 #    endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
 
+#    if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
 template <class _AtomicWaitable, class _Poll>
 struct __atomic_wait_backoff_impl {
   const _AtomicWaitable& __a_;
@@ -155,24 +160,70 @@ struct __atomic_wait_backoff_impl {
     if (__elapsed > chrono::microseconds(4)) {
       auto __contention_address = __waitable_traits::__atomic_contention_address(__a_);
 
-      if constexpr (__atomic_wait_native_type<__value_type>) {
+      if constexpr (__has_native_atomic_wait<__value_type>) {
         auto __atomic_value = __waitable_traits::__atomic_load(__a_, __order_);
         if (__poll_(__atomic_value))
           return true;
-        std::__libcpp_atomic_wait_native<sizeof(__value_type)>(__contention_address, std::addressof(__atomic_value));
+        std::__atomic_wait_native<sizeof(__value_type)>(__contention_address, std::addressof(__atomic_value));
       } else {
-        __cxx_contention_t __monitor_val = std::__libcpp_atomic_monitor_global(__contention_address);
+        __cxx_contention_t __monitor_val = std::__atomic_monitor_global(__contention_address);
         auto __atomic_value              = __waitable_traits::__atomic_load(__a_, __order_);
         if (__poll_(__atomic_value))
           return true;
-        std::__libcpp_atomic_wait_global_table(__contention_address, __monitor_val);
+        std::__atomic_wait_global_table(__contention_address, __monitor_val);
       }
     } else {
     } // poll
     return false;
   }
 };
 
+#    else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
+template <class _AtomicWaitable, class _Poll>
+struct __atomic_wait_backoff_impl {
+  const _AtomicWaitable& __a_;
+  _Poll __poll_;
+  memory_order __order_;
+
+  using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >;
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  __update_monitor_val_and_poll(__cxx_atomic_contention_t const volatile*, __cxx_contention_t& __monitor_val) const {
+    // In case the contention type happens to be __cxx_atomic_contention_t, i.e. __cxx_atomic_impl<int64_t>,
+    // the platform wait is directly monitoring the atomic value itself.
+    // `__poll_` takes the current value of the atomic as an in-out argument
+    // to potentially modify it. After it returns, `__monitor` has a value
+    // which can be safely waited on by `std::__libcpp_atomic_wait` without any
+    // ABA style issues.
+    __monitor_val = __waitable_traits::__atomic_load(__a_, __order_);
+    return __poll_(__monitor_val);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool
+  __update_monitor_val_and_poll(void const volatile* __contention_address, __cxx_contention_t& __monitor_val) const {
+    // In case the contention type is anything else, platform wait is monitoring a __cxx_atomic_contention_t
+    // from the global pool, the monitor comes from __libcpp_atomic_monitor
+    __monitor_val      = std::__libcpp_atomic_monitor(__contention_address);
+    auto __current_val = __waitable_traits::__atomic_load(__a_, __order_);
+    return __poll_(__current_val);
+  }
+
+  _LIBCPP_HIDE_FROM_ABI bool operator()(chrono::nanoseconds __elapsed) const {
+    if (__elapsed > chrono::microseconds(4)) {
+      auto __contention_address = __waitable_traits::__atomic_contention_address(__a_);
+      __cxx_contention_t __monitor_val;
+      if (__update_monitor_val_and_poll(__contention_address, __monitor_val))
+        return true;
+      std::__libcpp_atomic_wait(__contention_address, __monitor_val);
+    } else {
+    } // poll
+    return false;
+  }
+};
+
+#    endif // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
 // The semantics of this function are similar to `atomic`'s
 // `.wait(T old, std::memory_order order)`, but instead of having a hardcoded
 // predicate (is the loaded value unequal to `old`?), the predicate function is
@@ -194,15 +245,17 @@ _LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, memo
       /* backoff */ __backoff_fn);
 }
 
+#    if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+
 template <class _AtomicWaitable>
 _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
   static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
   using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type;
-  if constexpr (__atomic_wait_native_type<__value_type>) {
-    std::__libcpp_atomic_notify_one_native<sizeof(__value_type)>(
+  if constexpr (__has_native_atomic_wait<__value_type>) {
+    std::__atomic_notify_one_native<sizeof(__value_type)>(
         __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
   } else {
-    std::__libcpp_atomic_notify_one_global_table(
+    std::__atomic_notify_one_global_table(
         __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
   }
 }
@@ -211,15 +264,29 @@ template <class _AtomicWaitable>
 _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
   static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
   using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type;
-  if constexpr (__atomic_wait_native_type<__value_type>) {
-    std::__libcpp_atomic_notify_all_native<sizeof(__value_type)>(
+  if constexpr (__has_native_atomic_wait<__value_type>) {
+    std::__atomic_notify_all_native<sizeof(__value_type)>(
         __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
   } else {
-    std::__libcpp_atomic_notify_all_global_table(
+    std::__atomic_notify_all_global_table(
         __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
   }
 }
 
+#    else // _LIBCPP_AVAILABILITY_HAS_NEW_SYNC
+template <class _AtomicWaitable>
+_LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) {
+  static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+  std::__cxx_atomic_notify_one(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
+}
+
+template <class _AtomicWaitable>
+_LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) {
+  static_assert(__atomic_waitable<_AtomicWaitable>::value, "");
+  std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a));
+}
+#    endif
+
 #  else // _LIBCPP_HAS_THREADS
 
 template <class _AtomicWaitable, class _Poll>

libcxx/include/__atomic/contention_t.h

@@ -19,11 +19,29 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__))
+#if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE)
+
+#  ifdef __linux__
 using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
-#else
+#  elif defined(__APPLE__)
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+#  elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8
 using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
-#endif // __linux__ || (_AIX && !__64BIT__)
+#  elif defined(_AIX) && !defined(__64BIT__)
+using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
+#  else
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+#  endif // __linux__
+
+#else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+
+#  if defined(__linux__) || (defined(_AIX) && !defined(__64BIT__))
+using __cxx_contention_t _LIBCPP_NODEBUG = int32_t;
+#  else
+using __cxx_contention_t _LIBCPP_NODEBUG = int64_t;
+#  endif // __linux__ || (_AIX && !__64BIT__)
+
+#endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
 
 using __cxx_atomic_contention_t _LIBCPP_NODEBUG = __cxx_atomic_impl<__cxx_contention_t>;