diff --git a/libc/src/__support/threads/CMakeLists.txt b/libc/src/__support/threads/CMakeLists.txt index bd49bbb5ad2fe..5a39c9dff9216 100644 --- a/libc/src/__support/threads/CMakeLists.txt +++ b/libc/src/__support/threads/CMakeLists.txt @@ -105,3 +105,20 @@ add_header_library( libc.hdr.types.pid_t ${identifier_dependency_on_thread} ) + +# TODO: implement windows "futex" with WaitOnAddress +add_header_library( + futex_utils + HDRS + futex_utils.h + DEPENDS + .${LIBC_TARGET_OS}.futex_utils +) + +add_header_library( + futex_timeout + HDRS + futex_timeout.h + DEPENDS + .${LIBC_TARGET_OS}.futex_timeout +) diff --git a/libc/src/__support/threads/futex_timeout.h b/libc/src/__support/threads/futex_timeout.h new file mode 100644 index 0000000000000..7adfbc594bfca --- /dev/null +++ b/libc/src/__support/threads/futex_timeout.h @@ -0,0 +1,18 @@ +//===--- Futex With Timeout Support -----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC_SUPPORT_THREADS_FUTEX_TIMEOUT_H +#define LLVM_LIBC_SRC_SUPPORT_THREADS_FUTEX_TIMEOUT_H + +// TODO: implement futex for other platforms. +#ifdef __linux__ +#include "src/__support/threads/linux/futex_timeout.h" +#else +#error "Unsupported platform" +#endif + +#endif // LLVM_LIBC_SRC_SUPPORT_THREADS_FUTEX_TIMEOUT_H diff --git a/libc/src/__support/threads/futex_utils.h b/libc/src/__support/threads/futex_utils.h new file mode 100644 index 0000000000000..61ccabeadc85b --- /dev/null +++ b/libc/src/__support/threads/futex_utils.h @@ -0,0 +1,28 @@ +//===--- Futex Utilities ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +//===--- Futex Wrapper ------------------------------------------*- C++ -*-===// +// We take the name "futex" from Linux system. This library provides a general +// wrapper for waiting and notifying on atomic words. Various platforms support +// futex-like operations. +// - Windows: WaitOnAddress and WakeByAddressSingle/WakeByAddressAll +// (Windows futex cannot be used in inter-process synchronization) +// - MacOS: os_sync_wait_on_address or __ulock_wait/__ulock_wake +// - FreeBSD: _umtx_op + +#ifndef LLVM_LIBC_SRC_SUPPORT_THREADS_FUTEX_UTILS_H +#define LLVM_LIBC_SRC_SUPPORT_THREADS_FUTEX_UTILS_H + +// TODO: implement futex for other platforms. +#ifdef __linux__ +#include "src/__support/threads/linux/futex_utils.h" +#else +#error "Unsupported platform" +#endif + +#endif // LLVM_LIBC_SRC_SUPPORT_THREADS_FUTEX_UTILS_H diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt index 47598d98c9886..19ce75ce9cd6b 100644 --- a/libc/src/__support/threads/linux/CMakeLists.txt +++ b/libc/src/__support/threads/linux/CMakeLists.txt @@ -19,6 +19,14 @@ add_header_library( libc.src.__support.CPP.atomic libc.src.__support.CPP.limits libc.src.__support.CPP.optional +) + +add_header_library( + futex_timeout + HDRS + futex_timeout.h + DEPENDS + .futex_utils libc.src.__support.time.linux.abs_timeout ) @@ -34,7 +42,7 @@ add_header_library( HDRS mutex.h DEPENDS - .futex_utils + .futex_timeout libc.src.__support.threads.sleep libc.src.__support.time.linux.abs_timeout libc.src.__support.time.linux.monotonicity @@ -50,7 +58,7 @@ add_header_library( HDRS rwlock.h DEPENDS - .futex_utils + .futex_timeout .raw_mutex libc.src.__support.common libc.src.__support.OSUtil.osutil diff --git a/libc/src/__support/threads/linux/CndVar.cpp b/libc/src/__support/threads/linux/CndVar.cpp index be74c18dddf31..2f7297b743395 100644 --- a/libc/src/__support/threads/linux/CndVar.cpp +++ b/libc/src/__support/threads/linux/CndVar.cpp @@ -8,7 +8,7 @@ #include "src/__support/threads/CndVar.h" #include "src/__support/CPP/mutex.h" -#include "src/__support/OSUtil/syscall.h" // syscall_impl +#include "src/__support/OSUtil/syscall.h" // syscall_impl #include "src/__support/macros/config.h" #include "src/__support/threads/linux/futex_word.h" // FutexWordType #include "src/__support/threads/linux/raw_mutex.h" // RawMutex @@ -56,7 +56,7 @@ int CndVar::wait(Mutex *m) { } } - waiter.futex_word.wait(WS_Waiting, cpp::nullopt, true); + waiter.futex_word.wait(WS_Waiting, /*is_shared=*/true); // At this point, if locking |m| fails, we can simply return as the // queued up waiter would have been removed from the queue. diff --git a/libc/src/__support/threads/linux/futex_timeout.h b/libc/src/__support/threads/linux/futex_timeout.h new file mode 100644 index 0000000000000..ba12cab7d8066 --- /dev/null +++ b/libc/src/__support/threads/linux/futex_timeout.h @@ -0,0 +1,62 @@ +//===--- Futex Wrapper ------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_TIMEOUT_H +#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_TIMEOUT_H + +#include "src/__support/CPP/atomic.h" +#include "src/__support/CPP/limits.h" +#include "src/__support/CPP/optional.h" +#include "src/__support/OSUtil/syscall.h" +#include "src/__support/macros/attributes.h" +#include "src/__support/macros/config.h" +#include "src/__support/threads/linux/futex_utils.h" +#include "src/__support/time/linux/abs_timeout.h" + +namespace LIBC_NAMESPACE_DECL { +class TimedFutex : public Futex { +public: + using Timeout = internal::AbsTimeout; + using Futex::Futex; + using Futex::operator=; + LIBC_INLINE long wait(FutexWordType expected, + cpp::optional timeout = cpp::nullopt, + bool is_shared = false) { + // use bitset variants to enforce abs_time + uint32_t op = is_shared ? FUTEX_WAIT_BITSET : FUTEX_WAIT_BITSET_PRIVATE; + if (timeout && timeout->is_realtime()) { + op |= FUTEX_CLOCK_REALTIME; + } + for (;;) { + if (this->load(cpp::MemoryOrder::RELAXED) != expected) + return 0; + + long ret = syscall_impl( + /* syscall number */ FUTEX_SYSCALL_ID, + /* futex address */ this, + /* futex operation */ op, + /* expected value */ expected, + /* timeout */ timeout ? &timeout->get_timespec() : nullptr, + /* ignored */ nullptr, + /* bitset */ FUTEX_BITSET_MATCH_ANY); + + // continue waiting if interrupted; otherwise return the result + // which should normally be 0 or -ETIMEOUT + if (ret == -EINTR) + continue; + + return ret; + } + } +}; + +static_assert(__is_standard_layout(TimedFutex), + "Futex must be a standard layout type."); +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_FUTEX_TIMEOUT_H diff --git a/libc/src/__support/threads/linux/futex_utils.h b/libc/src/__support/threads/linux/futex_utils.h index 943a99ab38c8c..0d36eb92232fa 100644 --- a/libc/src/__support/threads/linux/futex_utils.h +++ b/libc/src/__support/threads/linux/futex_utils.h @@ -16,28 +16,21 @@ #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" #include "src/__support/threads/linux/futex_word.h" -#include "src/__support/time/linux/abs_timeout.h" #include #include namespace LIBC_NAMESPACE_DECL { class Futex : public cpp::Atomic { public: - using Timeout = internal::AbsTimeout; LIBC_INLINE constexpr Futex(FutexWordType value) : cpp::Atomic(value) {} LIBC_INLINE Futex &operator=(FutexWordType value) { cpp::Atomic::store(value); return *this; } - LIBC_INLINE long wait(FutexWordType expected, - cpp::optional timeout = cpp::nullopt, - bool is_shared = false) { + LIBC_INLINE long wait(FutexWordType expected, bool is_shared = false) { // use bitset variants to enforce abs_time uint32_t op = is_shared ? FUTEX_WAIT_BITSET : FUTEX_WAIT_BITSET_PRIVATE; - if (timeout && timeout->is_realtime()) { - op |= FUTEX_CLOCK_REALTIME; - } for (;;) { if (this->load(cpp::MemoryOrder::RELAXED) != expected) return 0; @@ -47,7 +40,7 @@ class Futex : public cpp::Atomic { /* futex address */ this, /* futex operation */ op, /* expected value */ expected, - /* timeout */ timeout ? &timeout->get_timespec() : nullptr, + /* timeout */ nullptr, /* ignored */ nullptr, /* bitset */ FUTEX_BITSET_MATCH_ANY); diff --git a/libc/src/__support/threads/linux/raw_mutex.h b/libc/src/__support/threads/linux/raw_mutex.h index 47f0aa70f1c46..717238b2c263e 100644 --- a/libc/src/__support/threads/linux/raw_mutex.h +++ b/libc/src/__support/threads/linux/raw_mutex.h @@ -14,7 +14,7 @@ #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" -#include "src/__support/threads/linux/futex_utils.h" +#include "src/__support/threads/linux/futex_timeout.h" #include "src/__support/threads/linux/futex_word.h" #include "src/__support/threads/sleep.h" #include "src/__support/time/linux/abs_timeout.h" @@ -38,7 +38,7 @@ namespace LIBC_NAMESPACE_DECL { // critical sections. class RawMutex { protected: - Futex futex; + TimedFutex futex; LIBC_INLINE_VAR static constexpr FutexWordType UNLOCKED = 0b00; LIBC_INLINE_VAR static constexpr FutexWordType LOCKED = 0b01; LIBC_INLINE_VAR static constexpr FutexWordType IN_CONTENTION = 0b10; @@ -63,7 +63,7 @@ class RawMutex { // Return true if the lock is acquired. Return false if timeout happens before // the lock is acquired. - LIBC_INLINE bool lock_slow(cpp::optional timeout, + LIBC_INLINE bool lock_slow(cpp::optional timeout, bool is_pshared, unsigned spin_count) { FutexWordType state = spin(spin_count); // Before go into contention state, try to grab the lock. @@ -102,7 +102,7 @@ class RawMutex { expected, LOCKED, cpp::MemoryOrder::ACQUIRE, cpp::MemoryOrder::RELAXED); } LIBC_INLINE bool - lock(cpp::optional timeout = cpp::nullopt, + lock(cpp::optional timeout = cpp::nullopt, bool is_shared = false, unsigned spin_count = LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT) { // Timeout will not be checked if immediate lock is possible. diff --git a/libc/src/__support/threads/linux/rwlock.h b/libc/src/__support/threads/linux/rwlock.h index 57fcc7bb67a6a..6d3e2f2ff85ea 100644 --- a/libc/src/__support/threads/linux/rwlock.h +++ b/libc/src/__support/threads/linux/rwlock.h @@ -55,9 +55,9 @@ class WaitingQueue final : private RawMutex { // Pending writer count (protected by the mutex) FutexWordType pending_writers; // Reader serialization (increases on each reader-waking operation) - Futex reader_serialization; + TimedFutex reader_serialization; // Writer serialization (increases on each writer-waking operation) - Futex writer_serialization; + TimedFutex writer_serialization; public: // RAII guard to lock and unlock the waiting queue. @@ -98,7 +98,7 @@ class WaitingQueue final : private RawMutex { template LIBC_INLINE long wait(FutexWordType expected, - cpp::optional timeout, + cpp::optional timeout, bool is_pshared) { if constexpr (role == Role::Reader) return reader_serialization.wait(expected, timeout, is_pshared); @@ -389,7 +389,7 @@ class RwLock { private: template LIBC_INLINE LockResult - lock_slow(cpp::optional timeout = cpp::nullopt, + lock_slow(cpp::optional timeout = cpp::nullopt, unsigned spin_count = LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT) { // Phase 1: deadlock detection. // A deadlock happens if this is a RAW/WAW lock in the same thread. @@ -468,7 +468,7 @@ class RwLock { public: [[nodiscard]] LIBC_INLINE LockResult - read_lock(cpp::optional timeout = cpp::nullopt, + read_lock(cpp::optional timeout = cpp::nullopt, unsigned spin_count = LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT) { LockResult result = try_read_lock(); if (LIBC_LIKELY(result != LockResult::Busy)) @@ -477,7 +477,7 @@ class RwLock { } [[nodiscard]] LIBC_INLINE LockResult - write_lock(cpp::optional timeout = cpp::nullopt, + write_lock(cpp::optional timeout = cpp::nullopt, unsigned spin_count = LIBC_COPT_RWLOCK_DEFAULT_SPIN_COUNT) { LockResult result = try_write_lock(); if (LIBC_LIKELY(result != LockResult::Busy)) diff --git a/libc/src/__support/threads/linux/thread.cpp b/libc/src/__support/threads/linux/thread.cpp index c531d74c53355..d55dc453f8279 100644 --- a/libc/src/__support/threads/linux/thread.cpp +++ b/libc/src/__support/threads/linux/thread.cpp @@ -377,7 +377,7 @@ void Thread::wait() { // We cannot do a FUTEX_WAIT_PRIVATE here as the kernel does a // FUTEX_WAKE and not a FUTEX_WAKE_PRIVATE. while (clear_tid->load() != 0) - clear_tid->wait(CLEAR_TID_VALUE, cpp::nullopt, true); + clear_tid->wait(CLEAR_TID_VALUE, /*is_shared=*/true); } bool Thread::operator==(const Thread &thread) const { diff --git a/libc/src/threads/linux/CMakeLists.txt b/libc/src/threads/linux/CMakeLists.txt index 6c8e0845faf4c..cf50e0ac56012 100644 --- a/libc/src/threads/linux/CMakeLists.txt +++ b/libc/src/threads/linux/CMakeLists.txt @@ -10,7 +10,7 @@ add_header_library( libc.src.__support.OSUtil.osutil libc.src.__support.threads.mutex libc.src.__support.threads.linux.raw_mutex - libc.src.__support.threads.linux.futex_utils + libc.src.__support.threads.futex_utils ) add_entrypoint_object(