From 5a326f91b542e26d0a0047fb2a70a05748dc33d7 Mon Sep 17 00:00:00 2001 From: Mike Ash Date: Fri, 17 Oct 2025 14:02:08 -0400 Subject: [PATCH] [Runtime] Faster dynamic exclusivity checking implemented in Swift. Replace C++ implementation of swift_beginAccess and swift_endAccess with (almost) pure Swift implementation. Helpers remain in C++ for TLS, getting return addresses, and raising a fatal error on violations. This change also moves the exclusivity access set head from the shared SwiftTLSContext structure to a dedicated TLS key. This improves performance, which is important for exclusivity checking. This is particularly the case where we can inline TLS access with a constant key, as on Darwin ARM64. The code that bridges exclusivity tracking into Concurrency remains in C++. The new Swift implementation exposes a few helpers for it to use as a replacement for directly manipulating the C++ implementation. rdar://161122309 --- Runtimes/Core/Core/CMakeLists.txt | 1 + include/swift/Threading/Impl/Darwin.h | 2 + include/swift/Threading/TLSKeys.h | 3 +- .../swift/shims/AssertionReporting.h | 7 + .../public/SwiftShims/swift/shims/LibcShims.h | 3 + .../SwiftShims/swift/shims/RuntimeStubs.h | 20 ++ .../swift/shims/ThreadLocalStorage.h | 44 +++ stdlib/public/core/CMakeLists.txt | 1 + stdlib/public/core/Exclusivity.swift | 262 ++++++++++++++++++ stdlib/public/core/GroupInfo.json | 3 +- .../public/runtime/ConcurrencyExclusivity.inc | 50 ++-- stdlib/public/runtime/Exclusivity.cpp | 160 ++--------- stdlib/public/runtime/ExclusivityPrivate.h | 139 ---------- stdlib/public/runtime/SwiftTLSContext.cpp | 1 + stdlib/public/runtime/SwiftTLSContext.h | 5 - stdlib/public/stubs/LibcShims.cpp | 5 + 16 files changed, 395 insertions(+), 311 deletions(-) create mode 100644 stdlib/public/core/Exclusivity.swift delete mode 100644 stdlib/public/runtime/ExclusivityPrivate.h diff --git a/Runtimes/Core/Core/CMakeLists.txt b/Runtimes/Core/Core/CMakeLists.txt index 023c23fe9e433..09e4a049dc202 100644 --- a/Runtimes/Core/Core/CMakeLists.txt +++ b/Runtimes/Core/Core/CMakeLists.txt @@ -74,6 +74,7 @@ add_library(swiftCore EmptyCollection.swift EnumeratedSequence.swift Equatable.swift + Exclusivity.swift ErrorType.swift ExistentialCollection.swift Filter.swift diff --git a/include/swift/Threading/Impl/Darwin.h b/include/swift/Threading/Impl/Darwin.h index 2cb48dc0bfff7..ac0f8baf4cb5f 100644 --- a/include/swift/Threading/Impl/Darwin.h +++ b/include/swift/Threading/Impl/Darwin.h @@ -285,6 +285,8 @@ inline tls_key_t tls_get_key(tls_key k) { return __PTK_FRAMEWORK_SWIFT_KEY5; case tls_key::observation_transaction: return __PTK_FRAMEWORK_SWIFT_KEY6; + case tls_key::exclusivity: + return __PTK_FRAMEWORK_SWIFT_KEY7; } } diff --git a/include/swift/Threading/TLSKeys.h b/include/swift/Threading/TLSKeys.h index 46f0398638faf..c7de97be4f8f9 100644 --- a/include/swift/Threading/TLSKeys.h +++ b/include/swift/Threading/TLSKeys.h @@ -22,7 +22,8 @@ enum class tls_key { concurrency_task, concurrency_executor_tracking_info, concurrency_fallback, - observation_transaction + observation_transaction, + exclusivity, }; } // namespace swift diff --git a/stdlib/public/SwiftShims/swift/shims/AssertionReporting.h b/stdlib/public/SwiftShims/swift/shims/AssertionReporting.h index faaf7d323c13e..2d105e9c73c5a 100644 --- a/stdlib/public/SwiftShims/swift/shims/AssertionReporting.h +++ b/stdlib/public/SwiftShims/swift/shims/AssertionReporting.h @@ -68,6 +68,13 @@ void _swift_stdlib_reportUnimplementedInitializer( const unsigned char *initName, int initNameLength, __swift_uint32_t flags); +SWIFT_RUNTIME_STDLIB_SPI +void _swift_reportExclusivityConflict(__swift_uintptr_t oldAction, + const void *_Nullable oldPC, + __swift_uintptr_t newFlags, + const void *_Nullable newPC, + const void *_Nullable pointer); + #ifdef __cplusplus } // extern "C" #endif diff --git a/stdlib/public/SwiftShims/swift/shims/LibcShims.h b/stdlib/public/SwiftShims/swift/shims/LibcShims.h index f727e6cf71c7a..27060e343f846 100644 --- a/stdlib/public/SwiftShims/swift/shims/LibcShims.h +++ b/stdlib/public/SwiftShims/swift/shims/LibcShims.h @@ -38,6 +38,9 @@ SWIFT_RUNTIME_STDLIB_INTERNAL __swift_size_t _swift_stdlib_fwrite_stdout(const void *ptr, __swift_size_t size, __swift_size_t nitems); +SWIFT_RUNTIME_STDLIB_INTERNAL +void _swift_stdlib_fputs_stderr(const char *str); + // General utilities // Memory management functions static inline void _swift_stdlib_free(void *_Nullable ptr) { diff --git a/stdlib/public/SwiftShims/swift/shims/RuntimeStubs.h b/stdlib/public/SwiftShims/swift/shims/RuntimeStubs.h index e17c3f766a30d..d7ac07c063100 100644 --- a/stdlib/public/SwiftShims/swift/shims/RuntimeStubs.h +++ b/stdlib/public/SwiftShims/swift/shims/RuntimeStubs.h @@ -20,6 +20,7 @@ #define SWIFT_STDLIB_SHIMS_RUNTIMESTUBS_H_ #include "LibcShims.h" +#include "Visibility.h" #ifdef __cplusplus extern "C" { @@ -31,6 +32,25 @@ SWIFT_RUNTIME_STDLIB_API __swift_ssize_t swift_stdlib_readLine_stdin(unsigned char * _Nullable * _Nonnull LinePtr); +// Pick a return-address strategy +#if defined(__wasm__) +// Wasm can't access call frame for security purposes +#define get_return_address() ((void*) 0) +#elif __GNUC__ +#define get_return_address() __builtin_return_address(0) +#elif _MSC_VER +#include +#define get_return_address() _ReturnAddress() +#else +#error missing implementation for get_return_address +#define get_return_address() ((void*) 0) +#endif + +SWIFT_ALWAYS_INLINE +static inline const void *_Nullable _swift_stdlib_get_return_address() { + return get_return_address(); +} + SWIFT_END_NULLABILITY_ANNOTATIONS #ifdef __cplusplus diff --git a/stdlib/public/SwiftShims/swift/shims/ThreadLocalStorage.h b/stdlib/public/SwiftShims/swift/shims/ThreadLocalStorage.h index a85d632edf1ca..aa0d1ed347e45 100644 --- a/stdlib/public/SwiftShims/swift/shims/ThreadLocalStorage.h +++ b/stdlib/public/SwiftShims/swift/shims/ThreadLocalStorage.h @@ -18,4 +18,48 @@ SWIFT_RUNTIME_STDLIB_INTERNAL void * _Nonnull _swift_stdlib_threadLocalStorageGet(void); +SWIFT_RUNTIME_STDLIB_INTERNAL +void * _Nullable _swift_getExclusivityTLSImpl(); + +SWIFT_RUNTIME_STDLIB_INTERNAL +void _swift_setExclusivityTLSImpl(void * _Nullable newValue); + +#if defined(__APPLE__) && __arm64__ + +// Use a fast path on Apple ARM64, where we have a dedicated TLS key and fast +// access to read/write it. + +#ifndef __PTK_FRAMEWORK_SWIFT_KEY7 +# define __PTK_FRAMEWORK_SWIFT_KEY7 107 +#endif + +#define SWIFT_RUNTIME_EXCLUSIVITY_KEY __PTK_FRAMEWORK_SWIFT_KEY7 + +static inline void * _Nullable * _Nonnull _swift_getExclusivityTLSPointer() { + unsigned long tsd; + __asm__ ("mrs %0, TPIDRRO_EL0" : "=r" (tsd)); + void **base = (void **)tsd; + return &base[SWIFT_RUNTIME_EXCLUSIVITY_KEY]; +} + +static inline void * _Nullable _swift_getExclusivityTLS() { + return *_swift_getExclusivityTLSPointer(); +} + +static inline void _swift_setExclusivityTLS(void * _Nullable newValue) { + *_swift_getExclusivityTLSPointer() = newValue; +} + +#else + +static inline void * _Nullable _swift_getExclusivityTLS() { + return _swift_getExclusivityTLSImpl(); +} + +static inline void _swift_setExclusivityTLS(void * _Nullable newValue) { + _swift_setExclusivityTLSImpl(newValue); +} + +#endif + #endif // SWIFT_STDLIB_SHIMS_THREADLOCALSTORAGE_H diff --git a/stdlib/public/core/CMakeLists.txt b/stdlib/public/core/CMakeLists.txt index b2238632d4e66..d7103f8533664 100644 --- a/stdlib/public/core/CMakeLists.txt +++ b/stdlib/public/core/CMakeLists.txt @@ -78,6 +78,7 @@ split_embedded_sources( EMBEDDED EnumeratedSequence.swift EMBEDDED Equatable.swift EMBEDDED ErrorType.swift + NORMAL Exclusivity.swift EMBEDDED ExistentialCollection.swift EMBEDDED Filter.swift EMBEDDED FlatMap.swift diff --git a/stdlib/public/core/Exclusivity.swift b/stdlib/public/core/Exclusivity.swift new file mode 100644 index 0000000000000..ede81d0db2c2f --- /dev/null +++ b/stdlib/public/core/Exclusivity.swift @@ -0,0 +1,262 @@ +import SwiftShims + +fileprivate let ValueBufferSize = unsafe 3 * MemoryLayout.stride + +fileprivate let TrackingFlag: UInt = 0x20 +fileprivate let ActionMask: UInt = 0x1 + +fileprivate typealias AccessPointer = UnsafeMutablePointer +@unsafe fileprivate struct Access { + + enum Action: UInt { + case read + case modify + } + + struct NextAndAction { + private var rawValue: UInt + + var action: Action { + get { + Action(rawValue: rawValue & ActionMask)! + } + set { + rawValue = (rawValue & ~ActionMask) | newValue.rawValue + } + } + + var next: AccessPointer? { + get { unsafe UnsafeMutablePointer(bitPattern: rawValue & ~ActionMask) } + set { rawValue = UInt(bitPattern: newValue) | (rawValue & ActionMask) } + } + } + + var location: UnsafeRawPointer? + var pc: UnsafeRawPointer? + var nextAndAction: NextAndAction + + var action: Action { + get { unsafe nextAndAction.action } + set { unsafe nextAndAction.action = newValue } + } + + var next: AccessPointer? { + get { unsafe nextAndAction.next } + set { unsafe nextAndAction.next = newValue } + } + + static func from(rawPointer: UnsafeMutableRawPointer?) -> AccessPointer? { + guard let rawPointer = unsafe rawPointer else { return nil } + return unsafe rawPointer.assumingMemoryBound(to: Access.self) + } + + @inline(__always) + static func search( + buffer: UnsafeMutableRawPointer, + location: UnsafeRawPointer, + pc: UnsafeRawPointer?, + action: Action, + inserting: Bool, + head: inout AccessPointer? + ) { + var cursor = unsafe head + while let nextPtr = unsafe cursor { + if unsafe nextPtr.pointee.location == location { + if unsafe nextPtr.pointee.action == Action.modify + || action == Action.modify { + unsafe _swift_reportExclusivityConflict( + nextPtr.pointee.action.rawValue, + nextPtr.pointee.pc, + action.rawValue, + pc, + location) + } + } + unsafe cursor = nextPtr.pointee.next + } + + if inserting { + guard let access = unsafe Access.from(rawPointer: buffer) else { + nullAccessBuffer() + } + + unsafe access.pointee.location = location + unsafe access.pointee.pc = pc + unsafe access.pointee.action = action + + unsafe access.pointee.next = head + unsafe head = access + } + } + + @inline(__always) + static func remove(access: AccessPointer, head: inout AccessPointer?) { + var cursor = unsafe head + var previous: AccessPointer? = nil + while let nextPtr = unsafe cursor { + if unsafe nextPtr == access { + if let previous = unsafe previous { + unsafe previous.pointee.next = access.pointee.next + } else { + unsafe head = access.pointee.next + } + return + } + unsafe previous = nextPtr + unsafe cursor = nextPtr.pointee.next + } + + unsafe accessNotFound(access) + } + + @inline(__always) + static func findParent( + access: AccessPointer, + child: AccessPointer? + ) -> AccessPointer? { + var cursor = unsafe access + while let next = unsafe cursor.pointee.next { + if unsafe next == child { + return unsafe cursor + } + unsafe cursor = next + } + + // If we were searching for nil, then we found it. + if unsafe child == nil { + return unsafe cursor + } + + // If we were searching for a non-nil node, we didn't find it. + return nil + } + + static func forEach(_ head: AccessPointer?, _ action: (Access) -> Void) { + var cursor = unsafe head + while let nextPtr = unsafe cursor { + unsafe action(nextPtr.pointee) + unsafe cursor = nextPtr.pointee.next + } + } +} + +fileprivate var accessHead: AccessPointer? { + get { unsafe Access.from(rawPointer: _swift_getExclusivityTLS()) } + set { unsafe _swift_setExclusivityTLS(newValue) } +} + +@_cdecl("swift_beginAccess") +@usableFromInline +@unsafe +internal func swift_beginAccess( + pointer: UnsafeRawPointer, + buffer: UnsafeMutableRawPointer, + flags: UInt, + pc: UnsafeRawPointer? +) { + // Make sure that Access fits in the value buffer. This should be a static + // assert, but we'll settle for a precondition for now. The optimizer should + // eliminate the check when it's true. + precondition(unsafe MemoryLayout.size <= ValueBufferSize) + + guard let action = Access.Action(rawValue: flags & ActionMask) else { + invalidFlags(flags) + } + + let isTracking = (flags & TrackingFlag) != 0 + + unsafe Access.search( + buffer: buffer, + location: pointer, + pc: pc ?? _swift_stdlib_get_return_address(), + action: action, + inserting: isTracking, + head: &accessHead) +} + +@_cdecl("swift_endAccess") +@usableFromInline +@unsafe +internal func swift_endAccess(buffer: UnsafeMutableRawPointer) { + guard let access = unsafe Access.from(rawPointer: buffer) else { + nullAccessBuffer() + } + unsafe Access.remove(access: access, head: &accessHead) +} + +@_cdecl("_swift_exclusivityAccessSetNext") +@usableFromInline +@unsafe +internal func _swift_exclusivityAccessSetNext( + access: UnsafeMutableRawPointer, + next: UnsafeMutableRawPointer +) { + let access = unsafe Access.from(rawPointer: access) + let next = unsafe Access.from(rawPointer: next) + unsafe access?.pointee.next = next +} + +@_cdecl("swift_dumpTrackedAccesses") +@usableFromInline +@unsafe +internal func swift_dumpTrackedAccesses() { + if let head = unsafe accessHead { + unsafe Access.forEach(head) { + unsafe _swift_stdlib_fputs_stderr(" Access. " + + "Pointer: \($0.location, default: "")." + + "PC: \($0.pc, default: ""). " + + "AccessAction: \($0.action)\n") + } + } else { + unsafe _swift_stdlib_fputs_stderr(" No Accesses.\n") + } +} + +/// Starting from `access`, find the access that is the parent node of `child`. +/// If `child` is `nil`, find the last access in the list. +@_cdecl("_swift_exclusivityAccessGetParent") +@usableFromInline +@unsafe +internal func _swift_exclusivityAccessGetParent( + access: UnsafeMutableRawPointer?, + child: UnsafeMutableRawPointer? +) -> UnsafeMutableRawPointer? { + if let access = unsafe Access.from(rawPointer: access) { + let result = unsafe Access.findParent( + access: access, child: Access.from(rawPointer: child)) + return UnsafeMutableRawPointer(result) + } + return nil +} + +@inline(never) +fileprivate func invalidFlags(_ flags: UInt) -> Never { + reportExclusivityError("Internal exclusivity error", + "unable to construct action from flags \(flags)") +} + +@inline(never) +fileprivate func accessNotFound(_ access: AccessPointer) -> Never { + unsafe reportExclusivityError("Internal exclusivity error", + "didn't find exclusive access buffer \(access)") +} + +@inline(never) +fileprivate func nullAccessBuffer() -> Never { + reportExclusivityError("Internal exclusivity error", "NULL access buffer") +} + +fileprivate func reportExclusivityError( + _ prefix: StaticString, _ message: String +) -> Never { + prefix.withUTF8Buffer { prefixBuffer in + var message = message + message.withUTF8 { messageBuffer in + unsafe _swift_stdlib_reportFatalError( + prefixBuffer.baseAddress!, CInt(prefixBuffer.count), + messageBuffer.baseAddress!, CInt(messageBuffer.count), + 0) + } + } + Builtin.int_trap() +} diff --git a/stdlib/public/core/GroupInfo.json b/stdlib/public/core/GroupInfo.json index 0c85960838b5b..8a02bb15773b4 100644 --- a/stdlib/public/core/GroupInfo.json +++ b/stdlib/public/core/GroupInfo.json @@ -276,7 +276,8 @@ "EmbeddedStubs.swift", "EmbeddedPrint.swift", "InlineArray.swift", - "_InlineArray.swift" + "_InlineArray.swift", + "Exclusivity.swift" ], "Result": [ "Result.swift" diff --git a/stdlib/public/runtime/ConcurrencyExclusivity.inc b/stdlib/public/runtime/ConcurrencyExclusivity.inc index 152201d15bb10..f28df7aafd681 100644 --- a/stdlib/public/runtime/ConcurrencyExclusivity.inc +++ b/stdlib/public/runtime/ConcurrencyExclusivity.inc @@ -215,17 +215,17 @@ struct SwiftTaskThreadLocalContext { return bool(state[0]); } - Access *getTaskAccessSetHead() const { - return reinterpret_cast(state[0]); + void *getTaskAccessSetHead() const { + return reinterpret_cast(state[0]); } - Access *getTaskAccessSetTail() const { - return reinterpret_cast(state[1]); + void *getTaskAccessSetTail() const { + return reinterpret_cast(state[1]); } - void setTaskAccessSetHead(Access *newHead) { state[0] = uintptr_t(newHead); } + void setTaskAccessSetHead(void *newHead) { state[0] = uintptr_t(newHead); } - void setTaskAccessSetTail(Access *newTail) { state[1] = uintptr_t(newTail); } + void setTaskAccessSetTail(void *newTail) { state[1] = uintptr_t(newTail); } #ifndef NDEBUG const char *getTaskAddress() const { @@ -251,8 +251,8 @@ struct SwiftTaskThreadLocalContext { // See algorithm description on SwiftTaskThreadLocalContext. void swift::swift_task_enterThreadLocalContext(char *state) { + auto *tlsHead = _swift_getExclusivityTLSImpl(); auto &taskCtx = *reinterpret_cast(state); - auto &tlsCtxAccessSet = SwiftTLSContext::get().accessSet; #ifndef NDEBUG if (isExclusivityLoggingEnabled()) { @@ -291,7 +291,7 @@ void swift::swift_task_enterThreadLocalContext(char *state) { // can just return early. // // Handles push cases 1-2. - if (!tlsCtxAccessSet) { + if (!tlsHead) { logEndState(); return; } @@ -302,7 +302,7 @@ void swift::swift_task_enterThreadLocalContext(char *state) { // nullptr. // // Handles push cases 3-4. - taskCtx.setTaskAccessSetHead(tlsCtxAccessSet.getHead()); + taskCtx.setTaskAccessSetHead(tlsHead); logEndState(); return; } @@ -316,8 +316,8 @@ void swift::swift_task_enterThreadLocalContext(char *state) { // TEnd to be nullptr and set the tlsCtx to point to TBegin. // // Handles push cases 5-6. - if (!bool(tlsCtxAccessSet)) { - tlsCtxAccessSet = taskCtx.getTaskAccessSetHead(); + if (!tlsHead) { + _swift_setExclusivityTLSImpl(taskCtx.getTaskAccessSetHead()); taskCtx.setTaskAccessSetHead(nullptr); taskCtx.setTaskAccessSetTail(nullptr); logEndState(); @@ -329,11 +329,11 @@ void swift::swift_task_enterThreadLocalContext(char *state) { // set tail->next to point at old head and stash oldhead into the task ctx. // // Handles push cases 7-8. - auto *oldHead = tlsCtxAccessSet.getHead(); + auto *oldHead = tlsHead; auto *tail = taskCtx.getTaskAccessSetTail(); - tlsCtxAccessSet.setHead(taskCtx.getTaskAccessSetHead()); - tail->setNext(oldHead); + _swift_setExclusivityTLSImpl(taskCtx.getTaskAccessSetHead()); + _swift_exclusivityAccessSetNext(tail, oldHead); taskCtx.setTaskAccessSetHead(oldHead); taskCtx.setTaskAccessSetTail(nullptr); logEndState(); @@ -341,8 +341,8 @@ void swift::swift_task_enterThreadLocalContext(char *state) { // See algorithm description on SwiftTaskThreadLocalContext. void swift::swift_task_exitThreadLocalContext(char *state) { + auto *tlsHead = _swift_getExclusivityTLSImpl(); auto &taskCtx = *reinterpret_cast(state); - auto &tlsCtxAccessSet = SwiftTLSContext::get().accessSet; #ifndef NDEBUG if (isExclusivityLoggingEnabled()) { @@ -379,7 +379,7 @@ void swift::swift_task_exitThreadLocalContext(char *state) { // initially tracking any accesses. // // Handles pop cases 1,2,5,6 - Access *oldHead = taskCtx.getTaskAccessSetHead(); + void *oldHead = taskCtx.getTaskAccessSetHead(); if (!oldHead) { // Then check if we are currently tracking an access set in the TLS. If we // aren't, then we know that either we did not start with a task specific @@ -389,7 +389,7 @@ void swift::swift_task_exitThreadLocalContext(char *state) { // we can just exit. // // Handles pop cases 1,5 - if (!tlsCtxAccessSet) { + if (!tlsHead) { assert(taskCtx.getTaskAccessSetTail() == nullptr && "Make sure we set this to nullptr when we pushed"); logEndState(); @@ -403,10 +403,10 @@ void swift::swift_task_exitThreadLocalContext(char *state) { // tail to the TLS linked list tail and set tlsCtx.accessSet to nullptr. // // Handles pop cases 2,6 - auto *newHead = tlsCtxAccessSet.getHead(); - auto *newTail = tlsCtxAccessSet.getTail(); + auto *newHead = tlsHead; + auto *newTail = _swift_exclusivityAccessGetParent(newHead, nullptr); assert(newTail && "Failed to find tail?!"); - tlsCtxAccessSet = nullptr; + _swift_setExclusivityTLSImpl(nullptr); taskCtx.setTaskAccessSetHead(newHead); taskCtx.setTaskAccessSetTail(newTail); logEndState(); @@ -424,7 +424,7 @@ void swift::swift_task_exitThreadLocalContext(char *state) { // access head/tail to nullptr. The end access should be nullptr. // // Handles pop cases 3. - if (tlsCtxAccessSet.getHead() == oldHead) { + if (tlsHead == oldHead) { taskCtx.setTaskAccessSetHead(nullptr); taskCtx.setTaskAccessSetTail(nullptr); logEndState(); @@ -440,10 +440,10 @@ void swift::swift_task_exitThreadLocalContext(char *state) { // we could perhaps avoid the search for newEnd. // // Handles pop cases 4,7,8. - auto *newHead = tlsCtxAccessSet.getHead(); - auto *newEnd = tlsCtxAccessSet.findParentAccess(oldHead); - tlsCtxAccessSet.setHead(oldHead); - newEnd->setNext(nullptr); + auto *newHead = tlsHead; + auto *newEnd = _swift_exclusivityAccessGetParent(tlsHead, oldHead); + _swift_setExclusivityTLSImpl(oldHead); + _swift_exclusivityAccessSetNext(newEnd, nullptr); taskCtx.setTaskAccessSetHead(newHead); taskCtx.setTaskAccessSetTail(newEnd); logEndState(); diff --git a/stdlib/public/runtime/Exclusivity.cpp b/stdlib/public/runtime/Exclusivity.cpp index fe769a91e2da5..2268c3d6ed57b 100644 --- a/stdlib/public/runtime/Exclusivity.cpp +++ b/stdlib/public/runtime/Exclusivity.cpp @@ -156,155 +156,35 @@ static void reportExclusivityConflict(ExclusivityFlags oldAction, void *oldPC, .notes = nullptr, }; _swift_reportToDebugger(RuntimeErrorFlagFatal, message, &details); + fatalError(0, "Fatal access conflict detected.\n"); } -bool AccessSet::insert(Access *access, void *pc, void *pointer, - ExclusivityFlags flags) { -#ifndef NDEBUG - if (isExclusivityLoggingEnabled()) { - withLoggingLock( - [&]() { fprintf(stderr, "Inserting new access: %p\n", access); }); - } -#endif - auto action = getAccessAction(flags); - - for (Access *cur = Head; cur != nullptr; cur = cur->getNext()) { - // Ignore accesses to different values. - if (cur->Pointer != pointer) - continue; - - // If both accesses are reads, it's not a conflict. - if (action == ExclusivityFlags::Read && action == cur->getAccessAction()) - continue; - - // Otherwise, it's a conflict. - reportExclusivityConflict(cur->getAccessAction(), cur->PC, flags, pc, - pointer); - - // 0 means no backtrace will be printed. - fatalError(0, "Fatal access conflict detected.\n"); - } - if (!isTracking(flags)) { -#ifndef NDEBUG - if (isExclusivityLoggingEnabled()) { - withLoggingLock([&]() { fprintf(stderr, " Not tracking!\n"); }); - } -#endif - return false; - } - - // Insert to the front of the array so that remove tends to find it faster. - access->initialize(pc, pointer, Head, action); - Head = access; -#ifndef NDEBUG - if (isExclusivityLoggingEnabled()) { - withLoggingLock([&]() { - fprintf(stderr, " Tracking!\n"); - swift_dumpTrackedAccesses(); - }); - } -#endif - return true; -} - -void AccessSet::remove(Access *access) { - assert(Head && "removal from empty AccessSet"); -#ifndef NDEBUG - if (isExclusivityLoggingEnabled()) { - withLoggingLock( - [&]() { fprintf(stderr, "Removing access: %p\n", access); }); - } -#endif - auto cur = Head; - // Fast path: stack discipline. - if (cur == access) { - Head = cur->getNext(); - return; - } - - Access *last = cur; - for (cur = cur->getNext(); cur != nullptr; last = cur, cur = cur->getNext()) { - assert(last->getNext() == cur); - if (cur == access) { - last->setNext(cur->getNext()); - return; - } - } - - swift_unreachable("access not found in set"); -} - -#ifndef NDEBUG -/// Only available with asserts. Intended to be used with -/// swift_dumpTrackedAccess(). -void AccessSet::forEach(std::function action) { - for (auto *iter = Head; iter != nullptr; iter = iter->getNext()) { - action(iter); - } -} -#endif - -// Each of these cases should define a function with this prototype: -// AccessSets &getAllSets(); - -/// Begin tracking a dynamic access. -/// -/// This may cause a runtime failure if an incompatible access is -/// already underway. -void swift::swift_beginAccess(void *pointer, ValueBuffer *buffer, - ExclusivityFlags flags, void *pc) { - assert(pointer && "beginning an access on a null pointer?"); - - Access *access = reinterpret_cast(buffer); - - // If exclusivity checking is disabled, record in the access buffer that we - // didn't track anything. pc is currently undefined in this case. - if (_swift_disableExclusivityChecking) { - access->Pointer = nullptr; - return; - } - - // If the provided `pc` is null, then the runtime may override it for - // diagnostics. - if (!pc) - pc = get_return_address(); - - if (!SwiftTLSContext::get().accessSet.insert(access, pc, pointer, flags)) - access->Pointer = nullptr; +SWIFT_RUNTIME_STDLIB_INTERNAL +void _swift_reportExclusivityConflict(uintptr_t oldAction, void *oldPC, + uintptr_t newFlags, void *newPC, + void *pointer) { + reportExclusivityConflict((ExclusivityFlags)oldAction, oldPC, + (ExclusivityFlags)newFlags, newPC, pointer); } -/// End tracking a dynamic access. -void swift::swift_endAccess(ValueBuffer *buffer) { - Access *access = reinterpret_cast(buffer); - auto pointer = access->Pointer; - - // If the pointer in the access is null, we must've declined - // to track it because exclusivity tracking was disabled. - if (!pointer) { - return; - } +static SWIFT_THREAD_LOCAL_TYPE(void *, swift::tls_key::exclusivity) AccessSetValue; - SwiftTLSContext::get().accessSet.remove(access); +SWIFT_RUNTIME_STDLIB_INTERNAL +void * _Nullable _swift_getExclusivityTLSImpl() { + return AccessSetValue.get(); } -#ifndef NDEBUG - -// Dump the accesses that are currently being tracked by the runtime. -// -// This is only intended to be used in the debugger. -void swift::swift_dumpTrackedAccesses() { - auto &accessSet = SwiftTLSContext::get().accessSet; - if (!accessSet) { - fprintf(stderr, " No Accesses.\n"); - return; - } - accessSet.forEach([](Access *a) { - fprintf(stderr, " Access. Pointer: %p. PC: %p. AccessAction: %s\n", - a->Pointer, a->PC, getAccessName(a->getAccessAction())); - }); +SWIFT_RUNTIME_STDLIB_INTERNAL +void _swift_setExclusivityTLSImpl(void * _Nullable newValue) { + AccessSetValue.set(newValue); } -#endif +// Declare two internal helpers from the Swift implementation that are used by +// the concurrency-specific code. +extern "C" void _swift_exclusivityAccessSetNext(void *access, + void *_Nullable next); +extern "C" void *_swift_exclusivityAccessGetParent(void *access, + void *_Nullable child); // Bring in the concurrency-specific exclusivity code. #include "ConcurrencyExclusivity.inc" diff --git a/stdlib/public/runtime/ExclusivityPrivate.h b/stdlib/public/runtime/ExclusivityPrivate.h deleted file mode 100644 index 35f0ffa134c2f..0000000000000 --- a/stdlib/public/runtime/ExclusivityPrivate.h +++ /dev/null @@ -1,139 +0,0 @@ -//===--- ExclusivityPrivate.h ---------------------------------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -#ifndef SWIFT_RUNTIME_EXCLUSIVITYPRIVATE_H -#define SWIFT_RUNTIME_EXCLUSIVITYPRIVATE_H - -#include "swift/Runtime/Exclusivity.h" -#include "swift/Runtime/Metadata.h" -#include - -namespace swift { -namespace runtime { - -/// A single access that we're tracking. -/// -/// The following inputs are accepted by the begin_access runtime entry -/// point. This table show the action performed by the current runtime to -/// convert those inputs into stored fields in the Access scratch buffer. -/// -/// Pointer | Runtime | Access | PC | Reported| Access -/// Argument| Behavior | Pointer| Arg | PC | PC -/// -------- ------------- -------- ------- --------- ---------- -/// null | [trap or missing enforcement] -/// nonnull | [nontracked]| null | null | caller | [discard] -/// nonnull | [nontracked]| null | valid | | [discard] -/// nonnull | [tracked] | | null | caller | caller -/// nonnull | [tracked] | | valid | | -/// -/// [nontracked] means that the Access scratch buffer will not be added to the -/// runtime's list of tracked accesses. However, it may be passed to a -/// subsequent call to end_unpaired_access. The null Pointer field then -/// identifies the Access record as nontracked. -/// -/// The runtime owns the contents of the scratch buffer, which is allocated by -/// the compiler but otherwise opaque. The runtime may later reuse the Pointer -/// or PC fields or any spare bits for additional flags, and/or a pointer to -/// out-of-line storage. -struct Access { - void *Pointer; - void *PC; - uintptr_t NextAndAction; - - enum : uintptr_t { - ActionMask = (uintptr_t)ExclusivityFlags::ActionMask, - NextMask = ~ActionMask - }; - - Access *getNext() const { - return reinterpret_cast(NextAndAction & NextMask); - } - - void setNext(Access *next) { - NextAndAction = - reinterpret_cast(next) | (NextAndAction & ActionMask); - } - - ExclusivityFlags getAccessAction() const { - return ExclusivityFlags(NextAndAction & ActionMask); - } - - void initialize(void *pc, void *pointer, Access *next, - ExclusivityFlags action) { - Pointer = pointer; - PC = pc; - NextAndAction = reinterpret_cast(next) | uintptr_t(action); - } -}; - -static_assert(sizeof(Access) <= sizeof(ValueBuffer) && - alignof(Access) <= alignof(ValueBuffer), - "Access doesn't fit in a value buffer!"); - -/// A set of accesses that we're tracking. Just a singly-linked list. -/// -/// NOTE: Please keep all implementations of methods of AccessSet within -/// Exclusivity.cpp. We want this to ensure that when compiled in the runtime -/// directly, the definitions of these methods are immediately available in that -/// file for inlining. -class AccessSet { - Access *Head = nullptr; - -public: - constexpr AccessSet() {} - constexpr AccessSet(Access *Head) : Head(Head) {} - - constexpr operator bool() const { return bool(Head); } - constexpr Access *getHead() const { return Head; } - void setHead(Access *newHead) { Head = newHead; } - constexpr bool isHead(Access *access) const { return Head == access; } - - bool insert(Access *access, void *pc, void *pointer, ExclusivityFlags flags); - void remove(Access *access); - - /// Return the parent access of \p childAccess in the list. - Access *findParentAccess(Access *childAccess) const { - auto cur = Head; - Access *last = cur; - for (cur = cur->getNext(); cur != nullptr; - last = cur, cur = cur->getNext()) { - assert(last->getNext() == cur); - if (cur == childAccess) { - return last; - } - } - return nullptr; - } - - Access *getTail() const { - auto cur = Head; - if (!cur) - return nullptr; - - while (auto *next = cur->getNext()) { - cur = next; - } - assert(cur != nullptr); - return cur; - } - -#ifndef NDEBUG - /// Only available with asserts. Intended to be used with - /// swift_dumpTrackedAccess(). - void forEach(std::function action); -#endif -}; - -} // namespace runtime -} // namespace swift - -#endif diff --git a/stdlib/public/runtime/SwiftTLSContext.cpp b/stdlib/public/runtime/SwiftTLSContext.cpp index 3959c1d4fa44a..c5e6e56a1783b 100644 --- a/stdlib/public/runtime/SwiftTLSContext.cpp +++ b/stdlib/public/runtime/SwiftTLSContext.cpp @@ -12,6 +12,7 @@ #include "SwiftTLSContext.h" +#include "swift/Runtime/Heap.h" #include "swift/Threading/Once.h" #include "swift/Threading/ThreadLocalStorage.h" diff --git a/stdlib/public/runtime/SwiftTLSContext.h b/stdlib/public/runtime/SwiftTLSContext.h index 94ffef8e0dcfe..4b69dbdda40e8 100644 --- a/stdlib/public/runtime/SwiftTLSContext.h +++ b/stdlib/public/runtime/SwiftTLSContext.h @@ -13,16 +13,11 @@ #ifndef SWIFT_RUNTIME_SWIFTTLSCONTEXT_H #define SWIFT_RUNTIME_SWIFTTLSCONTEXT_H -#include "ExclusivityPrivate.h" - namespace swift { namespace runtime { class SwiftTLSContext { public: - /// The set of tracked accesses. - AccessSet accessSet; - // The "implicit" boolean parameter which is passed to a dynamically // replaceable function. // If true, the original function should be executed instead of the diff --git a/stdlib/public/stubs/LibcShims.cpp b/stdlib/public/stubs/LibcShims.cpp index 1fbfaa7229096..f7623d8022bde 100644 --- a/stdlib/public/stubs/LibcShims.cpp +++ b/stdlib/public/stubs/LibcShims.cpp @@ -47,3 +47,8 @@ __swift_size_t _swift_stdlib_fwrite_stdout(const void *ptr, __swift_size_t nitems) { return fwrite(ptr, size, nitems, stdout); } + +SWIFT_RUNTIME_STDLIB_INTERNAL +void _swift_stdlib_fputs_stderr(const char *str) { + fputs(str, stderr); +}