[Runtime] Faster dynamic exclusivity checking implemented in Swift.

Replace C++ implementation of swift_beginAccess and swift_endAccess with (almost) pure Swift implementation. Helpers remain in C++ for TLS, getting return addresses, and raising a fatal error on violations.

This change also moves the exclusivity access set head from the shared SwiftTLSContext structure to a dedicated TLS key. This improves performance, which is important for exclusivity checking. This is particularly the case where we can inline TLS access with a constant key, as on Darwin ARM64.

The code that bridges exclusivity tracking into Concurrency remains in C++. The new Swift implementation exposes a few helpers for it to use as a replacement for directly manipulating the C++ implementation.

rdar://161122309

Author: mikeash

Runtimes/Core/Core/CMakeLists.txt

@@ -74,6 +74,7 @@ add_library(swiftCore
   EmptyCollection.swift
   EnumeratedSequence.swift
   Equatable.swift
+  Exclusivity.swift
   ErrorType.swift
   ExistentialCollection.swift
   Filter.swift

include/swift/Threading/Impl/Darwin.h

@@ -285,6 +285,8 @@ inline tls_key_t tls_get_key(tls_key k) {
     return __PTK_FRAMEWORK_SWIFT_KEY5;
   case tls_key::observation_transaction:
     return __PTK_FRAMEWORK_SWIFT_KEY6;
+  case tls_key::exclusivity:
+    return __PTK_FRAMEWORK_SWIFT_KEY7;
   }
 }
 

include/swift/Threading/TLSKeys.h

@@ -22,7 +22,8 @@ enum class tls_key {
   concurrency_task,
   concurrency_executor_tracking_info,
   concurrency_fallback,
-  observation_transaction
+  observation_transaction,
+  exclusivity,
 };
 
 } // namespace swift

stdlib/public/SwiftShims/swift/shims/AssertionReporting.h

@@ -68,6 +68,13 @@ void _swift_stdlib_reportUnimplementedInitializer(
     const unsigned char *initName, int initNameLength,
     __swift_uint32_t flags);
 
+SWIFT_RUNTIME_STDLIB_SPI
+void _swift_reportExclusivityConflict(__swift_uintptr_t oldAction,
+                                      const void *_Nullable oldPC,
+                                      __swift_uintptr_t newFlags,
+                                      const void *_Nullable newPC,
+                                      const void *_Nullable pointer);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

stdlib/public/SwiftShims/swift/shims/LibcShims.h

@@ -38,6 +38,9 @@ SWIFT_RUNTIME_STDLIB_INTERNAL
 __swift_size_t _swift_stdlib_fwrite_stdout(const void *ptr, __swift_size_t size,
                                            __swift_size_t nitems);
 
+SWIFT_RUNTIME_STDLIB_INTERNAL
+void _swift_stdlib_fputs_stderr(const char *str);
+
 // General utilities <stdlib.h>
 // Memory management functions
 static inline void _swift_stdlib_free(void *_Nullable ptr) {

stdlib/public/SwiftShims/swift/shims/RuntimeStubs.h

@@ -20,6 +20,7 @@
 #define SWIFT_STDLIB_SHIMS_RUNTIMESTUBS_H_
 
 #include "LibcShims.h"
+#include "Visibility.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -31,6 +32,25 @@ SWIFT_RUNTIME_STDLIB_API
 __swift_ssize_t
 swift_stdlib_readLine_stdin(unsigned char * _Nullable * _Nonnull LinePtr);
 
+// Pick a return-address strategy
+#if defined(__wasm__)
+// Wasm can't access call frame for security purposes
+#define get_return_address() ((void*) 0)
+#elif __GNUC__
+#define get_return_address() __builtin_return_address(0)
+#elif _MSC_VER
+#include <intrin.h>
+#define get_return_address() _ReturnAddress()
+#else
+#error missing implementation for get_return_address
+#define get_return_address() ((void*) 0)
+#endif
+
+SWIFT_ALWAYS_INLINE
+static inline const void *_Nullable _swift_stdlib_get_return_address() {
+  return get_return_address();
+}
+
 SWIFT_END_NULLABILITY_ANNOTATIONS
 
 #ifdef __cplusplus

stdlib/public/SwiftShims/swift/shims/ThreadLocalStorage.h

@@ -18,4 +18,48 @@
 SWIFT_RUNTIME_STDLIB_INTERNAL
 void * _Nonnull _swift_stdlib_threadLocalStorageGet(void);
 
+SWIFT_RUNTIME_STDLIB_INTERNAL
+void * _Nullable _swift_getExclusivityTLSImpl();
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+void _swift_setExclusivityTLSImpl(void * _Nullable newValue);
+
+#if defined(__APPLE__) && __arm64__
+
+// Use a fast path on Apple ARM64, where we have a dedicated TLS key and fast
+// access to read/write it.
+
+#ifndef __PTK_FRAMEWORK_SWIFT_KEY7
+# define __PTK_FRAMEWORK_SWIFT_KEY7 107
+#endif
+
+#define SWIFT_RUNTIME_EXCLUSIVITY_KEY __PTK_FRAMEWORK_SWIFT_KEY7
+
+static inline void * _Nullable * _Nonnull _swift_getExclusivityTLSPointer() {
+  unsigned long tsd;
+  __asm__ ("mrs %0, TPIDRRO_EL0" : "=r" (tsd));
+  void **base = (void **)tsd;
+  return &base[SWIFT_RUNTIME_EXCLUSIVITY_KEY];
+}
+
+static inline void * _Nullable _swift_getExclusivityTLS() {
+  return *_swift_getExclusivityTLSPointer();
+}
+
+static inline void _swift_setExclusivityTLS(void * _Nullable newValue) {
+  *_swift_getExclusivityTLSPointer() = newValue;
+}
+
+#else
+
+static inline void * _Nullable _swift_getExclusivityTLS() {
+  return _swift_getExclusivityTLSImpl();
+}
+
+static inline void _swift_setExclusivityTLS(void * _Nullable newValue) {
+  _swift_setExclusivityTLSImpl(newValue);
+}
+
+#endif
+
 #endif // SWIFT_STDLIB_SHIMS_THREADLOCALSTORAGE_H

stdlib/public/core/CMakeLists.txt

@@ -78,6 +78,7 @@ split_embedded_sources(
   EMBEDDED EnumeratedSequence.swift
   EMBEDDED Equatable.swift
   EMBEDDED ErrorType.swift
+    NORMAL Exclusivity.swift
   EMBEDDED ExistentialCollection.swift
   EMBEDDED Filter.swift
   EMBEDDED FlatMap.swift