[rtsan] Make sure rtsan gets initialized on mac (#100188)

Intermittently on my mac I was getting the same nullptr crash in dlsym.

We need to make sure rtsan gets initialized on mac between when the
binary starts running, and the first intercepted function is called.
Until that point we should use the DlsymAllocator.

Author: cjappl

compiler-rt/lib/rtsan/rtsan.cpp

@@ -12,23 +12,42 @@
 #include <rtsan/rtsan_context.h>
 #include <rtsan/rtsan_interceptors.h>
 
+#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_mutex.h"
+
 using namespace __rtsan;
+using namespace __sanitizer;
+
+static StaticSpinMutex rtsan_inited_mutex;
+static atomic_uint8_t rtsan_initialized = {0};
 
-bool __rtsan::rtsan_initialized;
-bool __rtsan::rtsan_init_is_running;
+static void SetInitialized() {
+  atomic_store(&rtsan_initialized, 1, memory_order_release);
+}
 
 extern "C" {
 
 SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_init() {
-  CHECK(!rtsan_init_is_running);
-  if (rtsan_initialized)
+  CHECK(!__rtsan_is_initialized());
+  InitializeInterceptors();
+  SetInitialized();
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_ensure_initialized() {
+  if (LIKELY(__rtsan_is_initialized()))
     return;
-  rtsan_init_is_running = true;
 
-  InitializeInterceptors();
+  SpinMutexLock lock(&rtsan_inited_mutex);
+
+  // Someone may have initialized us while we were waiting for the lock
+  if (__rtsan_is_initialized())
+    return;
+
+  __rtsan_init();
+}
 
-  rtsan_init_is_running = false;
-  rtsan_initialized = true;
+SANITIZER_INTERFACE_ATTRIBUTE bool __rtsan_is_initialized() {
+  return atomic_load(&rtsan_initialized, memory_order_acquire) == 1;
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_realtime_enter() {

compiler-rt/lib/rtsan/rtsan.h

@@ -14,17 +14,17 @@
 
 extern "C" {
 
-namespace __rtsan {
-
-extern bool rtsan_initialized;
-extern bool rtsan_init_is_running;
-
-} // namespace __rtsan
-
 // Initialise rtsan interceptors.
 // A call to this method is added to the preinit array on Linux systems.
 SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_init();
 
+// Initializes rtsan if it has not been initialized yet.
+// Used by the RTSan runtime to ensure that rtsan is initialized before any
+// other rtsan functions are called.
+SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_ensure_initialized();
+
+SANITIZER_INTERFACE_ATTRIBUTE bool __rtsan_is_initialized();
+
 // Enter real-time context.
 // When in a real-time context, RTSan interceptors will error if realtime
 // violations are detected. Calls to this method are injected at the code

compiler-rt/lib/rtsan/rtsan_interceptors.cpp

@@ -51,16 +51,15 @@ void OSSpinLockLock(volatile OSSpinLock *__lock);
 
 using namespace __sanitizer;
 
-using __rtsan::rtsan_init_is_running;
-using __rtsan::rtsan_initialized;
-
 namespace {
 struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
-  static bool UseImpl() { return !rtsan_initialized; }
+  static bool UseImpl() { return !__rtsan_is_initialized(); }
 };
 } // namespace
 
 void ExpectNotRealtime(const char *intercepted_function_name) {
+  __rtsan_ensure_initialized();
+
   __rtsan::GetContextForThisThread().ExpectNotRealtime(
       intercepted_function_name);
 }