[rtsan] Warn if instrumented rtsan library opened via dlopen and interceptors are not working

[cjappl]

Many of our users in audio land load RTSan instrumented shared libraries (plug-ins) into host processes (DAWs) via dlopen.

This surprised one user, as the interceptors were just silently failing with no indication as to what was going on.

To combat this, we wanted to get the same error message that ASan and TSan had, which happens when you call InitializePlatformEarly:

==9891==ERROR: Interceptors are not working. This may be because RealtimeSanitizer is loaded too late (e.g. via dlopen). Please launch the executable with:
DYLD_INSERT_LIBRARIES=/usr/local/lib/libclang_rt.rtsan_osx_dynamic.dylib
"interceptors not installed" && 0
fish: Job 1, '/Applications/REAPER.app/Conten…' terminated by signal SIGABRT (Abort)

The test in this PR is almost an exact copy of the tsan version of the same thing:

https://github.com/llvm/llvm-project/blob/main/compiler-rt/test/tsan/Darwin/dlopen.cpp

[llvmbot]

@llvm/pr-subscribers-compiler-rt-sanitizer

Author: Chris Apple (cjappl)

Changes

Many of our users in audio land load RTSan instrumented shared libraries (plug-ins) into host processes (DAWs) via dlopen.

This surprised one user, as the interceptors were just silently failing with no indication as to what was going on.

To combat this, we wanted to get the same error message that ASan and TSan had, which happens when you call InitializePlatformEarly:

==9891==ERROR: Interceptors are not working. This may be because RealtimeSanitizer is loaded too late (e.g. via dlopen). Please launch the executable with:
DYLD_INSERT_LIBRARIES=/usr/local/lib/libclang_rt.rtsan_osx_dynamic.dylib
"interceptors not installed" && 0
fish: Job 1, '/Applications/REAPER.app/Conten…' terminated by signal SIGABRT (Abort)

The test in this PR is almost an exact copy of the tsan version of the same thing:

https://github.com/llvm/llvm-project/blob/main/compiler-rt/test/tsan/Darwin/dlopen.cpp

---

Full diff: https://github.com/llvm/llvm-project/pull/119029.diff

4 Files Affected:

• (modified) compiler-rt/lib/rtsan/rtsan.cpp (+3)
• (modified) compiler-rt/lib/rtsan/tests/rtsan_test_main.cpp (+4-1)
• (modified) compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp (+3-2)
• (added) compiler-rt/test/rtsan/Darwin/dlopen.cpp (+44)

diff --git a/compiler-rt/lib/rtsan/rtsan.cpp b/compiler-rt/lib/rtsan/rtsan.cpp
index 81cedb3b5114f0..73340b34f6d4bb 100644
--- a/compiler-rt/lib/rtsan/rtsan.cpp
+++ b/compiler-rt/lib/rtsan/rtsan.cpp
@@ -83,6 +83,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_init() {
 
   SanitizerToolName = "RealtimeSanitizer";
   InitializeFlags();
+
+  InitializePlatformEarly();
+
   InitializeInterceptors();
 
   InitializeSuppressions();
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_main.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_main.cpp
index 50c726e09f287f..ca560fc21035c5 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_main.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_main.cpp
@@ -19,7 +19,10 @@ extern "C" const char *__rtsan_default_options() {
   // and make sure we do not overwhelm the syslog while testing. Also, let's
   // turn symbolization off to speed up testing, especially when not running
   // with llvm-symbolizer but with atos.
-  return "symbolize=false:abort_on_error=0:log_to_syslog=0";
+  return "symbolize=false:"
+         "abort_on_error=0:"
+         "log_to_syslog=0:"
+         "verify_interceptors=0:"; // some of our tests don't need interceptors
 #else
   // Let's turn symbolization off to speed up testing (more than 3 times speedup
   // observed).
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index 26d2e8d4ed7680..c8a0afccb254e5 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -972,8 +972,9 @@ static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES";
 LowLevelAllocator allocator_for_env;
 
 static bool ShouldCheckInterceptors() {
-  // Restrict "interceptors working?" check to ASan and TSan.
-  const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer"};
+  // Restrict "interceptors working?" check
+  const char *sanitizer_names[] = {"AddressSanitizer", "ThreadSanitizer",
+                                   "RealtimeSanitizer"};
   size_t count = sizeof(sanitizer_names) / sizeof(sanitizer_names[0]);
   for (size_t i = 0; i < count; i++) {
     if (internal_strcmp(sanitizer_names[i], SanitizerToolName) == 0)
diff --git a/compiler-rt/test/rtsan/Darwin/dlopen.cpp b/compiler-rt/test/rtsan/Darwin/dlopen.cpp
new file mode 100644
index 00000000000000..1aabe5cb6e580a
--- /dev/null
+++ b/compiler-rt/test/rtsan/Darwin/dlopen.cpp
@@ -0,0 +1,44 @@
+// Checks that on OS X 10.11+ dlopen'ing a RTsanified library from a
+// non-instrumented program exits with a user-friendly message.
+
+// REQUIRES: osx-autointerception
+
+// XFAIL: ios
+
+// RUN: %clangxx -fsanitize=realtime %s -o %t.so -shared -DSHARED_LIB
+// RUN: %clangxx %s -o %t
+
+// RUN: RTSAN_DYLIB_PATH=`%clangxx -fsanitize=realtime %s -### 2>&1 \
+// RUN:   | grep "libclang_rt.rtsan_osx_dynamic.dylib" \
+// RUN:   | sed -e 's/.*"\(.*libclang_rt.rtsan_osx_dynamic.dylib\)".*/\1/'`
+
+// Launching a non-instrumented binary that dlopen's an instrumented library should fail.
+// RUN: not %run %t %t.so 2>&1 | FileCheck %s --check-prefix=CHECK-FAIL
+// Launching a non-instrumented binary with an explicit DYLD_INSERT_LIBRARIES should work.
+// RUN: DYLD_INSERT_LIBRARIES=$RTSAN_DYLIB_PATH %run %t %t.so 2>&1 | FileCheck %s
+
+// Launching an instrumented binary with the DYLD_INSERT_LIBRARIES env variable has no error
+// RUN: %clangxx -fsanitize=realtime %s -o %t
+// RUN: DYLD_INSERT_LIBRARIES=$RTSAN_DYLIB_PATH %run %t %t.so 2>&1 | FileCheck %s --check-prefix=CHECK-INSTRUMENTED
+
+#include <dlfcn.h>
+#include <stdio.h>
+
+#if defined(SHARED_LIB)
+extern "C" void foo() { fprintf(stderr, "Hello world.\n"); }
+#else  // defined(SHARED_LIB)
+int main(int argc, char *argv[]) {
+  void *handle = dlopen(argv[1], RTLD_NOW);
+  void (*foo)() = (void (*)())dlsym(handle, "foo");
+  foo();
+}
+#endif // defined(SHARED_LIB)
+
+// CHECK: Hello world.
+// CHECK-NOT: ERROR: Interceptors are not working.
+
+// CHECK-FAIL-NOT: Hello world.
+// CHECK-FAIL: ERROR: Interceptors are not working.
+
+// CHECK-INSTRUMENTED-NOT: ERROR: Interceptors are not working
+// CHECK-INSTRUMENTED: Hello world.