[rtsan] Added mmap and shm interceptors (#114862)

# Why do we think these are unsafe?

mmap and munmap are used for mapping a specific manipulation of the
virtual memory of a process. Typically used in things like databases and
area allocators.

They call the system calls `mmap` and `munmap` under the hood (confirmed
on mac using dtrace)

shm_open/unlink interact with shared memory across processes. 

Similarly under the hood, they call the `shm_open`/`shm_unlink` system
calls (confirmed on mac using dtrace)

Author: cjappl

compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp

@@ -504,6 +504,38 @@ INTERCEPTOR(void *, pvalloc, size_t size) {
 }
 #endif
 
+INTERCEPTOR(void *, mmap, void *addr, size_t length, int prot, int flags,
+            int fd, off_t offset) {
+  __rtsan_notify_intercepted_call("mmap");
+  return REAL(mmap)(addr, length, prot, flags, fd, offset);
+}
+
+#if SANITIZER_INTERCEPT_MMAP64
+INTERCEPTOR(void *, mmap64, void *addr, size_t length, int prot, int flags,
+            int fd, off64_t offset) {
+  __rtsan_notify_intercepted_call("mmap64");
+  return REAL(mmap64)(addr, length, prot, flags, fd, offset);
+}
+#define RTSAN_MAYBE_INTERCEPT_MMAP64 INTERCEPT_FUNCTION(mmap64)
+#else
+#define RTSAN_MAYBE_INTERCEPT_MMAP64
+#endif // SANITIZER_INTERCEPT_MMAP64
+
+INTERCEPTOR(int, munmap, void *addr, size_t length) {
+  __rtsan_notify_intercepted_call("munmap");
+  return REAL(munmap)(addr, length);
+}
+
+INTERCEPTOR(int, shm_open, const char *name, int oflag, mode_t mode) {
+  __rtsan_notify_intercepted_call("shm_open");
+  return REAL(shm_open)(name, oflag, mode);
+}
+
+INTERCEPTOR(int, shm_unlink, const char *name) {
+  __rtsan_notify_intercepted_call("shm_unlink");
+  return REAL(shm_unlink)(name);
+}
+
 // Sockets
 INTERCEPTOR(int, socket, int domain, int type, int protocol) {
   __rtsan_notify_intercepted_call("socket");
@@ -558,6 +590,11 @@ void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(valloc);
   RTSAN_MAYBE_INTERCEPT_ALIGNED_ALLOC;
   INTERCEPT_FUNCTION(posix_memalign);
+  INTERCEPT_FUNCTION(mmap);
+  RTSAN_MAYBE_INTERCEPT_MMAP64;
+  INTERCEPT_FUNCTION(munmap);
+  INTERCEPT_FUNCTION(shm_open);
+  INTERCEPT_FUNCTION(shm_unlink);
 #if SANITIZER_INTERCEPT_MEMALIGN
   INTERCEPT_FUNCTION(memalign);
 #endif

compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp

@@ -36,6 +36,7 @@
 #include <fcntl.h>
 #include <pthread.h>
 #include <stdio.h>
+#include <sys/mman.h>
 #include <sys/socket.h>
 #include <sys/uio.h>
 
@@ -47,6 +48,7 @@ const char *const kOpenAtFunctionName = "openat64";
 const char *const kOpenFunctionName = "open64";
 const char *const kPreadFunctionName = "pread64";
 const char *const kPwriteFunctionName = "pwrite64";
+const char *const kMmapFunctionName = "mmap64";
 #else
 const char *const kCreatFunctionName = "creat";
 const char *const kFcntlFunctionName = "fcntl";
@@ -55,6 +57,7 @@ const char *const kOpenAtFunctionName = "openat";
 const char *const kOpenFunctionName = "open";
 const char *const kPreadFunctionName = "pread";
 const char *const kPwriteFunctionName = "pwrite";
+const char *const kMmapFunctionName = "mmap";
 #endif
 
 using namespace testing;
@@ -179,6 +182,37 @@ TEST(TestRtsanInterceptors, PvallocDiesWhenRealtime) {
 }
 #endif
 
+TEST(TestRtsanInterceptors, MmapDiesWhenRealtime) {
+  auto Func = []() {
+    void *_ = mmap(nullptr, 8, PROT_READ | PROT_WRITE,
+                   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  };
+  ExpectRealtimeDeath(Func, kMmapFunctionName);
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST(TestRtsanInterceptors, MunmapDiesWhenRealtime) {
+  void *ptr = mmap(nullptr, 8, PROT_READ | PROT_WRITE,
+                   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  EXPECT_NE(ptr, nullptr);
+  auto Func = [ptr]() { munmap(ptr, 8); };
+  printf("Right before death munmap\n");
+  ExpectRealtimeDeath(Func, "munmap");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST(TestRtsanInterceptors, ShmOpenDiesWhenRealtime) {
+  auto Func = []() { shm_open("/rtsan_test_shm", O_CREAT | O_RDWR, 0); };
+  ExpectRealtimeDeath(Func, "shm_open");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST(TestRtsanInterceptors, ShmUnlinkDiesWhenRealtime) {
+  auto Func = []() { shm_unlink("/rtsan_test_shm"); };
+  ExpectRealtimeDeath(Func, "shm_unlink");
+  ExpectNonRealtimeSurvival(Func);
+}
+
 /*
     Sleeping
 */