exclude shadow from coredump

Author: yingcong-wu

source/loader/layers/sanitizer/asan_shadow_setup.cpp

@@ -29,6 +29,7 @@ ur_result_t SetupShadowMemory(uptr &ShadowBegin, uptr &ShadowEnd) {
         if (SHADOW_BEGIN == 0) {
             return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY;
         }
+        DontCoredumpRange(SHADOW_BEGIN, SHADOW_SIZE);
         SHADOW_END = SHADOW_BEGIN + SHADOW_SIZE;
         IsShadowMemInited = true;
         return UR_RESULT_SUCCESS;

source/loader/layers/sanitizer/common.hpp

@@ -142,6 +142,7 @@ bool IsInASanContext();
 
 uptr MmapNoReserve(uptr Addr, uptr Size);
 bool Munmap(uptr Addr, uptr Size);
+bool DontCoredumpRange(uptr Addr, uptr Size);
 
 void *GetMemFunctionPointer(const char *);
 

source/loader/layers/sanitizer/linux/sanitizer_utils.cpp

@@ -37,6 +37,12 @@ uptr MmapNoReserve(uptr Addr, uptr Size) {
 
 bool Munmap(uptr Addr, uptr Size) { return munmap((void *)Addr, Size) == 0; }
 
+bool DontCoredumpRange(uptr Addr, uptr Size) {
+    Size = RoundUpTo(Size, EXEC_PAGESIZE);
+    Addr = RoundDownTo(Addr, EXEC_PAGESIZE);
+    return madvise((void *)Addr, Size, MADV_DONTDUMP) == 0;
+}
+
 void *GetMemFunctionPointer(const char *FuncName) {
     void *handle = dlopen(LIBC_SO, RTLD_LAZY | RTLD_NOLOAD);
     if (!handle) {