[HWASan] Build separate LAM runtime on x86_64.

Since we have both aliasing mode and Intel LAM on x86_64, we need to
choose the mode at either run time or compile time.  This patch
implements the plumbing to build both and choose between them at
compile time.

Reviewed By: vitalybuka, eugenis

Differential Revision: https://reviews.llvm.org/D102286

Author: morehouse

clang/lib/Driver/ToolChains/CommonArgs.cpp

@@ -816,8 +816,12 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
     }
     if (SanArgs.needsTsanRt() && SanArgs.linkRuntimes())
       SharedRuntimes.push_back("tsan");
-    if (SanArgs.needsHwasanRt() && SanArgs.linkRuntimes())
-      SharedRuntimes.push_back("hwasan");
+    if (SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) {
+      if (SanArgs.needsHwasanAliasesRt())
+        SharedRuntimes.push_back("hwasan_aliases");
+      else
+        SharedRuntimes.push_back("hwasan");
+    }
   }
 
   // The stats_client library is also statically linked into DSOs.
@@ -847,9 +851,15 @@ collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
   }
 
   if (!SanArgs.needsSharedRt() && SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) {
-    StaticRuntimes.push_back("hwasan");
-    if (SanArgs.linkCXXRuntimes())
-      StaticRuntimes.push_back("hwasan_cxx");
+    if (SanArgs.needsHwasanAliasesRt()) {
+      StaticRuntimes.push_back("hwasan_aliases");
+      if (SanArgs.linkCXXRuntimes())
+        StaticRuntimes.push_back("hwasan_aliases_cxx");
+    } else {
+      StaticRuntimes.push_back("hwasan");
+      if (SanArgs.linkCXXRuntimes())
+        StaticRuntimes.push_back("hwasan_cxx");
+    }
   }
   if (SanArgs.needsDfsanRt() && SanArgs.linkRuntimes())
     StaticRuntimes.push_back("dfsan");

compiler-rt/lib/hwasan/CMakeLists.txt

@@ -93,54 +93,86 @@ add_compiler_rt_object_libraries(RTHwasan_dynamic
   CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
   DEFS ${HWASAN_DEFINITIONS})
 
+# Compile a different runtime for x86 aliasing mode.
+set(HWASAN_ALIASES_RTL_CFLAGS ${HWASAN_RTL_CFLAGS})
+list(APPEND HWASAN_ALIASES_RTL_CFLAGS -DHWASAN_ALIASING_MODE)
+set(HWASAN_ALIASES_DYNAMIC_CFLAGS ${HWASAN_DYNAMIC_CFLAGS})
+list(APPEND HWASAN_ALIASES_DYNAMIC_CFLAGS -DHWASAN_ALIASING_MODE)
+add_compiler_rt_object_libraries(RTHwasanAliases
+  ARCHS ${HWASAN_SUPPORTED_ARCH}
+  SOURCES ${HWASAN_RTL_SOURCES}
+  ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
+  CFLAGS ${HWASAN_ALIASES_RTL_CFLAGS}
+  DEFS ${HWASAN_DEFINITIONS})
+add_compiler_rt_object_libraries(RTHwasanAliases_dynamic
+  ARCHS ${HWASAN_SUPPORTED_ARCH}
+  SOURCES ${HWASAN_RTL_SOURCES} ${HWASAN_RTL_CXX_SOURCES}
+  ADDITIONAL_HEADERS ${HWASAN_RTL_HEADERS}
+  CFLAGS ${HWASAN_ALIASES_DYNAMIC_CFLAGS}
+  DEFS ${HWASAN_DEFINITIONS})
+
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "")
 add_compiler_rt_object_libraries(RTHwasan_dynamic_version_script_dummy
   ARCHS ${HWASAN_SUPPORTED_ARCH}
   SOURCES ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp
   CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
   DEFS ${HWASAN_DEFINITIONS})
 
-foreach(arch ${HWASAN_SUPPORTED_ARCH})
-  add_compiler_rt_runtime(clang_rt.hwasan
+# If use_aliases is TRUE, adds the HWASan runtime built with alias support.
+# Otherwise adds the runtime without alias support.
+function(add_hwasan_runtimes arch use_aliases)
+  set(hwasan_object_lib RTHwasan)
+  set(hwasan_object_dyn_lib RTHwasan_dynamic)
+  set(hwasan_runtime clang_rt.hwasan)
+  set(hwasan_rtl_flags ${HWASAN_RTL_CFLAGS})
+  set(hwasan_dyn_flags ${HWASAN_DYNAMIC_CFLAGS})
+  if(use_aliases)
+    set(hwasan_object_lib RTHwasanAliases)
+    set(hwasan_object_dyn_lib RTHwasanAliases_dynamic)
+    set(hwasan_runtime clang_rt.hwasan_aliases)
+    set(hwasan_rtl_flags ${HWASAN_ALIASES_RTL_CFLAGS})
+    set(hwasan_dyn_flags ${HWASAN_ALIASES_DYNAMIC_CFLAGS})
+  endif()
+  add_compiler_rt_runtime(${hwasan_runtime}
     STATIC
     ARCHS ${arch}
-    OBJECT_LIBS RTHwasan
+    OBJECT_LIBS ${hwasan_object_lib}
                 RTInterception
                 RTSanitizerCommon
                 RTSanitizerCommonLibc
                 RTSanitizerCommonCoverage
                 RTSanitizerCommonSymbolizer
                 RTUbsan
-    CFLAGS ${HWASAN_RTL_CFLAGS}
+    CFLAGS ${hwasan_rtl_flags}
     PARENT_TARGET hwasan)
-  add_compiler_rt_runtime(clang_rt.hwasan_cxx
+  add_compiler_rt_runtime(${hwasan_runtime}_cxx
     STATIC
     ARCHS ${arch}
     OBJECT_LIBS RTHwasan_cxx
                 RTUbsan_cxx
-    CFLAGS ${HWASAN_RTL_CFLAGS}
+    CFLAGS ${hwasan_rtl_flags}
     PARENT_TARGET hwasan)
 
   if (UNIX)
-    add_sanitizer_rt_version_list(clang_rt.hwasan-dynamic-${arch}
-                                  LIBS clang_rt.hwasan-${arch} clang_rt.hwasan_cxx-${arch}
+    add_sanitizer_rt_version_list(${hwasan_runtime}-dynamic-${arch}
+                                  LIBS ${hwasan_runtime}-${arch} ${hwasan_runtime}_cxx-${arch}
                                   EXTRA hwasan.syms.extra)
     set(VERSION_SCRIPT_FLAG
-         -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers)
+         -Wl,--version-script,${CMAKE_CURRENT_BINARY_DIR}/${hwasan_runtime}-dynamic-${arch}.vers)
     set_property(SOURCE
       ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp
       APPEND PROPERTY
-      OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/clang_rt.hwasan-dynamic-${arch}.vers)
+      OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${hwasan_runtime}-dynamic-${arch}.vers)
   else()
     set(VERSION_SCRIPT_FLAG)
   endif()
 
 
-  add_compiler_rt_runtime(clang_rt.hwasan
+  add_compiler_rt_runtime(${hwasan_runtime}
     SHARED
     ARCHS ${arch}
     OBJECT_LIBS
-            RTHwasan_dynamic
+            ${hwasan_object_dyn_lib}
             RTInterception
             RTSanitizerCommon
             RTSanitizerCommonLibc
@@ -154,22 +186,29 @@ foreach(arch ${HWASAN_SUPPORTED_ARCH})
             # add_dependencies(clang_rt.asan-dynamic-${arch} clang_rt.asan-dynamic-${arch}-version-list)
             # generates an order-only dependency in ninja.
             RTHwasan_dynamic_version_script_dummy
-    CFLAGS ${HWASAN_DYNAMIC_CFLAGS}
+    CFLAGS ${hwasan_dyn_flags}
     LINK_FLAGS ${HWASAN_DYNAMIC_LINK_FLAGS}
               ${VERSION_SCRIPT_FLAG}
     LINK_LIBS ${HWASAN_DYNAMIC_LIBS}
     DEFS ${ASAN_DYNAMIC_DEFINITIONS}
     PARENT_TARGET hwasan)
 
   if(SANITIZER_USE_SYMBOLS)
-    add_sanitizer_rt_symbols(clang_rt.hwasan
+    add_sanitizer_rt_symbols(${hwasan_runtime}
       ARCHS ${arch}
       EXTRA hwasan.syms.extra)
-    add_sanitizer_rt_symbols(clang_rt.hwasan_cxx
+    add_sanitizer_rt_symbols(${hwasan_runtime}_cxx
       ARCHS ${arch}
       EXTRA hwasan.syms.extra)
-    add_dependencies(hwasan clang_rt.hwasan-${arch}-symbols
-                          clang_rt.hwasan_cxx-${arch}-symbols)
+    add_dependencies(hwasan ${hwasan_runtime}-${arch}-symbols
+                          ${hwasan_runtime}_cxx-${arch}-symbols)
+  endif()
+endfunction()
+
+foreach(arch ${HWASAN_SUPPORTED_ARCH})
+  add_hwasan_runtimes(${arch} FALSE)
+  if(${arch} MATCHES "x86_64")
+    add_hwasan_runtimes(${arch} TRUE)
   endif()
 endforeach()
 

compiler-rt/lib/hwasan/hwasan.h

@@ -36,7 +36,10 @@
 
 typedef u8 tag_t;
 
-#if defined(__x86_64__)
+#if defined(HWASAN_ALIASING_MODE)
+#  if !defined(__x86_64__)
+#    error Aliasing mode is only supported on x86_64
+#  endif
 // Tags are done in middle bits using userspace aliasing.
 constexpr unsigned kAddressTagShift = 39;
 constexpr unsigned kTagBits = 3;
@@ -49,12 +52,16 @@ constexpr unsigned kTagBits = 3;
 // simpler/faster shadow calculation.
 constexpr unsigned kTaggableRegionCheckShift =
     __sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U);
+#elif defined(__x86_64__)
+// Tags are done in upper bits using Intel LAM.
+constexpr unsigned kAddressTagShift = 57;
+constexpr unsigned kTagBits = 6;
 #else
 // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
 // translation and can be used to store a tag.
 constexpr unsigned kAddressTagShift = 56;
 constexpr unsigned kTagBits = 8;
-#endif  // defined(__x86_64__)
+#endif  // defined(HWASAN_ALIASING_MODE)
 
 // Mask for extracting tag bits from the lower 8 bits.
 constexpr uptr kTagMask = (1UL << kTagBits) - 1;

compiler-rt/lib/hwasan/hwasan_allocator.h

@@ -58,7 +58,7 @@ static const uptr kMaxAllowedMallocSize = 1UL << 40;  // 1T
 struct AP64 {
   static const uptr kSpaceBeg = ~0ULL;
 
-#if defined(__x86_64__)
+#if defined(HWASAN_ALIASING_MODE)
   static const uptr kSpaceSize = 1ULL << kAddressTagShift;
 #else
   static const uptr kSpaceSize = 0x2000000000ULL;
@@ -110,7 +110,7 @@ typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer;
 void GetAllocatorStats(AllocatorStatCounters s);
 
 inline bool InTaggableRegion(uptr addr) {
-#if defined(__x86_64__)
+#if defined(HWASAN_ALIASING_MODE)
   // Aliases are mapped next to shadow so that the upper bits match the shadow
   // base.
   return (addr >> kTaggableRegionCheckShift) ==

compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp

@@ -119,12 +119,12 @@ namespace __hwasan {
 void InitShadowGOT() {}
 
 uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
-#if defined(__x86_64__)
+#  if defined(HWASAN_ALIASING_MODE)
   constexpr uptr kAliasSize = 1ULL << kAddressTagShift;
   constexpr uptr kNumAliases = 1ULL << kTagBits;
   return MapDynamicShadowAndAliases(shadow_size_bytes, kAliasSize, kNumAliases,
                                     RingBufferSize());
-#endif
+#  endif
   return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment,
                           kHighMemEnd);
 }

compiler-rt/lib/hwasan/hwasan_linux.cpp

@@ -76,7 +76,7 @@ uptr kHighShadowEnd;
 uptr kHighMemStart;
 uptr kHighMemEnd;
 
-uptr kAliasRegionStart;  // Always 0 on non-x86.
+uptr kAliasRegionStart;  // Always 0 when aliases aren't used.
 
 static void PrintRange(uptr start, uptr end, const char *name) {
   Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
@@ -125,33 +125,50 @@ void InitPrctl() {
   if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0),
                        &local_errno) &&
       local_errno == EINVAL) {
-#if SANITIZER_ANDROID || defined(__x86_64__)
+#  if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
     // Some older Android kernels have the tagged pointer ABI on
     // unconditionally, and hence don't have the tagged-addr prctl while still
     // allow the ABI.
     // If targeting Android and the prctl is not around we assume this is the
     // case.
     return;
-#else
+#  else
     if (flags()->fail_without_syscall_abi) {
       Printf(
           "FATAL: "
           "HWAddressSanitizer requires a kernel with tagged address ABI.\n");
       Die();
     }
-#endif
+#  endif
   }
 
   // Turn on the tagged address ABI.
   if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
                                        PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) ||
-       !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) &&
-      flags()->fail_without_syscall_abi) {
-    Printf(
-        "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
-        "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
-        "configuration.\n");
-    Die();
+       !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) {
+#  if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+    // Try the new prctl API for Intel LAM.  The API is based on a currently
+    // unsubmitted patch to the Linux kernel (as of May 2021) and is thus
+    // subject to change.  Patch is here:
+    // https://lore.kernel.org/linux-mm/[email protected]/
+    int tag_bits = kTagBits;
+    int tag_shift = kAddressTagShift;
+    if (!internal_iserror(
+            internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE,
+                           reinterpret_cast<unsigned long>(&tag_bits),
+                           reinterpret_cast<unsigned long>(&tag_shift), 0))) {
+      CHECK_EQ(tag_bits, kTagBits);
+      CHECK_EQ(tag_shift, kAddressTagShift);
+      return;
+    }
+#  endif  // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+    if (flags()->fail_without_syscall_abi) {
+      Printf(
+          "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
+          "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
+          "configuration.\n");
+      Die();
+    }
   }
 #undef PR_SET_TAGGED_ADDR_CTRL
 #undef PR_GET_TAGGED_ADDR_CTRL
@@ -181,7 +198,7 @@ bool InitShadow() {
   // High memory starts where allocated shadow allows.
   kHighMemStart = ShadowToMem(kHighShadowStart);
 
-#if defined(__x86_64__)
+#  if defined(HWASAN_ALIASING_MODE)
   constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1);
   kAliasRegionStart =
       __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset;
@@ -191,7 +208,7 @@ bool InitShadow() {
   CHECK_EQ(
       (kAliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift,
       __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
-#endif
+#  endif
 
   // Check the sanity of the defined memory ranges (there might be gaps).
   CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
@@ -236,9 +253,11 @@ void InitThreads() {
 }
 
 bool MemIsApp(uptr p) {
-#if !defined(__x86_64__)  // Memory outside the alias range has non-zero tags.
+// Memory outside the alias range has non-zero tags.
+#  if !defined(HWASAN_ALIASING_MODE)
   CHECK(GetTagFromPointer(p) == 0);
-#endif
+#  endif
+
   return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
 }