8377771: Race on allocating overreserved large pages on a NUMA system

Author: jsikstro

src/hotspot/os/bsd/gc/z/zNUMA_bsd.cpp

@@ -27,6 +27,7 @@
 #include "runtime/globals_extension.hpp"
 
 void ZNUMA::pd_initialize() {
+  _is_numa_system = false;
   _enabled = false;
   _count = !FLAG_IS_DEFAULT(ZFakeNUMA)
       ? ZFakeNUMA

src/hotspot/os/linux/gc/z/zNUMA_linux.cpp

@@ -36,6 +36,7 @@ static uint* z_numa_id_to_node = nullptr;
 static uint32_t* z_node_to_numa_id = nullptr;
 
 void ZNUMA::pd_initialize() {
+  _is_numa_system = os::Linux::numa_available() != -1;
   _enabled = UseNUMA;
 
   size_t configured_nodes = 0;

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp

@@ -385,9 +385,9 @@ bool ZPhysicalMemoryBacking::tmpfs_supports_transparent_huge_pages() const {
   return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0;
 }
 
-ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(zbacking_offset offset, size_t length, bool touch) const {
+ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(zbacking_offset offset, size_t length, bool force_touch) const {
   // On hugetlbfs, mapping a file segment will fail immediately, without
-  // the need to touch the mapped pages first, if there aren't enough huge
+  // the need to touch the mapped pages first, if there aren't enough large
   // pages available to back the mapping.
   void* const addr = mmap(nullptr, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, untype(offset));
   if (addr == MAP_FAILED) {
@@ -396,44 +396,34 @@ ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(zbacking_offset o
   }
 
   // Once mapped, the huge pages are only reserved. We need to touch them
-  // to associate them with the file segment. Note that we can not punch
-  // hole in file segments which only have reserved pages.
-  if (touch) {
-    char* const start = (char*)addr;
-    char* const end = start + length;
-    os::pretouch_memory(start, end, _block_size);
+  // to associate them with the file segment. This needs to be done immediately
+  // if running on a NUMA system, where shared memory may overreserve the number
+  // of large pages available.
+  const bool should_touch = force_touch || ZNUMA::is_numa_system();
+  bool touched = false;
+
+  if (should_touch) {
+    // Touch the large pages safely
+    touched = os::Linux::safe_touch_memory(addr, length, ZGranuleSize);
   }
 
-  // Unmap again. From now on, the huge pages that were mapped are allocated
-  // to this file. There's no risk of getting a SIGBUS when mapping and
-  // touching these pages again.
+  // Unmap again
   if (munmap(addr, length) == -1) {
     // Failed
     return errno;
   }
 
-  // Success
-  return 0;
-}
-
-static bool safe_touch_mapping(void* addr, size_t length, size_t page_size) {
-  char* const start = (char*)addr;
-  char* const end = start + length;
-
-  // Touching a mapping that can't be backed by memory will generate a
-  // SIGBUS. By using SafeFetch32 any SIGBUS will be safely caught and
-  // handled. On tmpfs, doing a fetch (rather than a store) is enough
-  // to cause backing pages to be allocated (there's no zero-page to
-  // worry about).
-  for (char *p = start; p < end; p += page_size) {
-    if (SafeFetch32((int*)p, -1) == -1) {
-      // Failed
-      return false;
-    }
+  if (should_touch && !touched) {
+    // Failed to touch all large pages
+    return errno;
   }
 
+  // From now on, the large pages that were mapped are allocated to this file.
+  // There's no risk of getting a SIGBUS when mapping and touching these pages
+  // again.
+
   // Success
-  return true;
+  return 0;
 }
 
 ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(zbacking_offset offset, size_t length) const {
@@ -451,7 +441,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(zbacking_offset offse
   }
 
   // Touch the mapping (safely) to make sure it's backed by memory
-  const bool backed = safe_touch_mapping(addr, length, _block_size);
+  const bool backed = os::Linux::safe_touch_memory(addr, length, _block_size);
 
   // Unmap again. If successfully touched, the backing memory will
   // be allocated to this file. There's no risk of getting a SIGBUS
@@ -461,7 +451,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(zbacking_offset offse
     return errno;
   }
 
-  // Success
+  // Success?
   return backed ? 0 : ENOMEM;
 }
 
@@ -486,7 +476,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_compat(zbacking_offset offset
   // mmap/munmap (for hugetlbfs and tmpfs with transparent huge pages) or pwrite
   // (for tmpfs without transparent huge pages and other filesystem types).
   if (ZLargePages::is_explicit()) {
-    return fallocate_compat_mmap_hugetlbfs(offset, length, false /* touch */);
+    return fallocate_compat_mmap_hugetlbfs(offset, length, false /* force_touch */);
   } else if (ZLargePages::is_transparent()) {
     return fallocate_compat_mmap_tmpfs(offset, length);
   } else {
@@ -534,14 +524,16 @@ ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole(zbacking_offset offset, size_
 }
 
 ZErrno ZPhysicalMemoryBacking::fallocate_punch_hole(zbacking_offset offset, size_t length) const {
-  if (ZLargePages::is_explicit()) {
+  // On a NUMA system we have to touch all the large pages when committing, so
+  // there is no need to touch them again here.
+  if (ZLargePages::is_explicit() && !ZNUMA::is_numa_system()) {
     // We can only punch hole in pages that have been touched. Non-touched
     // pages are only reserved, and not associated with any specific file
     // segment. We don't know which pages have been previously touched, so
     // we always touch them here to guarantee that we can punch hole.
-    const ZErrno err = fallocate_compat_mmap_hugetlbfs(offset, length, true /* touch */);
+    //
+    const ZErrno err = fallocate_compat_mmap_hugetlbfs(offset, length, true /* force_touch */);
     if (err) {
-      // Failed
       return err;
     }
   }
@@ -665,9 +657,7 @@ size_t ZPhysicalMemoryBacking::commit_default(zbacking_offset offset, size_t len
 }
 
 size_t ZPhysicalMemoryBacking::commit(zbacking_offset offset, size_t length, uint32_t numa_id) const {
-  if (ZNUMA::is_enabled() && !ZLargePages::is_explicit()) {
-    // The memory is required to be preferred at the time it is paged in. As a
-    // consequence we must prefer the memory when committing non-large pages.
+  if (ZNUMA::is_enabled()) {
     return commit_numa_preferred(offset, length, numa_id);
   }
 

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp

@@ -48,7 +48,7 @@ class ZPhysicalMemoryBacking {
   bool is_hugetlbfs() const;
   bool tmpfs_supports_transparent_huge_pages() const;
 
-  ZErrno fallocate_compat_mmap_hugetlbfs(zbacking_offset offset, size_t length, bool touch) const;
+  ZErrno fallocate_compat_mmap_hugetlbfs(zbacking_offset offset, size_t length, bool force_touch) const;
   ZErrno fallocate_compat_mmap_tmpfs(zbacking_offset offset, size_t length) const;
   ZErrno fallocate_compat_pwrite(zbacking_offset offset, size_t length) const;
   ZErrno fallocate_fill_hole_compat(zbacking_offset offset, size_t length) const;

src/hotspot/os/linux/os_linux.cpp

@@ -56,6 +56,7 @@
 #include "runtime/osInfo.hpp"
 #include "runtime/osThread.hpp"
 #include "runtime/perfMemory.hpp"
+#include "runtime/safefetch.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
 #include "runtime/threads.hpp"
@@ -3005,6 +3006,36 @@ void os::Linux::madvise_transparent_huge_pages(void* addr, size_t bytes) {
   ::madvise(addr, bytes, MADV_HUGEPAGE);
 }
 
+bool os::Linux::safe_touch_memory(void* addr, size_t bytes, size_t page_size) {
+  const int result = ::madvise(addr, bytes, MADV_POPULATE_WRITE);
+  if (result == 0) {
+    // Success
+    return true;
+  } else if (errno != EINVAL) {
+    // Failed call to madvise for some other reason than EINVAL
+    return false;
+  }
+
+  // If we failed because of EINVAL it might be because MADV_POPULATE_WRITE is
+  // not supported. We then try touching the memory using SafeFetch.
+
+  char* const start = (char*)addr;
+  char* const end = start + bytes;
+
+  // Touching a mapping that can't be backed by memory will generate a
+  // SIGBUS. By using SafeFetch32 any SIGBUS will be safely caught and
+  // handled. A fetch is enough to cause backing pages to be allocated.
+  for (char *p = start; p < end; p += page_size) {
+    if (SafeFetch32((int*)p, -1) == -1) {
+      // Failed
+      return false;
+    }
+  }
+
+  // Success
+  return true;
+}
+
 void os::pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint) {
   if (Linux::should_madvise_anonymous_thps() && alignment_hint > vm_page_size()) {
     Linux::madvise_transparent_huge_pages(addr, bytes);
@@ -3203,6 +3234,19 @@ static bool numa_syscall_check() {
   return true;
 }
 
+void os::Linux::libnuma_early_init() {
+  // This function only sets up functionality that we need regardless of
+  // whether NUMA is enabled/disabled in the JVM via UseNUMA. This is a
+  // best effort, as libnuma is likely not available on all systems.
+  void *handle = dlopen("libnuma.so.1", RTLD_LAZY);
+  if (handle != nullptr) {
+    log_info(gc, numa)("Setting up numa available");
+    set_numa_available(CAST_TO_FN_PTR(numa_available_func_t,
+                                      libnuma_dlsym(handle, "numa_available")));
+
+  }
+}
+
 bool os::Linux::libnuma_init() {
   // Requires sched_getcpu() and numa dependent syscalls support
   if ((sched_getcpu() != -1) && numa_syscall_check()) {
@@ -3216,8 +3260,6 @@ bool os::Linux::libnuma_init() {
                                        libnuma_dlsym(handle, "numa_max_node")));
       set_numa_num_configured_nodes(CAST_TO_FN_PTR(numa_num_configured_nodes_func_t,
                                                    libnuma_dlsym(handle, "numa_num_configured_nodes")));
-      set_numa_available(CAST_TO_FN_PTR(numa_available_func_t,
-                                        libnuma_dlsym(handle, "numa_available")));
       set_numa_tonode_memory(CAST_TO_FN_PTR(numa_tonode_memory_func_t,
                                             libnuma_dlsym(handle, "numa_tonode_memory")));
       set_numa_interleave_memory(CAST_TO_FN_PTR(numa_interleave_memory_func_t,
@@ -4194,9 +4236,27 @@ char* os::pd_reserve_memory_special(size_t bytes, size_t alignment, size_t page_
 
   char* const addr = reserve_memory_special_huge_tlbfs(bytes, alignment, page_size, req_addr, exec);
 
+  log_info(gc,numa)("Using ZGC: %s, numa_available: %d", UseZGC ? "true" : "false", os::Linux::numa_available());
+
   if (addr != nullptr) {
     if (UseNUMAInterleaving) {
       numa_make_global(addr, bytes);
+    } else if (UseZGC && os::Linux::numa_available() != -1) {
+      // Large pages are committed during reservation so that they are reserved for us.
+      // However, large pages may be overreserved on a NUMA system if using shared memory,
+      // which ZGC does. To make sure we can back the pages we need to fault them in
+      // immediately.
+      if (!os::Linux::safe_touch_memory(addr, bytes, page_size)) {
+        log_info(gc,numa)("Failed to touch all special memory");
+        if (::munmap(addr, bytes) != 0) {
+          ErrnoPreserver ep;
+          log_trace(os, map)("munmap failed: " RANGEFMT " errno=(%s)",
+                             RANGEFMTARGS(addr, bytes),
+                             os::strerror(ep.saved_errno()));
+        }
+        return nullptr;
+      }
+      log_info(gc,numa)("Touched %zu bytes special", bytes);
     }
   }
 
@@ -4610,6 +4670,10 @@ jint os::init_2(void) {
   init_adjust_stacksize_for_guard_pages();
 #endif
 
+  // Sets up functionality that may be needed regardless of whether the
+  // JVM should enable NUMA optimizations or not.
+  Linux::libnuma_early_init();
+
   if (UseNUMA || UseNUMAInterleaving) {
     Linux::numa_init();
   }

src/hotspot/os/linux/os_linux.hpp

@@ -126,6 +126,7 @@ class os::Linux {
 
   static void libpthread_init();
   static void sched_getcpu_init();
+  static void libnuma_early_init();
   static bool libnuma_init();
   static void* libnuma_dlsym(void* handle, const char* name);
   // libnuma v2 (libnuma_1.2) symbols
@@ -195,6 +196,8 @@ class os::Linux {
 
   static void madvise_transparent_huge_pages(void* addr, size_t bytes);
 
+  static bool safe_touch_memory(void* addr, size_t bytes, size_t page_size);
+
   // Stack repair handling
 
   // none present

src/hotspot/os/windows/gc/z/zNUMA_windows.cpp

@@ -27,6 +27,7 @@
 #include "runtime/globals_extension.hpp"
 
 void ZNUMA::pd_initialize() {
+  _is_numa_system = false;
   _enabled = false;
   _count = !FLAG_IS_DEFAULT(ZFakeNUMA)
       ? ZFakeNUMA

src/hotspot/share/gc/z/zNUMA.cpp

@@ -26,6 +26,7 @@
 #include "gc/z/zNUMA.inline.hpp"
 #include "utilities/macros.hpp"
 
+bool ZNUMA::_is_numa_system;
 bool ZNUMA::_enabled;
 uint32_t ZNUMA::_count;
 

src/hotspot/share/gc/z/zNUMA.hpp

@@ -34,6 +34,7 @@ class ZNUMA : public AllStatic {
   friend class ZTest;
 
 private:
+  static bool     _is_numa_system;
   static bool     _enabled;
   static uint32_t _count;
 
@@ -42,6 +43,7 @@ class ZNUMA : public AllStatic {
 public:
   static void initialize();
 
+  static bool is_numa_system();
   static bool is_enabled();
   static bool is_faked();
 

src/hotspot/share/gc/z/zNUMA.inline.hpp

@@ -30,6 +30,10 @@
 #include "gc/z/zGlobals.hpp"
 #include "utilities/align.hpp"
 
+inline bool ZNUMA::is_numa_system() {
+  return _is_numa_system;
+}
+
 inline bool ZNUMA::is_enabled() {
   return _enabled;
 }