8377771: Race on allocating overreserved large pages on a NUMA system

Author: jsikstro

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp

@@ -385,7 +385,7 @@ bool ZPhysicalMemoryBacking::tmpfs_supports_transparent_huge_pages() const {
   return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0;
 }
 
-ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(zbacking_offset offset, size_t length, bool touch) const {
+ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(zbacking_offset offset, size_t length) const {
   // On hugetlbfs, mapping a file segment will fail immediately, without
   // the need to touch the mapped pages first, if there aren't enough huge
   // pages available to back the mapping.
@@ -398,11 +398,9 @@ ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(zbacking_offset o
   // Once mapped, the huge pages are only reserved. We need to touch them
   // to associate them with the file segment. Note that we can not punch
   // hole in file segments which only have reserved pages.
-  if (touch) {
-    char* const start = (char*)addr;
-    char* const end = start + length;
-    os::pretouch_memory(start, end, _block_size);
-  }
+
+  // Touch the mapping (safely) to make sure it's backed by memory
+  const bool backed = os::Linux::safe_fault_memory(addr, length, ZGranuleSize);
 
   // Unmap again. From now on, the huge pages that were mapped are allocated
   // to this file. There's no risk of getting a SIGBUS when mapping and
@@ -412,28 +410,8 @@ ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(zbacking_offset o
     return errno;
   }
 
-  // Success
-  return 0;
-}
-
-static bool safe_touch_mapping(void* addr, size_t length, size_t page_size) {
-  char* const start = (char*)addr;
-  char* const end = start + length;
-
-  // Touching a mapping that can't be backed by memory will generate a
-  // SIGBUS. By using SafeFetch32 any SIGBUS will be safely caught and
-  // handled. On tmpfs, doing a fetch (rather than a store) is enough
-  // to cause backing pages to be allocated (there's no zero-page to
-  // worry about).
-  for (char *p = start; p < end; p += page_size) {
-    if (SafeFetch32((int*)p, -1) == -1) {
-      // Failed
-      return false;
-    }
-  }
-
-  // Success
-  return true;
+  // Success?
+  return backed ? 0 : errno;
 }
 
 ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(zbacking_offset offset, size_t length) const {
@@ -451,7 +429,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(zbacking_offset offse
   }
 
   // Touch the mapping (safely) to make sure it's backed by memory
-  const bool backed = safe_touch_mapping(addr, length, _block_size);
+  const bool backed = os::Linux::safe_fault_memory(addr, length, _block_size);
 
   // Unmap again. If successfully touched, the backing memory will
   // be allocated to this file. There's no risk of getting a SIGBUS
@@ -461,7 +439,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(zbacking_offset offse
     return errno;
   }
 
-  // Success
+  // Success?
   return backed ? 0 : ENOMEM;
 }
 
@@ -486,7 +464,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_compat(zbacking_offset offset
   // mmap/munmap (for hugetlbfs and tmpfs with transparent huge pages) or pwrite
   // (for tmpfs without transparent huge pages and other filesystem types).
   if (ZLargePages::is_explicit()) {
-    return fallocate_compat_mmap_hugetlbfs(offset, length, false /* touch */);
+    return fallocate_compat_mmap_hugetlbfs(offset, length);
   } else if (ZLargePages::is_transparent()) {
     return fallocate_compat_mmap_tmpfs(offset, length);
   } else {
@@ -534,18 +512,6 @@ ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole(zbacking_offset offset, size_
 }
 
 ZErrno ZPhysicalMemoryBacking::fallocate_punch_hole(zbacking_offset offset, size_t length) const {
-  if (ZLargePages::is_explicit()) {
-    // We can only punch hole in pages that have been touched. Non-touched
-    // pages are only reserved, and not associated with any specific file
-    // segment. We don't know which pages have been previously touched, so
-    // we always touch them here to guarantee that we can punch hole.
-    const ZErrno err = fallocate_compat_mmap_hugetlbfs(offset, length, true /* touch */);
-    if (err) {
-      // Failed
-      return err;
-    }
-  }
-
   const int mode = FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE;
   if (ZSyscall::fallocate(_fd, mode, untype(offset), length) == -1) {
     // Failed
@@ -665,9 +631,7 @@ size_t ZPhysicalMemoryBacking::commit_default(zbacking_offset offset, size_t len
 }
 
 size_t ZPhysicalMemoryBacking::commit(zbacking_offset offset, size_t length, uint32_t numa_id) const {
-  if (ZNUMA::is_enabled() && !ZLargePages::is_explicit()) {
-    // The memory is required to be preferred at the time it is paged in. As a
-    // consequence we must prefer the memory when committing non-large pages.
+  if (ZNUMA::is_enabled()) {
     return commit_numa_preferred(offset, length, numa_id);
   }
 

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp

@@ -48,7 +48,7 @@ class ZPhysicalMemoryBacking {
   bool is_hugetlbfs() const;
   bool tmpfs_supports_transparent_huge_pages() const;
 
-  ZErrno fallocate_compat_mmap_hugetlbfs(zbacking_offset offset, size_t length, bool touch) const;
+  ZErrno fallocate_compat_mmap_hugetlbfs(zbacking_offset offset, size_t length) const;
   ZErrno fallocate_compat_mmap_tmpfs(zbacking_offset offset, size_t length) const;
   ZErrno fallocate_compat_pwrite(zbacking_offset offset, size_t length) const;
   ZErrno fallocate_fill_hole_compat(zbacking_offset offset, size_t length) const;

src/hotspot/os/linux/os_linux.cpp

@@ -56,6 +56,7 @@
 #include "runtime/osInfo.hpp"
 #include "runtime/osThread.hpp"
 #include "runtime/perfMemory.hpp"
+#include "runtime/safefetch.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
 #include "runtime/threads.hpp"
@@ -3005,6 +3006,36 @@ void os::Linux::madvise_transparent_huge_pages(void* addr, size_t bytes) {
   ::madvise(addr, bytes, MADV_HUGEPAGE);
 }
 
+bool os::Linux::safe_fault_memory(void* addr, size_t bytes, size_t page_size) {
+  const int result = ::madvise(addr, bytes, MADV_POPULATE_WRITE);
+  if (result == 0) {
+    // Success
+    return true;
+  } else if (errno != EINVAL) {
+    // Failed call to madvise for some other reason than EINVAL
+    return false;
+  }
+
+  // If we failed because of EINVAL it might be because MADV_POPULATE_WRITE is
+  // not supported. We then try faulting in the memory using SafeFetch.
+
+  char* const start = (char*)addr;
+  char* const end = start + bytes;
+
+  // Touching a mapping that can't be backed by memory will generate a
+  // SIGBUS. By using SafeFetch32 any SIGBUS will be safely caught and
+  // handled. A fetch is enough to cause backing pages to be allocated.
+  for (char *p = start; p < end; p += page_size) {
+    if (SafeFetch32((int*)p, -1) == -1) {
+      // Failed
+      return false;
+    }
+  }
+
+  // Success
+  return true;
+}
+
 void os::pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint) {
   if (Linux::should_madvise_anonymous_thps() && alignment_hint > vm_page_size()) {
     Linux::madvise_transparent_huge_pages(addr, bytes);
@@ -4197,6 +4228,21 @@ char* os::pd_reserve_memory_special(size_t bytes, size_t alignment, size_t page_
   if (addr != nullptr) {
     if (UseNUMAInterleaving) {
       numa_make_global(addr, bytes);
+    } else {
+      // Large pages are committed during reservation so that they are reserved for us.
+      // However, under special circumstances we might overreserve pages, so we must
+      // also make sure that we can back those pages immediately, not only reserve them.
+      // We do this by faulting in the large pages and associating them with the
+      // virtual memory reservation here.
+      if (!os::Linux::safe_fault_memory(addr, bytes, page_size)) {
+        if (::munmap(addr, bytes) != 0) {
+          ErrnoPreserver ep;
+          log_trace(os, map)("munmap failed: " RANGEFMT " errno=(%s)",
+                             RANGEFMTARGS(addr, bytes),
+                             os::strerror(ep.saved_errno()));
+        }
+        return nullptr;
+      }
     }
   }
 

src/hotspot/os/linux/os_linux.hpp

@@ -195,6 +195,8 @@ class os::Linux {
 
   static void madvise_transparent_huge_pages(void* addr, size_t bytes);
 
+  static bool safe_fault_memory(void* addr, size_t bytes, size_t page_size);
+
   // Stack repair handling
 
   // none present

src/hotspot/share/gc/z/zPhysicalMemoryManager.cpp

@@ -290,11 +290,6 @@ void ZPhysicalMemoryManager::map(const ZVirtualMemory& vmem, uint32_t numa_id) c
   });
 
   postcond(mapped == size);
-
-  // Setup NUMA preferred for large pages
-  if (ZNUMA::is_enabled() && ZLargePages::is_explicit()) {
-    os::numa_make_local((char*)addr, size, ZNUMA::numa_id_to_node(numa_id));
-  }
 }
 
 // Unmap virtual memory from physical memory