Bugfix for NUMA preferred explicit large pages

Author: jsikstro

src/hotspot/share/gc/z/zPageAllocator.cpp

@@ -463,7 +463,13 @@ bool ZPageAllocator::prime_cache(ZWorkers* workers, size_t size) {
       return false;
     }
 
-    map_virtual_to_physical(vmem);
+    map_virtual_to_physical(vmem, numa_id);
+
+    // Memory should have ended up on the desired NUMA id, if that's not the case, print error.
+    int actual = ZNUMA::memory_id(untype(ZOffset::address(vmem.start())));
+    if (actual != numa_id) {
+      log_debug(gc, heap)("NUMA Mismatch (priming): desired %d, actual %d", numa_id, actual);
+    }
 
     if (AlwaysPreTouch) {
       // Pre-touch memory
@@ -638,9 +644,9 @@ void ZPageAllocator::uncommit_physical(const ZMemoryRange& vmem) {
   _physical.uncommit(_physical_mappings.get_addr(vmem.start()), vmem.size());
 }
 
-void ZPageAllocator::map_virtual_to_physical(const ZMemoryRange& vmem) {
+void ZPageAllocator::map_virtual_to_physical(const ZMemoryRange& vmem, int numa_id) {
   // Map virtual memory to physical memory
-  _physical.map(vmem.start(), _physical_mappings.get_addr(vmem.start()), vmem.size());
+  _physical.map(vmem.start(), _physical_mappings.get_addr(vmem.start()), vmem.size(), numa_id);
 }
 
 void ZPageAllocator::unmap_virtual(const ZMemoryRange& vmem) {
@@ -677,9 +683,10 @@ void ZPageAllocator::remap_and_defragment_mapping(const ZMemoryRange& vmem, ZArr
 
   // The entries array may contain entires from other defragmentations as well,
   // so we only operate on the last ranges that we have just inserted
+  int numa_id = _virtual.get_numa_id(vmem);
   for (int idx = entries->length() - num_ranges; idx < entries->length(); idx++) {
     ZMemoryRange v = entries->at(idx);
-    map_virtual_to_physical(v);
+    map_virtual_to_physical(v, numa_id);
     pretouch_memory(v.start(), v.size());
   }
 }
@@ -897,7 +904,7 @@ bool ZPageAllocator::claim_virtual_memory(ZPageAllocation* allocation) {
   if (allocation->harvested() > 0) {
     ZArrayIterator<ZMemoryRange> iter(allocation->claimed_mappings());
     for (ZMemoryRange vmem; iter.next(&vmem);) {
-      map_virtual_to_physical(vmem);
+      map_virtual_to_physical(vmem, allocation->numa_id());
     }
   }
 
@@ -921,9 +928,15 @@ bool ZPageAllocator::commit_and_map_memory(ZPageAllocation* allocation, const ZM
   }
 
   sort_segments_physical(committed_vmem);
-  map_virtual_to_physical(committed_vmem);
+  map_virtual_to_physical(committed_vmem, allocation->numa_id());
   allocation->claimed_mappings()->append(committed_vmem);
 
+  // Memory should have ended up on the desired NUMA id, if that's not the case, print error.
+  int actual = ZNUMA::memory_id(untype(ZOffset::address(vmem.start())));
+  if (actual != allocation->numa_id()) {
+    log_debug(gc, heap)("NUMA Mismatch (allocation): desired %d, actual %d", allocation->numa_id(), actual);
+  }
+
   if (committed_vmem.size() != vmem.size()) {
     log_trace(gc, page)("Split memory [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT "]",
         untype(committed_vmem.start()),
@@ -935,7 +948,6 @@ bool ZPageAllocator::commit_and_map_memory(ZPageAllocation* allocation, const ZM
   return true;
 }
 
-
 ZPage* ZPageAllocator::alloc_page_inner(ZPageAllocation* allocation) {
 retry:
   // Claim physical memory by taking it from the mapped cache or by increasing

src/hotspot/share/gc/z/zPageAllocator.hpp

@@ -75,7 +75,7 @@ class ZPageAllocator {
   bool commit_physical(ZMemoryRange* vmem, int numa_id);
   void uncommit_physical(const ZMemoryRange& vmem);
 
-  void map_virtual_to_physical(const ZMemoryRange& vmem);
+  void map_virtual_to_physical(const ZMemoryRange& vmem, int numa_id);
 
   void unmap_virtual(const ZMemoryRange& vmem);
   void free_virtual(const ZMemoryRange& vmem);

src/hotspot/share/gc/z/zPhysicalMemoryManager.cpp

@@ -25,6 +25,7 @@
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zArray.inline.hpp"
 #include "gc/z/zGlobals.hpp"
+#include "gc/z/zLargePages.inline.hpp"
 #include "gc/z/zList.inline.hpp"
 #include "gc/z/zNMT.hpp"
 #include "gc/z/zNUMA.inline.hpp"
@@ -75,7 +76,7 @@ void ZPhysicalMemoryManager::try_enable_uncommit(size_t min_capacity, size_t max
   // Test if uncommit is supported by the operating system by committing
   // and then uncommitting a granule.
   const zoffset offset{};
-  if (!commit(&offset, ZGranuleSize) || !uncommit(&offset, ZGranuleSize)) {
+  if (!commit(&offset, ZGranuleSize, -1) || !uncommit(&offset, ZGranuleSize)) {
     log_info_p(gc, init)("Uncommit: Implicitly Disabled (Not supported by operating system)");
     FLAG_SET_ERGO(ZUncommit, false);
     return;
@@ -189,7 +190,7 @@ size_t ZPhysicalMemoryManager::uncommit(const zoffset* pmem, size_t size) {
 }
 
 // Map virtual memory to physical memory
-void ZPhysicalMemoryManager::map(zoffset offset, const zoffset* pmem, size_t size) const {
+void ZPhysicalMemoryManager::map(zoffset offset, const zoffset* pmem, size_t size, int numa_id) const {
   const zaddress_unsafe addr = ZOffset::address_unsafe(offset);
 
   size_t mapped = 0;
@@ -198,6 +199,11 @@ void ZPhysicalMemoryManager::map(zoffset offset, const zoffset* pmem, size_t siz
     mapped += segment_size;
   });
   postcond(mapped == size);
+
+  // Setup NUMA preferred for large pages
+  if (ZNUMA::is_enabled() && ZLargePages::is_explicit()) {
+    os::numa_make_local((char*)addr, size, numa_id);
+  }
 }
 
 // Unmap virtual memory from physical memory

src/hotspot/share/gc/z/zPhysicalMemoryManager.hpp

@@ -46,13 +46,13 @@ class ZPhysicalMemoryManager {
   void warn_commit_limits(size_t max_capacity) const;
   void try_enable_uncommit(size_t min_capacity, size_t max_capacity);
 
-  void alloc(zoffset* pmem, size_t size, int numa_id = -1);
+  void alloc(zoffset* pmem, size_t size, int numa_id);
   void free(const zoffset* pmem, size_t size, int numa_id);
 
-  size_t commit(const zoffset* pmem, size_t size, int numa_id = -1);
+  size_t commit(const zoffset* pmem, size_t size, int numa_id);
   size_t uncommit(const zoffset* pmem, size_t size);
 
-  void map(zoffset offset, const zoffset* pmem, size_t size) const;
+  void map(zoffset offset, const zoffset* pmem, size_t size, int numa_id) const;
   void unmap(zoffset offset, const zoffset* pmem, size_t size) const;
 
   size_t count_segments(const zoffset* pmem, size_t size);