Initial NUMA implementation

Author: jsikstro

src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp

@@ -117,7 +117,9 @@ bool ZPhysicalMemoryBacking::commit_inner(zoffset offset, size_t length) const {
   return true;
 }
 
-size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length) const {
+size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length, int nid) const {
+  // nid is unused as NUMA is not supported on bsd.
+
   // Try to commit the whole region
   if (commit_inner(offset, length)) {
     // Success

src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.hpp

@@ -40,7 +40,7 @@ class ZPhysicalMemoryBacking {
 
   void warn_commit_limits(size_t max_capacity) const;
 
-  size_t commit(zoffset offset, size_t length) const;
+  size_t commit(zoffset offset, size_t length, int nid) const;
   size_t uncommit(zoffset offset, size_t length) const;
 
   void map(zaddress_unsafe addr, size_t size, zoffset offset) const;

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp

@@ -628,30 +628,11 @@ bool ZPhysicalMemoryBacking::commit_inner(zoffset offset, size_t length) const {
   return true;
 }
 
-static int offset_to_node(zoffset offset) {
-  const GrowableArray<int>* mapping = os::Linux::numa_nindex_to_node();
-  const size_t nindex = (untype(offset) >> ZGranuleSizeShift) % mapping->length();
-  return mapping->at((int)nindex);
-}
-
-size_t ZPhysicalMemoryBacking::commit_numa_interleaved(zoffset offset, size_t length) const {
-  size_t committed = 0;
-
-  // Commit one granule at a time, so that each granule
-  // can be allocated from a different preferred node.
-  while (committed < length) {
-    const zoffset granule_offset = offset + committed;
-
-    // Setup NUMA policy to allocate memory from a preferred node
-    os::Linux::numa_set_preferred(offset_to_node(granule_offset));
+size_t ZPhysicalMemoryBacking::commit_numa_preferred(zoffset offset, size_t length, int nid) const {
+  // Setup NUMA policy to allocate memory from a preferred node
+  os::Linux::numa_set_preferred(nid);
 
-    if (!commit_inner(granule_offset, ZGranuleSize)) {
-      // Failed
-      break;
-    }
-
-    committed += ZGranuleSize;
-  }
+  size_t committed = commit_default(offset, length);
 
   // Restore NUMA policy
   os::Linux::numa_set_preferred(-1);
@@ -687,11 +668,11 @@ size_t ZPhysicalMemoryBacking::commit_default(zoffset offset, size_t length) con
   }
 }
 
-size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length) const {
-  if (ZNUMA::is_enabled() && !ZLargePages::is_explicit()) {
-    // To get granule-level NUMA interleaving when using non-large pages,
-    // we must explicitly interleave the memory at commit/fallocate time.
-    return commit_numa_interleaved(offset, length);
+size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length, int nid) const {
+  if (ZNUMA::is_enabled()) {
+    // We want to prefer allocating on a specific NUMA node. nid = -1 means allocate
+    // on the local node.
+    return commit_numa_preferred(offset, length, nid);
   }
 
   return commit_default(offset, length);

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp

@@ -59,7 +59,7 @@ class ZPhysicalMemoryBacking {
   ZErrno fallocate(bool punch_hole, zoffset offset, size_t length) const;
 
   bool commit_inner(zoffset offset, size_t length) const;
-  size_t commit_numa_interleaved(zoffset offset, size_t length) const;
+  size_t commit_numa_preferred(zoffset offset, size_t length, int nid) const;
   size_t commit_default(zoffset offset, size_t length) const;
 
 public:
@@ -69,7 +69,7 @@ class ZPhysicalMemoryBacking {
 
   void warn_commit_limits(size_t max_capacity) const;
 
-  size_t commit(zoffset offset, size_t length) const;
+  size_t commit(zoffset offset, size_t length, int nid) const;
   size_t uncommit(zoffset offset, size_t length) const;
 
   void map(zaddress_unsafe addr, size_t size, zoffset offset) const;

src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.cpp

@@ -223,7 +223,9 @@ void ZPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
   // Does nothing
 }
 
-size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length) {
+size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length, int nid) {
+  // nid does nothing as NUMA is not supported on bsd.
+
   log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
                       untype(offset) / M, untype(to_zoffset_end(offset, length)) / M, length / M);
 

src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.hpp

@@ -42,7 +42,7 @@ class ZPhysicalMemoryBacking {
 
   void warn_commit_limits(size_t max_capacity) const;
 
-  size_t commit(zoffset offset, size_t length);
+  size_t commit(zoffset offset, size_t length, int nid);
   size_t uncommit(zoffset offset, size_t length);
 
   void map(zaddress_unsafe addr, size_t size, zoffset offset) const;

src/hotspot/share/gc/z/vmStructs_z.hpp

@@ -85,10 +85,6 @@ typedef ZAttachedArray<ZForwarding, ZForwardingEntry> ZAttachedArrayForForwardin
   volatile_nonstatic_field(ZPage,               _seqnum,              uint32_t)                      \
   volatile_nonstatic_field(ZPage,               _top,                 zoffset_end)                   \
                                                                                                      \
-  nonstatic_field(ZPageAllocator,               _max_capacity,        const size_t)                  \
-  volatile_nonstatic_field(ZPageAllocator,      _capacity,            size_t)                        \
-  volatile_nonstatic_field(ZPageAllocator,      _used,                size_t)                        \
-                                                                                                     \
   nonstatic_field(ZPageTable,                   _map,                 ZGranuleMapForPageTable)       \
                                                                                                      \
   nonstatic_field(ZGranuleMapForPageTable,      _map,                 ZPage** const)                 \

src/hotspot/share/gc/z/zGeneration.cpp

@@ -927,7 +927,7 @@ void ZGenerationYoung::flip_promote(ZPage* from_page, ZPage* to_page) {
   _page_table->replace(from_page, to_page);
 
   // Update statistics
-  _page_allocator->promote_used(from_page->size());
+  _page_allocator->promote_used(from_page, to_page);
   increase_freed(from_page->size());
   increase_promoted(from_page->live_bytes());
 }
@@ -936,7 +936,7 @@ void ZGenerationYoung::in_place_relocate_promote(ZPage* from_page, ZPage* to_pag
   _page_table->replace(from_page, to_page);
 
   // Update statistics
-  _page_allocator->promote_used(from_page->size());
+  _page_allocator->promote_used(from_page, to_page);
 }
 
 void ZGenerationYoung::register_flip_promoted(const ZArray<ZPage*>& pages) {

src/hotspot/share/gc/z/zMappedCache.cpp

@@ -399,3 +399,7 @@ size_t ZMappedCache::remove_from_min(ZArray<ZVirtualMemory>* mappings, size_t ma
   }
   return remove_mappings(mappings, size);
 }
+
+size_t ZMappedCache::size() const {
+  return _size;
+}

src/hotspot/share/gc/z/zMappedCache.hpp

@@ -64,6 +64,7 @@ class ZMappedCache {
 public:
   ZMappedCache();
 
+
   void insert_mapping(const ZVirtualMemory& vmem);
 
   size_t remove_mappings(ZArray<ZVirtualMemory>* mappings, size_t size);
@@ -73,6 +74,8 @@ class ZMappedCache {
   size_t min() const;
   size_t reset_min();
   size_t remove_from_min(ZArray<ZVirtualMemory>* mappings, size_t max_size);
+
+  size_t size() const;
 };
 
 #endif // SHARE_GC_Z_ZMAPPEDCACHE_HPP