Implementation of a mapped cache

Author: jsikstro

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp

@@ -628,30 +628,11 @@ bool ZPhysicalMemoryBacking::commit_inner(zoffset offset, size_t length) const {
   return true;
 }
 
-static int offset_to_node(zoffset offset) {
-  const GrowableArray<int>* mapping = os::Linux::numa_nindex_to_node();
-  const size_t nindex = (untype(offset) >> ZGranuleSizeShift) % mapping->length();
-  return mapping->at((int)nindex);
-}
-
-size_t ZPhysicalMemoryBacking::commit_numa_interleaved(zoffset offset, size_t length) const {
-  size_t committed = 0;
-
-  // Commit one granule at a time, so that each granule
-  // can be allocated from a different preferred node.
-  while (committed < length) {
-    const zoffset granule_offset = offset + committed;
-
-    // Setup NUMA policy to allocate memory from a preferred node
-    os::Linux::numa_set_preferred(offset_to_node(granule_offset));
+size_t ZPhysicalMemoryBacking::commit_numa_preferred(zoffset offset, size_t length, int numa_id) const {
+  // Setup NUMA policy to allocate memory from a preferred node
+  os::Linux::numa_set_preferred(numa_id);
 
-    if (!commit_inner(granule_offset, ZGranuleSize)) {
-      // Failed
-      break;
-    }
-
-    committed += ZGranuleSize;
-  }
+  size_t committed = commit_default(offset, length);
 
   // Restore NUMA policy
   os::Linux::numa_set_preferred(-1);
@@ -687,11 +668,9 @@ size_t ZPhysicalMemoryBacking::commit_default(zoffset offset, size_t length) con
   }
 }
 
-size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length) const {
-  if (ZNUMA::is_enabled() && !ZLargePages::is_explicit()) {
-    // To get granule-level NUMA interleaving when using non-large pages,
-    // we must explicitly interleave the memory at commit/fallocate time.
-    return commit_numa_interleaved(offset, length);
+size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length, int numa_id) const {
+  if (ZNUMA::is_enabled()) {
+    return commit_numa_preferred(offset, length, numa_id);
   }
 
   return commit_default(offset, length);

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp

@@ -59,7 +59,7 @@ class ZPhysicalMemoryBacking {
   ZErrno fallocate(bool punch_hole, zoffset offset, size_t length) const;
 
   bool commit_inner(zoffset offset, size_t length) const;
-  size_t commit_numa_interleaved(zoffset offset, size_t length) const;
+  size_t commit_numa_preferred(zoffset offset, size_t length, int numa_id) const;
   size_t commit_default(zoffset offset, size_t length) const;
 
 public:
@@ -69,7 +69,7 @@ class ZPhysicalMemoryBacking {
 
   void warn_commit_limits(size_t max_capacity) const;
 
-  size_t commit(zoffset offset, size_t length) const;
+  size_t commit(zoffset offset, size_t length, int numa_id) const;
   size_t uncommit(zoffset offset, size_t length) const;
 
   void map(zaddress_unsafe addr, size_t size, zoffset offset) const;

src/hotspot/os/windows/gc/z/zVirtualMemory_windows.cpp

@@ -27,6 +27,7 @@
 #include "gc/z/zLargePages.inline.hpp"
 #include "gc/z/zMapper_windows.hpp"
 #include "gc/z/zSyscall_windows.hpp"
+#include "gc/z/zValue.inline.hpp"
 #include "gc/z/zVirtualMemory.inline.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
@@ -222,7 +223,7 @@ void ZVirtualMemoryManager::pd_initialize_before_reserve() {
 }
 
 void ZVirtualMemoryManager::pd_initialize_after_reserve() {
-  _impl->initialize_after_reserve(&_manager);
+  _impl->initialize_after_reserve(&_managers.get(0));
 }
 
 bool ZVirtualMemoryManager::pd_reserve(zaddress_unsafe addr, size_t size) {

src/hotspot/share/gc/z/vmStructs_z.hpp

@@ -83,13 +83,8 @@ typedef ZAttachedArray<ZForwarding, ZForwardingEntry> ZAttachedArrayForForwardin
                                                                                                      \
   nonstatic_field(ZPage,                        _type,                const ZPageType)               \
   volatile_nonstatic_field(ZPage,               _seqnum,              uint32_t)                      \
-  nonstatic_field(ZPage,                        _virtual,             const ZVirtualMemory)          \
   volatile_nonstatic_field(ZPage,               _top,                 zoffset_end)                   \
                                                                                                      \
-  nonstatic_field(ZPageAllocator,               _max_capacity,        const size_t)                  \
-  volatile_nonstatic_field(ZPageAllocator,      _capacity,            size_t)                        \
-  volatile_nonstatic_field(ZPageAllocator,      _used,                size_t)                        \
-                                                                                                     \
   nonstatic_field(ZPageTable,                   _map,                 ZGranuleMapForPageTable)       \
                                                                                                      \
   nonstatic_field(ZGranuleMapForPageTable,      _map,                 ZPage** const)                 \

src/hotspot/share/gc/z/zAllocationFlags.hpp

@@ -31,25 +31,22 @@
 // Allocation flags layout
 // -----------------------
 //
-//   7     2 1 0
-//  +-----+-+-+-+
-//  |00000|1|1|1|
-//  +-----+-+-+-+
-//  |     | | |
-//  |     | | * 0-0 Non-Blocking Flag (1-bit)
-//  |     | |
-//  |     | * 1-1 GC Relocation Flag (1-bit)
-//  |     |
-//  |     * 2-2 Low Address Flag (1-bit)
+//   7       1 0
+//  +-------+-+-+
+//  |0000000|1|1|
+//  +-------+-+-+
+//  |       | |
+//  |       | * 0-0 Non-Blocking Flag (1-bit)
+//  |       |
+//  |       * 1-1 GC Relocation Flag (1-bit)
 //  |
-//  * 7-3 Unused (5-bits)
+//  * 7-2 Unused (6-bits)
 //
 
 class ZAllocationFlags {
 private:
   typedef ZBitField<uint8_t, bool, 0, 1> field_non_blocking;
   typedef ZBitField<uint8_t, bool, 1, 1> field_gc_relocation;
-  typedef ZBitField<uint8_t, bool, 2, 1> field_low_address;
 
   uint8_t _flags;
 
@@ -65,21 +62,13 @@ class ZAllocationFlags {
     _flags |= field_gc_relocation::encode(true);
   }
 
-  void set_low_address() {
-    _flags |= field_low_address::encode(true);
-  }
-
   bool non_blocking() const {
     return field_non_blocking::decode(_flags);
   }
 
   bool gc_relocation() const {
     return field_gc_relocation::decode(_flags);
   }
-
-  bool low_address() const {
-    return field_low_address::decode(_flags);
-  }
 };
 
 #endif // SHARE_GC_Z_ZALLOCATIONFLAGS_HPP

src/hotspot/share/gc/z/zGeneration.cpp

@@ -195,17 +195,6 @@ void ZGeneration::select_relocation_set(ZGenerationId generation, bool promote_a
     for (ZPage* page; pt_iter.next(&page);) {
       if (!page->is_relocatable()) {
         // Not relocatable, don't register
-        // Note that the seqnum can change under our feet here as the page
-        // can be concurrently freed and recycled by a concurrent generation
-        // collection. However this property is stable across such transitions.
-        // If it was not relocatable before recycling, then it won't be
-        // relocatable after it gets recycled either, as the seqnum atomically
-        // becomes allocating for the given generation. The opposite property
-        // also holds: if the page is relocatable, then it can't have been
-        // concurrently freed; if it was re-allocated it would not be
-        // relocatable, and if it was not re-allocated we know that it was
-        // allocated earlier than mark start of the current generation
-        // collection.
         continue;
       }
 
@@ -218,15 +207,14 @@ void ZGeneration::select_relocation_set(ZGenerationId generation, bool promote_a
 
         // Reclaim empty pages in bulk
 
-        // An active iterator blocks immediate recycle and delete of pages.
-        // The intent it to allow the code that iterates over the pages to
-        // safely read the properties of the pages without them being changed
-        // by another thread. However, this function both iterates over the
-        // pages AND frees/recycles them. We "yield" the iterator, so that we
-        // can perform immediate recycling (as long as no other thread is
-        // iterating over the pages). The contract is that the pages that are
-        // about to be freed are "owned" by this thread, and no other thread
-        // will change their states.
+        // An active iterator blocks immediate deletion of pages. The intent is
+        // to allow the code that iterates over pages to safely read properties
+        // of the pages without them being freed/deleted. However, this function
+        // both iterates over the pages AND frees them. We "yield" the iterator,
+        // so that we can perform immediate deletion (as long as no other thread
+        // is iterating over the pages). The contract is that the pages that are
+        // about to be freed are "owned" by this thread, and no other thread will
+        // change their states.
         pt_iter.yield([&]() {
           free_empty_pages(&selector, 64 /* bulk */);
         });
@@ -939,7 +927,7 @@ void ZGenerationYoung::flip_promote(ZPage* from_page, ZPage* to_page) {
   _page_table->replace(from_page, to_page);
 
   // Update statistics
-  _page_allocator->promote_used(from_page->size());
+  _page_allocator->promote_used(from_page, to_page);
   increase_freed(from_page->size());
   increase_promoted(from_page->live_bytes());
 }
@@ -948,7 +936,7 @@ void ZGenerationYoung::in_place_relocate_promote(ZPage* from_page, ZPage* to_pag
   _page_table->replace(from_page, to_page);
 
   // Update statistics
-  _page_allocator->promote_used(from_page->size());
+  _page_allocator->promote_used(from_page, to_page);
 }
 
 void ZGenerationYoung::register_flip_promoted(const ZArray<ZPage*>& pages) {

src/hotspot/share/gc/z/zGranuleMap.hpp

@@ -49,6 +49,7 @@ class ZGranuleMap {
   ~ZGranuleMap();
 
   T get(zoffset offset) const;
+  T* get_addr(zoffset offset) const;
   void put(zoffset offset, T value);
   void put(zoffset offset, size_t size, T value);
 

src/hotspot/share/gc/z/zGranuleMap.inline.hpp

@@ -66,6 +66,12 @@ inline T ZGranuleMap<T>::get(zoffset offset) const {
   return at(index);
 }
 
+template <typename T>
+inline T* ZGranuleMap<T>::get_addr(zoffset offset) const {
+  const size_t index = index_for_offset(offset);
+  return _map + index;
+}
+
 template <typename T>
 inline void ZGranuleMap<T>::put(zoffset offset, T value) {
   const size_t index = index_for_offset(offset);