Consistency in initialization logic

Author: jsikstro

src/hotspot/share/gc/z/zMemory.cpp

@@ -227,14 +227,17 @@ void ZMemoryManager::register_callbacks(const Callbacks& callbacks) {
   _callbacks = callbacks;
 }
 
-size_t ZMemoryManager::range_size() const {
+ZMemoryRange ZMemoryManager::total_range() const {
   ZLocker<ZLock> locker(&_lock);
 
   if (_freelist.is_empty()) {
-    return 0;
+    return ZMemoryRange();
   }
 
-  return _freelist.last()->end() - _freelist.first()->start();
+  const zoffset start = _freelist.first()->start();
+  const size_t size = _freelist.last()->end() - start;
+
+  return ZMemoryRange(start, size);
 }
 
 zoffset ZMemoryManager::peek_low_address() const {
@@ -282,24 +285,24 @@ ZMemoryRange ZMemoryManager::alloc_high_address(size_t size) {
   return ZMemoryRange();
 }
 
-void ZMemoryManager::transfer_high_address(ZMemoryManager& other, size_t size) {
+void ZMemoryManager::transfer_low_address(ZMemoryManager& other, size_t size) {
   assert(other._freelist.is_empty(), "Should only be used for initializatiion");
 
   ZLocker<ZLock> locker(&_lock);
   size_t to_move = size;
 
-  ZListReverseIterator<ZMemory> iter(&_freelist);
+  ZListIterator<ZMemory> iter(&_freelist);
   for (ZMemory* area; iter.next(&area);) {
     if (area->size() <= to_move) {
       // Smaller than or equal to requested, remove from this freelist and
       // insert in other's freelist
       to_move -= area->size();
       _freelist.remove(area);
-      other._freelist.insert_first(area);
+      other._freelist.insert_last(area);
     } else {
       // Larger than requested, shrink area
       const zoffset_end end = area->end();
-      shrink_from_back(area, to_move);
+      shrink_from_front(area, to_move);
       other.free(zoffset(end - to_move), to_move);
       to_move = 0;
     }

src/hotspot/share/gc/z/zMemory.hpp

@@ -103,14 +103,14 @@ class ZMemoryManager {
 
   void register_callbacks(const Callbacks& callbacks);
 
-  size_t range_size() const;
+  ZMemoryRange total_range() const;
 
   zoffset peek_low_address() const;
   ZMemoryRange alloc_low_address(size_t size);
   ZMemoryRange alloc_low_address_at_most(size_t size);
   ZMemoryRange alloc_high_address(size_t size);
 
-  void transfer_high_address(ZMemoryManager& other, size_t size);
+  void transfer_low_address(ZMemoryManager& other, size_t size);
   int shuffle_memory_low_addresses(zoffset start, size_t size, ZArray<ZMemoryRange>* out);
   void shuffle_memory_low_addresses_contiguous(size_t size, ZArray<ZMemoryRange>* out);
 

src/hotspot/share/gc/z/zNUMA.hpp

@@ -42,6 +42,11 @@ class ZNUMA : public AllStatic {
 
   static uint32_t memory_id(uintptr_t addr);
 
+  static size_t calculate_share(uint32_t numa_id, size_t total);
+
+  template <typename Function>
+  static void divide_resource(size_t resource, Function function);
+
   static const char* to_string();
 };
 

src/hotspot/share/gc/z/zNUMA.inline.hpp

@@ -26,8 +26,30 @@
 
 #include "gc/z/zNUMA.hpp"
 
+#include "gc/z/zGlobals.hpp"
+#include "utilities/align.hpp"
+
 inline bool ZNUMA::is_enabled() {
   return _enabled;
 }
 
+inline size_t ZNUMA::calculate_share(uint32_t numa_id, size_t total) {
+  const uint32_t num_nodes = count();
+  const size_t base_share = align_down(total / num_nodes, ZGranuleSize);
+
+  const size_t extra_share_nodes = (total - base_share * num_nodes) / ZGranuleSize;
+  if (numa_id < extra_share_nodes) {
+    return base_share + ZGranuleSize;
+  }
+
+  return base_share;
+}
+
+template <typename Function>
+inline void ZNUMA::divide_resource(size_t total, Function function) {
+  for (uint32_t numa_id = 0; numa_id < count(); numa_id++) {
+    function(numa_id, calculate_share(numa_id, total));
+  }
+}
+
 #endif // SHARE_GC_Z_ZNUMA_INLINE_HPP

src/hotspot/share/gc/z/zPageAllocator.cpp

@@ -33,7 +33,7 @@
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zMappedCache.hpp"
 #include "gc/z/zMemory.inline.hpp"
-#include "gc/z/zNUMA.hpp"
+#include "gc/z/zNUMA.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPageAge.hpp"
 #include "gc/z/zPageAllocator.inline.hpp"
@@ -361,18 +361,9 @@ ZPageAllocator::ZPageAllocator(size_t min_capacity,
     return;
   }
 
-  const int numa_nodes = ZNUMA::count();
-  const size_t capacity_per_state = align_up(max_capacity / numa_nodes, ZGranuleSize);
-  size_t capacity_left = max_capacity;
-
-  for (int numa_id = 0; numa_id < numa_nodes; numa_id++) {
-    const size_t capacity = MIN2(capacity_per_state, capacity_left);
-    capacity_left -= capacity;
-
-    ZCacheState& state = _states.get(numa_id);
-    state.initialize(capacity);
-    _physical.install_capacity(numa_id, zoffset(capacity_per_state * numa_id), capacity);
-  }
+  ZNUMA::divide_resource(max_capacity, [&](uint32_t numa_id, size_t capacity) {
+    _states.get(numa_id).initialize(capacity);
+  });
 
   log_info_p(gc, init)("Min Capacity: %zuM", min_capacity / M);
   log_info_p(gc, init)("Initial Capacity: %zuM", initial_capacity / M);
@@ -438,48 +429,51 @@ class ZPreTouchTask : public ZTask {
   }
 };
 
-bool ZPageAllocator::prime_cache(ZWorkers* workers, size_t size) {
-  const int numa_nodes = ZNUMA::count();
-  const size_t size_per_state = align_up(size / numa_nodes, ZGranuleSize);
-  size_t size_left = size;
-
-  for (int numa_id = 0; numa_id < numa_nodes; numa_id++) {
-    const size_t to_prime = MIN2(size_left, size_per_state);
-    size_left -= to_prime;
-
-    if (to_prime == 0) {
-      // Nothing more to prime, exit
-      return true;
-    }
+bool ZPageAllocator::prime_state_cache(ZWorkers* workers, int numa_id, size_t to_prime) {
+  if (to_prime == 0) {
+    return true;
+  }
 
-    ZCacheState& state = _states.get(numa_id);
-    ZMemoryRange vmem = _virtual.alloc(to_prime, numa_id, true /* force_low_address */);
+  ZMemoryRange vmem = _virtual.alloc(to_prime, numa_id, true /* force_low_address */);
+  ZCacheState& state = _states.get(numa_id);
 
-    // Increase capacity, allocate and commit physical memory
-    state.increase_capacity(to_prime);
-    _physical.alloc(_physical_mappings.get_addr(vmem.start()), to_prime, numa_id);
-    if (!commit_physical(&vmem, numa_id)) {
-      // This is a failure state. We do not cleanup the maybe partially committed memory.
-      return false;
-    }
+  // Increase capacity, allocate and commit physical memory
+  state.increase_capacity(to_prime);
+  _physical.alloc(_physical_mappings.get_addr(vmem.start()), to_prime, numa_id);
+  if (!commit_physical(&vmem, numa_id)) {
+    // This is a failure state. We do not cleanup the maybe partially committed memory.
+    return false;
+  }
 
-    map_virtual_to_physical(vmem, numa_id);
+  map_virtual_to_physical(vmem, numa_id);
 
+  if (ZNUMA::is_enabled()) {
     // Memory should have ended up on the desired NUMA id, if that's not the case, print error.
     int actual = ZNUMA::memory_id(untype(ZOffset::address(vmem.start())));
     if (actual != numa_id) {
       log_debug(gc, heap)("NUMA Mismatch (priming): desired %d, actual %d", numa_id, actual);
     }
+  }
 
-    if (AlwaysPreTouch) {
-      // Pre-touch memory
-      ZPreTouchTask task(vmem.start(), vmem.end());
-      workers->run_all(&task);
-    }
+  if (AlwaysPreTouch) {
+    // Pre-touch memory
+    ZPreTouchTask task(vmem.start(), vmem.end());
+    workers->run_all(&task);
+  }
 
-    // We don't have to take a lock here as no other threads will access the
-    // mapped cache until we're finished.
-    state._cache.insert_mapping(vmem);
+  // We don't have to take a lock here as no other threads will access the
+  // mapped cache until we're finished.
+  state._cache.insert_mapping(vmem);
+
+  return true;
+}
+
+bool ZPageAllocator::prime_cache(ZWorkers* workers, size_t size) {
+  for (uint32_t numa_id = 0; numa_id < ZNUMA::count(); numa_id++) {
+    const size_t to_prime = ZNUMA::calculate_share(numa_id, size);
+    if (!prime_state_cache(workers, numa_id, to_prime)) {
+      return false;
+    }
   }
 
   return true;

src/hotspot/share/gc/z/zPageAllocator.hpp

@@ -108,6 +108,8 @@ class ZPageAllocator {
   void notify_out_of_memory();
   void restart_gc() const;
 
+  bool prime_state_cache(ZWorkers* workers, int numa_id, size_t size);
+
 public:
   ZPageAllocator(size_t min_capacity,
                  size_t initial_capacity,

src/hotspot/share/gc/z/zPhysicalMemoryManager.cpp

@@ -42,7 +42,15 @@
 #include "utilities/powerOfTwo.hpp"
 
 ZPhysicalMemoryManager::ZPhysicalMemoryManager(size_t max_capacity)
-  : _backing(max_capacity) {}
+  : _backing(max_capacity) {
+
+  // Install capacity into manager(s)
+  size_t installed_capacity = 0;
+  ZNUMA::divide_resource(max_capacity, [&](uint32_t numa_id, size_t capacity) {
+    _managers.get(numa_id).free(zoffset(installed_capacity), capacity);
+    installed_capacity += capacity;
+  });
+}
 
 bool ZPhysicalMemoryManager::is_initialized() const {
   return _backing.is_initialized();

src/hotspot/share/gc/z/zVirtualMemoryManager.cpp

@@ -35,7 +35,9 @@
 #include "utilities/debug.hpp"
 
 ZVirtualMemoryManager::ZVirtualMemoryManager(size_t max_capacity)
-  : _managers(),
+  : _reserved_memory(),
+    _managers(),
+    _vmem_ranges(),
     _initialized(false) {
 
   assert(max_capacity <= ZAddressOffsetMax, "Too large max_capacity");
@@ -44,14 +46,26 @@ ZVirtualMemoryManager::ZVirtualMemoryManager(size_t max_capacity)
   pd_initialize_before_reserve();
 
   // Reserve address space
-  if (!reserve(max_capacity)) {
+  const size_t reserved_total = reserve(max_capacity);
+  if (reserved_total < max_capacity) {
     ZInitialize::error_d("Failed to reserve enough address space for Java heap");
     return;
   }
 
   // Initialize platform specific parts after reserving address space
   pd_initialize_after_reserve();
 
+  // Install reserved memory into manager(s)
+  ZNUMA::divide_resource(reserved_total, [&](int numa_id, size_t reserved) {
+    ZMemoryManager& manager = _managers.get(numa_id);
+
+    // Transfer reserved memory
+    _reserved_memory.transfer_low_address(manager, reserved);
+
+    // Store the range for the manager
+    _vmem_ranges.set(manager.total_range(), numa_id);
+  });
+
   // Successfully initialized
   _initialized = true;
 }
@@ -151,9 +165,7 @@ bool ZVirtualMemoryManager::reserve_contiguous(zoffset start, size_t size) {
   // Register address views with native memory tracker
   ZNMT::reserve(addr, size);
 
-  // We put all the reserved memory inside the first manager and potentially
-  // divide it when we've reserved all of the memory the user has requested.
-  _managers.get(0).free(start, size);
+  _reserved_memory.free(start, size);
 
   return true;
 }
@@ -174,7 +186,7 @@ bool ZVirtualMemoryManager::reserve_contiguous(size_t size) {
   return false;
 }
 
-bool ZVirtualMemoryManager::reserve(size_t max_capacity) {
+size_t ZVirtualMemoryManager::reserve(size_t max_capacity) {
   const size_t limit = MIN2(ZAddressOffsetMax, ZAddressSpaceLimit::heap());
   const size_t size = MIN2(max_capacity * ZVirtualToPhysicalRatio, limit);
   bool contiguous_reservation = false;
@@ -197,39 +209,14 @@ bool ZVirtualMemoryManager::reserve(size_t max_capacity) {
   };
 
   const size_t reserved = do_reserve();
-  initialize_managers(reserved);
 
   log_info_p(gc, init)("Address Space Type: %s/%s/%s",
                        (contiguous_reservation ? "Contiguous" : "Discontiguous"),
                        (limit == ZAddressOffsetMax ? "Unrestricted" : "Restricted"),
                        (reserved == size ? "Complete" : "Degraded"));
   log_info_p(gc, init)("Address Space Size: %zuM", reserved / M);
 
-  return reserved >= max_capacity;
-}
-
-void ZVirtualMemoryManager::initialize_managers(size_t size) {
-  // All reserved memory is initially stored in the manager with id 0. We need
-  // to divide it equally among all the managers.
-  ZMemoryManager& initial_manager = _managers.get(0);
-  const int numa_nodes = ZNUMA::count();
-  const size_t local_reservation = align_up(size / numa_nodes, ZGranuleSize);
-  size_t reservation_left = size;
-
-  for (int numa_id = numa_nodes - 1; numa_id >= 0; numa_id--) {
-    const size_t reservation = MIN2(local_reservation, reservation_left);
-    reservation_left -= reservation;
-
-    ZMemoryManager& manager = _managers.get(numa_id);
-    if (numa_id != 0) {
-      initial_manager.transfer_high_address(manager, reservation);
-    }
-
-    // Update the range
-    const zoffset range_start = manager.peek_low_address();
-    const size_t range_size = manager.range_size();
-    _vmem_ranges.set(ZMemoryRange(range_start, range_size), numa_id);
-  }
+  return reserved;
 }
 
 bool ZVirtualMemoryManager::is_initialized() const {

src/hotspot/share/gc/z/zVirtualMemoryManager.hpp

@@ -35,8 +35,9 @@ class ZVirtualMemoryManager {
 private:
   static size_t calculate_min_range(size_t size);
 
+  ZMemoryManager           _reserved_memory;
   ZPerNUMA<ZMemoryManager> _managers;
-  ZPerNUMA<ZMemoryRange> _vmem_ranges;
+  ZPerNUMA<ZMemoryRange>   _vmem_ranges;
   bool                     _initialized;
 
   // Platform specific implementation
@@ -49,10 +50,9 @@ class ZVirtualMemoryManager {
   bool reserve_contiguous(size_t size);
   size_t reserve_discontiguous(zoffset start, size_t size, size_t min_range);
   size_t reserve_discontiguous(size_t size);
-  bool reserve(size_t max_capacity);
+  size_t reserve(size_t max_capacity);
 
   void set_vmem_range_for_manager(int numa_id);
-  void initialize_managers(size_t size);
 
   DEBUG_ONLY(size_t force_reserve_discontiguous(size_t size);)