8344665: Refactor PartialArrayState allocation for reuse

Reviewed-by: tschatzl, ayang, iwalulya, zgu

Author: Kim Barrett

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -86,6 +86,7 @@
 #include "gc/shared/isGCActiveMark.hpp"
 #include "gc/shared/locationPrinter.inline.hpp"
 #include "gc/shared/oopStorageParState.hpp"
+#include "gc/shared/partialArrayState.hpp"
 #include "gc/shared/referenceProcessor.inline.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
@@ -1165,6 +1166,7 @@ G1CollectedHeap::G1CollectedHeap() :
   _cm_thread(nullptr),
   _cr(nullptr),
   _task_queues(nullptr),
+  _partial_array_state_manager(nullptr),
   _ref_processor_stw(nullptr),
   _is_alive_closure_stw(this),
   _is_subject_to_discovery_stw(this),
@@ -1198,9 +1200,13 @@ G1CollectedHeap::G1CollectedHeap() :
     _task_queues->register_queue(i, q);
   }
 
+  _partial_array_state_manager = new PartialArrayStateManager(n_queues);
+
   _gc_tracer_stw->initialize();
+}
 
-  guarantee(_task_queues != nullptr, "task_queues allocation failure.");
+PartialArrayStateManager* G1CollectedHeap::partial_array_state_manager() const {
+  return _partial_array_state_manager;
 }
 
 G1RegionToSpaceMapper* G1CollectedHeap::create_aux_memory_mapper(const char* description,

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -82,6 +82,7 @@ class GCMemoryManager;
 class G1HeapRegion;
 class MemoryPool;
 class nmethod;
+class PartialArrayStateManager;
 class ReferenceProcessor;
 class STWGCTimer;
 class WorkerThreads;
@@ -807,8 +808,9 @@ class G1CollectedHeap : public CollectedHeap {
   // The concurrent refiner.
   G1ConcurrentRefine* _cr;
 
-  // The parallel task queues
-  G1ScannerTasksQueueSet *_task_queues;
+  // Reusable parallel task queues and partial array manager.
+  G1ScannerTasksQueueSet* _task_queues;
+  PartialArrayStateManager* _partial_array_state_manager;
 
   // ("Weak") Reference processing support.
   //
@@ -874,6 +876,8 @@ class G1CollectedHeap : public CollectedHeap {
   G1ScannerTasksQueueSet* task_queues() const;
   G1ScannerTasksQueue* task_queue(uint i) const;
 
+  PartialArrayStateManager* partial_array_state_manager() const;
+
   // Create a G1CollectedHeap.
   // Must call the initialize method afterwards.
   // May not return if something goes wrong.

src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp

@@ -105,6 +105,7 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
   _gc_par_phases[UpdateDerivedPointers] = new WorkerDataArray<double>("UpdateDerivedPointers", "Update Derived Pointers (ms):", max_gc_threads);
 #endif
   _gc_par_phases[EagerlyReclaimHumongousObjects] = new WorkerDataArray<double>("EagerlyReclaimHumongousObjects", "Eagerly Reclaim Humongous Objects (ms):", max_gc_threads);
+  _gc_par_phases[ResetPartialArrayStateManager] = new WorkerDataArray<double>("ResetPartialArrayStateManager", "Reset Partial Array State Manager (ms):", max_gc_threads);
   _gc_par_phases[ProcessEvacuationFailedRegions] = new WorkerDataArray<double>("ProcessEvacuationFailedRegions", "Process Evacuation Failed Regions (ms):", max_gc_threads);
 
   _gc_par_phases[ScanHR]->create_thread_work_items("Scanned Cards:", ScanHRScannedCards);
@@ -517,6 +518,7 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set(bool evacuation_failed
   debug_phase(_gc_par_phases[UpdateDerivedPointers], 1);
 #endif
   debug_phase(_gc_par_phases[EagerlyReclaimHumongousObjects], 1);
+  trace_phase(_gc_par_phases[ResetPartialArrayStateManager]);
 
   if (G1CollectedHeap::heap()->should_sample_collection_set_candidates()) {
     debug_phase(_gc_par_phases[SampleCollectionSetCandidates], 1);

src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp

@@ -87,6 +87,7 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
     UpdateDerivedPointers,
 #endif
     EagerlyReclaimHumongousObjects,
+    ResetPartialArrayStateManager,
     ProcessEvacuationFailedRegions,
     ResetMarkingState,
     NoteStartOfMark,

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -61,8 +61,7 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
                                            uint worker_id,
                                            uint num_workers,
                                            G1CollectionSet* collection_set,
-                                           G1EvacFailureRegions* evac_failure_regions,
-                                           PartialArrayStateAllocator* pas_allocator)
+                                           G1EvacFailureRegions* evac_failure_regions)
   : _g1h(g1h),
     _task_queue(g1h->task_queue(worker_id)),
     _rdc_local_qset(rdcqs),
@@ -81,7 +80,7 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
     _surviving_young_words(nullptr),
     _surviving_words_length(collection_set->young_region_length() + 1),
     _old_gen_is_full(false),
-    _partial_array_state_allocator(pas_allocator),
+    _partial_array_state_allocator(g1h->partial_array_state_manager()),
     _partial_array_stepper(num_workers, ParGCArrayScanChunk),
     _string_dedup_requests(),
     _max_num_optional_regions(collection_set->optional_region_length()),
@@ -254,7 +253,7 @@ void G1ParScanThreadState::do_partial_array(PartialArrayState* state) {
                               checked_cast<int>(step._index),
                               checked_cast<int>(step._index + _partial_array_stepper.chunk_size()));
   // Release reference to the state, now that we're done with it.
-  _partial_array_state_allocator->release(_worker_id, state);
+  _partial_array_state_allocator.release(state);
 }
 
 MAYBE_INLINE_EVACUATION
@@ -277,11 +276,10 @@ void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr,
     assert(((array_length - step._index) % _partial_array_stepper.chunk_size()) == 0,
            "invariant");
     PartialArrayState* state =
-      _partial_array_state_allocator->allocate(_worker_id,
-                                               from_obj, to_obj,
-                                               step._index,
-                                               array_length,
-                                               step._ncreate);
+      _partial_array_state_allocator.allocate(from_obj, to_obj,
+                                              step._index,
+                                              array_length,
+                                              step._ncreate);
     for (uint i = 0; i < step._ncreate; ++i) {
       push_on_queue(ScannerTask(state));
     }
@@ -601,8 +599,7 @@ G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id)
                                worker_id,
                                _num_workers,
                                _collection_set,
-                               _evac_failure_regions,
-                               &_partial_array_state_allocator);
+                               _evac_failure_regions);
   }
   return _states[worker_id];
 }
@@ -732,8 +729,7 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
     _surviving_young_words_total(NEW_C_HEAP_ARRAY(size_t, collection_set->young_region_length() + 1, mtGC)),
     _num_workers(num_workers),
     _flushed(false),
-    _evac_failure_regions(evac_failure_regions),
-    _partial_array_state_allocator(num_workers)
+    _evac_failure_regions(evac_failure_regions)
 {
   for (uint i = 0; i < num_workers; ++i) {
     _states[i] = nullptr;

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -84,7 +84,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   // Indicates whether in the last generation (old) there is no more space
   // available for allocation.
   bool _old_gen_is_full;
-  PartialArrayStateAllocator* _partial_array_state_allocator;
+  PartialArrayStateAllocator _partial_array_state_allocator;
   PartialArrayTaskStepper _partial_array_stepper;
   StringDedup::Requests _string_dedup_requests;
 
@@ -124,8 +124,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
                        uint worker_id,
                        uint num_workers,
                        G1CollectionSet* collection_set,
-                       G1EvacFailureRegions* evac_failure_regions,
-                       PartialArrayStateAllocator* partial_array_state_allocator);
+                       G1EvacFailureRegions* evac_failure_regions);
   virtual ~G1ParScanThreadState();
 
   void set_ref_discoverer(ReferenceDiscoverer* rd) { _scanner.set_ref_discoverer(rd); }
@@ -247,7 +246,6 @@ class G1ParScanThreadStateSet : public StackObj {
   uint _num_workers;
   bool _flushed;
   G1EvacFailureRegions* _evac_failure_regions;
-  PartialArrayStateAllocator _partial_array_state_allocator;
 
  public:
   G1ParScanThreadStateSet(G1CollectedHeap* g1h,

src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp

@@ -42,6 +42,7 @@
 #include "gc/g1/g1RemSet.hpp"
 #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
 #include "gc/shared/bufferNode.hpp"
+#include "gc/shared/partialArrayState.hpp"
 #include "jfr/jfrEvents.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
@@ -944,6 +945,25 @@ class G1PostEvacuateCollectionSetCleanupTask2::ResizeTLABsTask : public G1Abstra
   }
 };
 
+class G1PostEvacuateCollectionSetCleanupTask2::ResetPartialArrayStateManagerTask
+  : public G1AbstractSubTask
+{
+public:
+  ResetPartialArrayStateManagerTask()
+    : G1AbstractSubTask(G1GCPhaseTimes::ResetPartialArrayStateManager)
+  {}
+
+  double worker_cost() const override {
+    return AlmostNoWork;
+  }
+
+  void do_work(uint worker_id) override {
+    // This must be in phase2 cleanup, after phase1 has destroyed all of the
+    // associated allocators.
+    G1CollectedHeap::heap()->partial_array_state_manager()->reset();
+  }
+};
+
 G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2(G1ParScanThreadStateSet* per_thread_states,
                                                                                  G1EvacInfo* evacuation_info,
                                                                                  G1EvacFailureRegions* evac_failure_regions) :
@@ -955,6 +975,7 @@ G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2
   if (G1CollectedHeap::heap()->has_humongous_reclaim_candidates()) {
     add_serial_task(new EagerlyReclaimHumongousObjectsTask());
   }
+  add_serial_task(new ResetPartialArrayStateManagerTask());
 
   if (evac_failure_regions->has_regions_evac_failed()) {
     add_parallel_task(new ProcessEvacuationFailedRegionsTask(evac_failure_regions));

src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp

@@ -58,6 +58,7 @@ class G1PostEvacuateCollectionSetCleanupTask1 : public G1BatchedTask {
 // - Redirty Logged Cards
 // - Free Collection Set
 // - Resize TLABs
+// - Reset the reusable PartialArrayStateManager.
 class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask {
   class EagerlyReclaimHumongousObjectsTask;
 #if COMPILER2_OR_JVMCI
@@ -68,6 +69,7 @@ class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask {
   class RedirtyLoggedCardsTask;
   class FreeCollectionSetTask;
   class ResizeTLABsTask;
+  class ResetPartialArrayStateManagerTask;
 
 public:
   G1PostEvacuateCollectionSetCleanupTask2(G1ParScanThreadStateSet* per_thread_states,

src/hotspot/share/gc/parallel/psPromotionManager.cpp

@@ -51,7 +51,7 @@ PSPromotionManager::PSScannerTasksQueueSet* PSPromotionManager::_stack_array_dep
 PreservedMarksSet*             PSPromotionManager::_preserved_marks_set = nullptr;
 PSOldGen*                      PSPromotionManager::_old_gen = nullptr;
 MutableSpace*                  PSPromotionManager::_young_space = nullptr;
-PartialArrayStateAllocator*    PSPromotionManager::_partial_array_state_allocator = nullptr;
+PartialArrayStateManager*      PSPromotionManager::_partial_array_state_manager = nullptr;
 
 void PSPromotionManager::initialize() {
   ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
@@ -61,21 +61,20 @@ void PSPromotionManager::initialize() {
 
   const uint promotion_manager_num = ParallelGCThreads;
 
+  assert(_partial_array_state_manager == nullptr, "Attempt to initialize twice");
+  _partial_array_state_manager
+    = new PartialArrayStateManager(promotion_manager_num);
+
   // To prevent false sharing, we pad the PSPromotionManagers
   // and make sure that the first instance starts at a cache line.
   assert(_manager_array == nullptr, "Attempt to initialize twice");
   _manager_array = PaddedArray<PSPromotionManager, mtGC>::create_unfreeable(promotion_manager_num);
 
-  assert(_partial_array_state_allocator == nullptr, "Attempt to initialize twice");
-  _partial_array_state_allocator
-    = new PartialArrayStateAllocator(ParallelGCThreads);
-
-  _stack_array_depth = new PSScannerTasksQueueSet(ParallelGCThreads);
+  _stack_array_depth = new PSScannerTasksQueueSet(promotion_manager_num);
 
   // Create and register the PSPromotionManager(s) for the worker threads.
   for(uint i=0; i<ParallelGCThreads; i++) {
     stack_array_depth()->register_queue(i, _manager_array[i].claimed_stack_depth());
-    _manager_array[i]._partial_array_state_allocator_index = i;
   }
   // The VMThread gets its own PSPromotionManager, which is not available
   // for work stealing.
@@ -187,7 +186,8 @@ void PSPromotionManager::reset_stats() {
 
 // Most members are initialized either by initialize() or reset().
 PSPromotionManager::PSPromotionManager()
-  : _partial_array_stepper(ParallelGCThreads, ParGCArrayScanChunk)
+  : _partial_array_state_allocator(_partial_array_state_manager),
+    _partial_array_stepper(ParallelGCThreads, ParGCArrayScanChunk)
 {
   // We set the old lab's start array.
   _old_lab.set_start_array(old_gen()->start_array());
@@ -198,9 +198,6 @@ PSPromotionManager::PSPromotionManager()
     _target_stack_size = GCDrainStackTargetSize;
   }
 
-  // Initialize to a bad value; fixed by initialize().
-  _partial_array_state_allocator_index = UINT_MAX;
-
   // let's choose 1.5x the chunk size
   _min_array_size_for_chunking = (3 * ParGCArrayScanChunk / 2);
 
@@ -317,7 +314,7 @@ void PSPromotionManager::process_array_chunk(PartialArrayState* state) {
     process_array_chunk_work<oop>(state->destination(), start, end);
   }
   // Release reference to state, now that we're done with it.
-  _partial_array_state_allocator->release(_partial_array_state_allocator_index, state);
+  _partial_array_state_allocator.release(state);
 }
 
 void PSPromotionManager::push_objArray(oop old_obj, oop new_obj) {
@@ -331,11 +328,10 @@ void PSPromotionManager::push_objArray(oop old_obj, oop new_obj) {
   if (step._ncreate > 0) {
     TASKQUEUE_STATS_ONLY(++_arrays_chunked);
     PartialArrayState* state =
-      _partial_array_state_allocator->allocate(_partial_array_state_allocator_index,
-                                               old_obj, new_obj,
-                                               step._index,
-                                               array_length,
-                                               step._ncreate);
+      _partial_array_state_allocator.allocate(old_obj, new_obj,
+                                              step._index,
+                                              array_length,
+                                              step._ncreate);
     for (uint i = 0; i < step._ncreate; ++i) {
       push_depth(ScannerTask(state));
     }

src/hotspot/share/gc/parallel/psPromotionManager.hpp

@@ -28,6 +28,7 @@
 #include "gc/parallel/psPromotionLAB.hpp"
 #include "gc/shared/copyFailedInfo.hpp"
 #include "gc/shared/gcTrace.hpp"
+#include "gc/shared/partialArrayState.hpp"
 #include "gc/shared/partialArrayTaskStepper.hpp"
 #include "gc/shared/preservedMarks.hpp"
 #include "gc/shared/stringdedup/stringDedup.hpp"
@@ -50,8 +51,6 @@
 class MutableSpace;
 class PSOldGen;
 class ParCompactionManager;
-class PartialArrayState;
-class PartialArrayStateAllocator;
 
 class PSPromotionManager {
   friend class PSScavenge;
@@ -88,9 +87,9 @@ class PSPromotionManager {
 
   uint                                _target_stack_size;
 
-  static PartialArrayStateAllocator*  _partial_array_state_allocator;
+  static PartialArrayStateManager*    _partial_array_state_manager;
+  PartialArrayStateAllocator          _partial_array_state_allocator;
   PartialArrayTaskStepper             _partial_array_stepper;
-  uint                                _partial_array_state_allocator_index;
   uint                                _min_array_size_for_chunking;
 
   PreservedMarks*                     _preserved_marks;