8350898: Shenandoah: Eliminate final roots safepoint

Reviewed-by: rkennke, kdnilsen, cslucas

Author: William Kemper

src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp

@@ -36,6 +36,7 @@ class ShenandoahBarrierSet;
 class ShenandoahHeap;
 class ShenandoahMarkingContext;
 class ShenandoahReferenceProcessor;
+class SATBMarkQueueSet;
 
 //
 // ========= Super
@@ -55,6 +56,14 @@ class ShenandoahSuperClosure : public MetadataVisitingOopIterateClosure {
 // ========= Marking
 //
 
+class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure {
+private:
+  SATBMarkQueueSet& _qset;
+public:
+  inline explicit ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset);
+  inline void do_thread(Thread* thread) override;
+};
+
 class ShenandoahMarkRefsSuperClosure : public ShenandoahSuperClosure {
 private:
   ShenandoahObjToScanQueue* _queue;

src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp

@@ -59,6 +59,13 @@ void ShenandoahSuperClosure::do_nmethod(nmethod* nm) {
 //
 // ========= Marking
 //
+ShenandoahFlushSATBHandshakeClosure::ShenandoahFlushSATBHandshakeClosure(SATBMarkQueueSet& qset) :
+  HandshakeClosure("Shenandoah Flush SATB"),
+  _qset(qset) {}
+
+void ShenandoahFlushSATBHandshakeClosure::do_thread(Thread* thread) {
+  _qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread));
+}
 
 ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q,
                                                                ShenandoahReferenceProcessor* rp,

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -102,6 +102,16 @@ ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
   return _degen_point;
 }
 
+void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap* const heap) {
+  TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
+  const char* msg = conc_init_update_refs_event_message();
+  ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs_prepare);
+  EventMark em("%s", msg);
+
+  // Evacuation is complete, retire gc labs and change gc state
+  heap->concurrent_prepare_for_update_refs();
+}
+
 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 
@@ -192,8 +202,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
       return false;
     }
 
-    // Evacuation is complete, retire gc labs
-    heap->concurrent_prepare_for_update_refs();
+    entry_concurrent_update_refs_prepare(heap);
 
     // Perform update-refs phase.
     if (ShenandoahVerify || ShenandoahPacing) {
@@ -216,24 +225,14 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
     // Update references freed up collection set, kick the cleanup to reclaim the space.
     entry_cleanup_complete();
   } else {
-    // We chose not to evacuate because we found sufficient immediate garbage.
-    // However, there may still be regions to promote in place, so do that now.
-    if (has_in_place_promotions(heap)) {
-      entry_promote_in_place();
-
-      // If the promote-in-place operation was cancelled, we can have the degenerated
-      // cycle complete the operation. It will see that no evacuations are in progress,
-      // and that there are regions wanting promotion. The risk with not handling the
-      // cancellation would be failing to restore top for these regions and leaving
-      // them unable to serve allocations for the old generation.
-      if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
-        return false;
-      }
+    if (!entry_final_roots()) {
+      assert(_degen_point != _degenerated_unset, "Need to know where to start degenerated cycle");
+      return false;
     }
 
-    // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
-    // the control thread will detect it on its next iteration and run a degenerated young cycle.
-    vmop_entry_final_roots();
+    if (VerifyAfterGC) {
+      vmop_entry_verify_final_roots();
+    }
     _abbreviated = true;
   }
 
@@ -251,6 +250,52 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
   return true;
 }
 
+bool ShenandoahConcurrentGC::complete_abbreviated_cycle() {
+  shenandoah_assert_generational();
+
+  ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap();
+
+  // We chose not to evacuate because we found sufficient immediate garbage.
+  // However, there may still be regions to promote in place, so do that now.
+  if (heap->old_generation()->has_in_place_promotions()) {
+    entry_promote_in_place();
+
+    // If the promote-in-place operation was cancelled, we can have the degenerated
+    // cycle complete the operation. It will see that no evacuations are in progress,
+    // and that there are regions wanting promotion. The risk with not handling the
+    // cancellation would be failing to restore top for these regions and leaving
+    // them unable to serve allocations for the old generation.This will leave the weak
+    // roots flag set (the degenerated cycle will unset it).
+    if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
+      return false;
+    }
+  }
+
+  // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
+  // the control thread will detect it on its next iteration and run a degenerated young cycle.
+  if (!_generation->is_old()) {
+    heap->update_region_ages(_generation->complete_marking_context());
+  }
+
+  if (!heap->is_concurrent_old_mark_in_progress()) {
+    heap->concurrent_final_roots();
+  } else {
+    // Since the cycle was shortened for having enough immediate garbage, this will be
+    // the last phase before concurrent marking of old resumes. We must be sure
+    // that old mark threads don't see any pointers to garbage in the SATB queues. Even
+    // though nothing was evacuated, overwriting unreachable weak roots with null may still
+    // put pointers to regions that become trash in the SATB queues. The following will
+    // piggyback flushing the thread local SATB queues on the same handshake that propagates
+    // the gc state change.
+    ShenandoahSATBMarkQueueSet& satb_queues = ShenandoahBarrierSet::satb_mark_queue_set();
+    ShenandoahFlushSATBHandshakeClosure complete_thread_local_satb_buffers(satb_queues);
+    heap->concurrent_final_roots(&complete_thread_local_satb_buffers);
+    heap->old_generation()->concurrent_transfer_pointers_from_satb();
+  }
+  return true;
+}
+
+
 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
@@ -291,7 +336,7 @@ void ShenandoahConcurrentGC::vmop_entry_final_update_refs() {
   VMThread::execute(&op);
 }
 
-void ShenandoahConcurrentGC::vmop_entry_final_roots() {
+void ShenandoahConcurrentGC::vmop_entry_verify_final_roots() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
@@ -347,12 +392,12 @@ void ShenandoahConcurrentGC::entry_final_update_refs() {
   op_final_update_refs();
 }
 
-void ShenandoahConcurrentGC::entry_final_roots() {
-  const char* msg = final_roots_event_message();
+void ShenandoahConcurrentGC::entry_verify_final_roots() {
+  const char* msg = verify_final_roots_event_message();
   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
   EventMark em("%s", msg);
 
-  op_final_roots();
+  op_verify_final_roots();
 }
 
 void ShenandoahConcurrentGC::entry_reset() {
@@ -526,19 +571,12 @@ void ShenandoahConcurrentGC::entry_evacuate() {
   op_evacuate();
 }
 
-void ShenandoahConcurrentGC::entry_promote_in_place() {
+void ShenandoahConcurrentGC::entry_promote_in_place() const {
   shenandoah_assert_generational();
 
-  ShenandoahHeap* const heap = ShenandoahHeap::heap();
-  TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
-
-  static const char* msg = "Promote in place";
-  ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::promote_in_place);
-  EventMark em("%s", msg);
-
-  ShenandoahWorkerScope scope(heap->workers(),
-                              ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
-                              "promote in place");
+  ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::promote_in_place);
+  ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::promote_in_place);
+  EventMark em("%s", "Promote in place");
 
   ShenandoahGenerationalHeap::heap()->promote_regions_in_place(true);
 }
@@ -663,6 +701,7 @@ void ShenandoahConcurrentGC::op_init_mark() {
   }
 
   if (ShenandoahVerify) {
+    ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_mark_verify);
     heap->verifier()->verify_before_concmark();
   }
 
@@ -751,6 +790,7 @@ void ShenandoahConcurrentGC::op_final_mark() {
       }
 
       if (ShenandoahVerify) {
+        ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
         heap->verifier()->verify_before_evacuation();
       }
 
@@ -767,6 +807,7 @@ void ShenandoahConcurrentGC::op_final_mark() {
       }
     } else {
       if (ShenandoahVerify) {
+        ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_mark_verify);
         if (has_in_place_promotions(heap)) {
           heap->verifier()->verify_after_concmark_with_promotions();
         } else {
@@ -1088,6 +1129,7 @@ void ShenandoahConcurrentGC::op_evacuate() {
 void ShenandoahConcurrentGC::op_init_update_refs() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   if (ShenandoahVerify) {
+    ShenandoahTimingsTracker v(ShenandoahPhaseTimings::init_update_refs_verify);
     heap->verifier()->verify_before_update_refs();
   }
   if (ShenandoahPacing) {
@@ -1175,6 +1217,7 @@ void ShenandoahConcurrentGC::op_final_update_refs() {
   }
 
   if (ShenandoahVerify) {
+    ShenandoahTimingsTracker v(ShenandoahPhaseTimings::final_update_refs_verify);
     heap->verifier()->verify_after_update_refs();
   }
 
@@ -1190,33 +1233,32 @@ void ShenandoahConcurrentGC::op_final_update_refs() {
   }
 }
 
-void ShenandoahConcurrentGC::op_final_roots() {
+bool ShenandoahConcurrentGC::entry_final_roots() {
+  ShenandoahHeap* const heap = ShenandoahHeap::heap();
+  TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 
-  ShenandoahHeap *heap = ShenandoahHeap::heap();
-  heap->set_concurrent_weak_root_in_progress(false);
-  heap->set_evacuation_in_progress(false);
 
-  if (heap->mode()->is_generational()) {
-    // If the cycle was shortened for having enough immediate garbage, this could be
-    // the last GC safepoint before concurrent marking of old resumes. We must be sure
-    // that old mark threads don't see any pointers to garbage in the SATB buffers.
-    if (heap->is_concurrent_old_mark_in_progress()) {
-      heap->old_generation()->transfer_pointers_from_satb();
-    }
+  const char* msg = conc_final_roots_event_message();
+  ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_final_roots);
+  EventMark em("%s", msg);
+  ShenandoahWorkerScope scope(heap->workers(),
+                              ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
+                              msg);
 
-    if (!_generation->is_old()) {
-      ShenandoahGenerationalHeap::heap()->update_region_ages(_generation->complete_marking_context());
+  if (!heap->mode()->is_generational()) {
+    heap->concurrent_final_roots();
+  } else {
+    if (!complete_abbreviated_cycle()) {
+      return false;
     }
   }
+  return true;
+}
 
+void ShenandoahConcurrentGC::op_verify_final_roots() {
   if (VerifyAfterGC) {
     Universe::verify();
   }
-
-  {
-    ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_propagate_gc_state);
-    heap->propagate_gc_state_to_all_threads();
-  }
 }
 
 void ShenandoahConcurrentGC::op_cleanup_complete() {
@@ -1301,11 +1343,19 @@ const char* ShenandoahConcurrentGC::conc_reset_after_collect_event_message() con
   }
 }
 
-const char* ShenandoahConcurrentGC::final_roots_event_message() const {
+const char* ShenandoahConcurrentGC::verify_final_roots_event_message() const {
+  if (ShenandoahHeap::heap()->unload_classes()) {
+    SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final Roots", " (unload classes)");
+  } else {
+    SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Verify Final Roots", "");
+  }
+}
+
+const char* ShenandoahConcurrentGC::conc_final_roots_event_message() const {
   if (ShenandoahHeap::heap()->unload_classes()) {
-    SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", " (unload classes)");
+    SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", " (unload classes)");
   } else {
-    SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", "");
+    SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Final Roots", "");
   }
 }
 
@@ -1332,3 +1382,11 @@ const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", "");
   }
 }
+
+const char* ShenandoahConcurrentGC::conc_init_update_refs_event_message() const {
+  if (ShenandoahHeap::heap()->unload_classes()) {
+    SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", " (unload classes)");
+  } else {
+    SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent Init Update Refs", "");
+  }
+}