8373720: GenShen: Count live-at-old mark using Snapshot at Beginning

Reviewed-by: ysr

Author: kdnilsen

src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp

@@ -335,7 +335,13 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() {
 
     size_t garbage = region->garbage();
     size_t live_bytes = region->get_live_data_bytes();
-    live_data += live_bytes;
+    if (!region->was_promoted_in_place()) {
+      // As currently implemented, region->get_live_data_bytes() represents bytes concurrently marked.
+      // Expansion of the region by promotion during concurrent marking is above TAMS, and is not included
+      // as live-data at [start of] old marking.
+      live_data += live_bytes;
+    }
+    // else, regions that were promoted in place had 0 old live data at mark start
 
     if (region->is_regular() || region->is_regular_pinned()) {
         // Only place regular or pinned regions with live data into the candidate set.
@@ -374,7 +380,6 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() {
     }
   }
 
-  // TODO: subtract from live_data bytes promoted during concurrent GC.
   _old_generation->set_live_bytes_at_last_mark(live_data);
 
   // Unlike young, we are more interested in efficiently packing OLD-gen than in reclaiming garbage first.  We sort by live-data.

src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp

@@ -223,7 +223,6 @@ void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion
   // We do not need to scan above TAMS because restored top equals tams
   assert(obj_addr == tams, "Expect loop to terminate when obj_addr equals tams");
 
-
   {
     ShenandoahHeapLocker locker(_heap->lock());
 
@@ -251,6 +250,7 @@ void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion
     // Transfer this region from young to old, increasing promoted_reserve if available space exceeds plab_min_size()
     _heap->free_set()->add_promoted_in_place_region_to_old_collector(region);
     region->set_affiliation(OLD_GENERATION);
+    region->set_promoted_in_place();
   }
 }
 
@@ -289,6 +289,7 @@ void ShenandoahGenerationalEvacuationTask::promote_humongous(ShenandoahHeapRegio
               r->index(), p2i(r->bottom()), p2i(r->top()));
       // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here
       r->set_affiliation(OLD_GENERATION);
+      r->set_promoted_in_place();
     }
 
     ShenandoahFreeSet* freeset = _heap->free_set();

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -262,6 +262,7 @@ class ShenandoahHeapRegion {
   HeapWord* volatile _update_watermark;
 
   uint _age;
+  bool _promoted_in_place;
   CENSUS_NOISE(uint _youth;)   // tracks epochs of retrograde ageing (rejuvenation)
 
   ShenandoahSharedFlag _recycling; // Used to indicate that the region is being recycled; see try_recycle*().
@@ -354,6 +355,15 @@ class ShenandoahHeapRegion {
 
   inline void save_top_before_promote();
   inline HeapWord* get_top_before_promote() const { return _top_before_promoted; }
+
+  inline void set_promoted_in_place() {
+    _promoted_in_place = true;
+  }
+
+  // Returns true iff this region was promoted in place subsequent to the most recent start of concurrent old marking.
+  inline bool was_promoted_in_place() {
+    return _promoted_in_place;
+  }
   inline void restore_top_before_promote();
   inline size_t garbage_before_padded_for_promote() const;
 
@@ -379,7 +389,13 @@ class ShenandoahHeapRegion {
   inline void increase_live_data_gc_words(size_t s);
 
   inline bool has_live() const;
+
+  // Represents the number of live bytes identified by most recent marking effort.  Does not include the bytes
+  // above TAMS.
   inline size_t get_live_data_bytes() const;
+
+  // Represents the number of live words identified by most recent marking effort.  Does not include the words
+  // above TAMS.
   inline size_t get_live_data_words() const;
 
   inline size_t garbage() const;

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp

@@ -152,6 +152,7 @@ inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) {
 
 inline void ShenandoahHeapRegion::clear_live_data() {
   AtomicAccess::store(&_live_data, (size_t)0);
+  _promoted_in_place = false;
 }
 
 inline size_t ShenandoahHeapRegion::get_live_data_words() const {