8337511: Implement JEP 404: Generational Shenandoah (Experimental)

Co-authored-by: Kelvin Nilsen <[email protected]>
Co-authored-by: Y. Srinivas Ramakrishna <[email protected]>
Co-authored-by: Bernd Mathiske <[email protected]>
Co-authored-by: Martin Doerr <[email protected]>
Co-authored-by: Fei Yang <[email protected]>
Reviewed-by: rkennke, shade, phh

Author: 6 people

src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -86,6 +87,10 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI
       LIR_Opr result = gen->new_register(T_INT);
 
       __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
+
+      if (ShenandoahCardBarrier) {
+        post_barrier(access, access.resolved_addr(), new_value.result());
+      }
       return result;
     }
   }
@@ -113,6 +118,9 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
       pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
                   result /* pre_val */);
     }
+    if (ShenandoahCardBarrier) {
+      post_barrier(access, access.resolved_addr(), result);
+    }
   }
 
   return result;

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,6 +32,7 @@
 #include "gc/shenandoah/shenandoahRuntime.hpp"
 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
 #include "interpreter/interpreter.hpp"
 #include "interpreter/interp_masm.hpp"
 #include "runtime/javaThread.hpp"
@@ -77,6 +79,13 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
   }
 }
 
+void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                                       Register start, Register count, Register tmp, RegSet saved_regs) {
+  if (ShenandoahCardBarrier && is_oop) {
+    gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs);
+  }
+}
+
 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
                                                                  Register obj,
                                                                  Register pre_val,
@@ -362,6 +371,26 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
   }
 }
 
+void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
+  assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+  __ lsr(obj, obj, CardTable::card_shift());
+
+  assert(CardTable::dirty_card_val() == 0, "must be");
+
+  __ load_byte_map_base(rscratch1);
+
+  if (UseCondCardMark) {
+    Label L_already_dirty;
+    __ ldrb(rscratch2, Address(obj, rscratch1));
+    __ cbz(rscratch2, L_already_dirty);
+    __ strb(zr, Address(obj, rscratch1));
+    __ bind(L_already_dirty);
+  } else {
+    __ strb(zr, Address(obj, rscratch1));
+  }
+}
+
 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                              Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
   bool on_oop = is_reference_type(type);
@@ -387,18 +416,13 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet
                                val != noreg /* tosca_live */,
                                false /* expand_call */);
 
-  if (val == noreg) {
-    BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), noreg, noreg, noreg, noreg);
-  } else {
-    // Barrier needs uncompressed oop for region cross check.
-    Register new_val = val;
-    if (UseCompressedOops) {
-      new_val = rscratch2;
-      __ mov(new_val, val);
-    }
-    BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
-  }
+  BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
 
+  bool in_heap = (decorators & IN_HEAP) != 0;
+  bool needs_post_barrier = (val != noreg) && in_heap && ShenandoahCardBarrier;
+  if (needs_post_barrier) {
+    store_check(masm, tmp3);
+  }
 }
 
 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
@@ -581,6 +605,35 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
   }
 }
 
+void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+                                                                     Register start, Register count, Register scratch, RegSet saved_regs) {
+  assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+  Label L_loop, L_done;
+  const Register end = count;
+
+  // Zero count? Nothing to do.
+  __ cbz(count, L_done);
+
+  // end = start + count << LogBytesPerHeapOop
+  // last element address to make inclusive
+  __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
+  __ sub(end, end, BytesPerHeapOop);
+  __ lsr(start, start, CardTable::card_shift());
+  __ lsr(end, end, CardTable::card_shift());
+
+  // number of bytes to copy
+  __ sub(count, end, start);
+
+  __ load_byte_map_base(scratch);
+  __ add(start, start, scratch);
+  __ bind(L_loop);
+  __ strb(zr, Address(start, count));
+  __ subs(count, count, 1);
+  __ br(Assembler::GE, L_loop);
+  __ bind(L_done);
+}
+
 #undef __
 
 #ifdef COMPILER1

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -55,10 +56,16 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
                                     bool tosca_live,
                                     bool expand_call);
 
+  void store_check(MacroAssembler* masm, Register obj);
+
   void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
   void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
   void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators);
 
+  void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+                                        Register start, Register count,
+                                        Register scratch, RegSet saved_regs);
+
 public:
   virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
 
@@ -71,6 +78,8 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
 
   virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
                                   Register src, Register dst, Register count, RegSet saved_regs);
+  virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                  Register start, Register count, Register tmp, RegSet saved_regs);
   virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                        Register dst, Address src, Register tmp1, Register tmp2);
   virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,

src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp

@@ -102,6 +102,10 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI
 
       __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
 
+      if (ShenandoahCardBarrier) {
+        post_barrier(access, access.resolved_addr(), new_value.result());
+      }
+
       return result;
     }
   }
@@ -132,6 +136,10 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRIt
     if (ShenandoahSATBBarrier) {
       pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, result);
     }
+
+    if (ShenandoahCardBarrier) {
+      post_barrier(access, access.resolved_addr(), result);
+    }
   }
 
   return result;

src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp

@@ -36,6 +36,7 @@
 #include "gc/shenandoah/shenandoahRuntime.hpp"
 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/mode/shenandoahMode.hpp"
 #include "interpreter/interpreter.hpp"
 #include "macroAssembler_ppc.hpp"
 #include "runtime/javaThread.hpp"
@@ -76,8 +77,6 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm,
 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
                                                        Register src, Register dst, Register count,
                                                        Register preserve1, Register preserve2) {
-  __ block_comment("arraycopy_prologue (shenandoahgc) {");
-
   Register R11_tmp = R11_scratch1;
 
   assert_different_registers(src, dst, count, R11_tmp, noreg);
@@ -100,6 +99,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, Dec
     return;
   }
 
+  __ block_comment("arraycopy_prologue (shenandoahgc) {");
   Label skip_prologue;
 
   // Fast path: Array is of length zero.
@@ -173,6 +173,16 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, Dec
   __ block_comment("} arraycopy_prologue (shenandoahgc)");
 }
 
+void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                                       Register dst, Register count,
+                                                       Register preserve) {
+  if (ShenandoahCardBarrier && is_reference_type(type)) {
+    __ block_comment("arraycopy_epilogue (shenandoahgc) {");
+    gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve);
+    __ block_comment("} arraycopy_epilogue (shenandoahgc)");
+  }
+}
+
 // The to-be-enqueued value can either be determined
 // - dynamically by passing the reference's address information (load mode) or
 // - statically by passing a register the value is stored in (preloaded mode)
@@ -576,6 +586,25 @@ void ShenandoahBarrierSetAssembler::load_at(
   }
 }
 
+void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) {
+  assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+  ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
+  CardTable* ct = ctbs->card_table();
+  assert_different_registers(base, tmp, R0);
+
+  if (ind_or_offs.is_constant()) {
+    __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp);
+  } else {
+    __ add(base, ind_or_offs.as_register(), base);
+  }
+
+  __ load_const_optimized(tmp, (address)ct->byte_map_base(), R0);
+  __ srdi(base, base, CardTable::card_shift());
+  __ li(R0, CardTable::dirty_card_val());
+  __ stbx(R0, tmp, base);
+}
+
 // base:        Base register of the reference's address.
 // ind_or_offs: Index or offset of the reference's address.
 // val:         To-be-stored value/reference's new value.
@@ -594,6 +623,11 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet
                                 val,
                                 tmp1, tmp2, tmp3,
                                 preservation_level);
+
+  // No need for post barrier if storing NULL
+  if (ShenandoahCardBarrier && is_reference_type(type) && val != noreg) {
+    store_check(masm, base, ind_or_offs, tmp1);
+  }
 }
 
 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm,
@@ -743,6 +777,40 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler *masm, Register b
   __ block_comment("} cmpxchg_oop (shenandoahgc)");
 }
 
+void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+                                                                     Register addr, Register count, Register preserve) {
+  assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+  ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+  CardTable* ct = bs->card_table();
+  assert_different_registers(addr, count, R0);
+
+  Label L_skip_loop, L_store_loop;
+
+  __ sldi_(count, count, LogBytesPerHeapOop);
+
+  // Zero length? Skip.
+  __ beq(CCR0, L_skip_loop);
+
+  __ addi(count, count, -BytesPerHeapOop);
+  __ add(count, addr, count);
+  // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.)
+  __ srdi(addr, addr, CardTable::card_shift());
+  __ srdi(count, count, CardTable::card_shift());
+  __ subf(count, addr, count);
+  __ add_const_optimized(addr, addr, (address)ct->byte_map_base(), R0);
+  __ addi(count, count, 1);
+  __ li(R0, 0);
+  __ mtctr(count);
+
+  // Byte store loop
+  __ bind(L_store_loop);
+  __ stb(R0, 0, addr);
+  __ addi(addr, addr, 1);
+  __ bdnz(L_store_loop);
+  __ bind(L_skip_loop);
+}
+
 #undef __
 
 #ifdef COMPILER1

src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp

@@ -51,6 +51,10 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
                                Register tmp1, Register tmp2,
                                MacroAssembler::PreservationLevel preservation_level);
 
+  void store_check(MacroAssembler* masm,
+                   Register base, RegisterOrConstant ind_or_offs,
+                   Register tmp);
+
   void load_reference_barrier_impl(MacroAssembler* masm, DecoratorSet decorators,
                                    Register base, RegisterOrConstant ind_or_offs,
                                    Register dst,
@@ -60,6 +64,10 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
   /* ==== Helper methods for barrier implementations ==== */
   void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp);
 
+  void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+                                        Register addr, Register count,
+                                        Register preserve);
+
 public:
   virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
 
@@ -95,7 +103,11 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
 
   /* ==== Access api ==== */
   virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
-                          Register src, Register dst, Register count, Register preserve1, Register preserve2);
+                                  Register src, Register dst, Register count,
+                                  Register preserve1, Register preserve2);
+  virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                  Register dst, Register count,
+                                  Register preserve);
 
   virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                         Register base, RegisterOrConstant ind_or_offs, Register val,

src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp

@@ -78,9 +78,14 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI
       LIR_Opr result = gen->new_register(T_INT);
 
       __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), tmp1, tmp2, result));
+
+      if (ShenandoahCardBarrier) {
+        post_barrier(access, access.resolved_addr(), new_value.result());
+      }
       return result;
     }
   }
+
   return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
 }
 
@@ -105,6 +110,9 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
       pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
                   result /* pre_val */);
     }
+    if (ShenandoahCardBarrier) {
+      post_barrier(access, access.resolved_addr(), result);
+    }
   }
 
   return result;