8377554: Load card table base and other values via AOTRuntimeConstants in AOT code

Reviewed-by: kvn, asmehra

Author: adinn

src/hotspot/cpu/aarch64/aarch64.ad

@@ -3403,11 +3403,13 @@ encode %{
       } else if (rtype == relocInfo::metadata_type) {
         __ mov_metadata(dst_reg, (Metadata*)con);
       } else {
-        assert(rtype == relocInfo::none, "unexpected reloc type");
+        assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
+        // load fake address constants using a normal move
         if (! __ is_valid_AArch64_address(con) ||
             con < (address)(uintptr_t)os::vm_page_size()) {
           __ mov(dst_reg, con);
         } else {
+          // no reloc so just use adrp and add
           uint64_t offset;
           __ adrp(dst_reg, con, offset);
           __ add(dst_reg, dst_reg, offset);
@@ -4535,6 +4537,18 @@ operand immP_1()
   interface(CONST_INTER);
 %}
 
+// AOT Runtime Constants Address
+operand immAOTRuntimeConstantsAddress()
+%{
+  // Check if the address is in the range of AOT Runtime Constants
+  predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
+  match(ConP);
+
+  op_cost(0);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
 // Float and Double operands
 // Double Immediate
 operand immD()
@@ -6898,6 +6912,20 @@ instruct loadConP1(iRegPNoSp dst, immP_1 con)
   ins_pipe(ialu_imm);
 %}
 
+instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
+%{
+  match(Set dst con);
+
+  ins_cost(INSN_COST);
+  format %{ "adr  $dst, $con\t# AOT Runtime Constants Address" %}
+
+  ins_encode %{
+    __ load_aotrc_address($dst$$Register, (address)$con$$constant);
+  %}
+
+  ins_pipe(ialu_imm);
+%}
+
 // Load Narrow Pointer Constant
 
 instruct loadConN(iRegNNoSp dst, immN con)

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

@@ -33,6 +33,7 @@
 #include "c1/c1_ValueStack.hpp"
 #include "ci/ciArrayKlass.hpp"
 #include "ci/ciInstance.hpp"
+#include "code/aotCodeCache.hpp"
 #include "code/compiledIC.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/gc_globals.hpp"
@@ -532,6 +533,15 @@ void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_cod
 
     case T_LONG: {
       assert(patch_code == lir_patch_none, "no patching handled here");
+#if INCLUDE_CDS
+      if (AOTCodeCache::is_on_for_dump()) {
+        address b = c->as_pointer();
+        if (AOTRuntimeConstants::contains(b)) {
+          __ load_aotrc_address(dest->as_register_lo(), b);
+          break;
+        }
+      }
+#endif
       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
       break;
     }

src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp

@@ -23,6 +23,7 @@
  */
 
 #include "asm/macroAssembler.inline.hpp"
+#include "code/aotCodeCache.hpp"
 #include "gc/g1/g1BarrierSet.hpp"
 #include "gc/g1/g1BarrierSetAssembler.hpp"
 #include "gc/g1/g1BarrierSetRuntime.hpp"
@@ -243,9 +244,25 @@ static void generate_post_barrier(MacroAssembler* masm,
   assert_different_registers(store_addr, new_val, thread, tmp1, tmp2, noreg, rscratch1);
 
   // Does store cross heap regions?
-  __ eor(tmp1, store_addr, new_val);                     // tmp1 := store address ^ new value
-  __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);   // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
-  __ cbz(tmp1, done);
+ #if INCLUDE_CDS
+  // AOT code needs to load the barrier grain shift from the aot
+  // runtime constants area in the code cache otherwise we can compile
+  // it as an immediate operand
+  if (AOTCodeCache::is_on_for_dump()) {
+    address grain_shift_address = (address)AOTRuntimeConstants::grain_shift_address();
+    __ eor(tmp1, store_addr, new_val);
+    __ lea(tmp2, ExternalAddress(grain_shift_address));
+    __ ldrb(tmp2, tmp2);
+    __ lsrv(tmp1, tmp1, tmp2);
+    __ cbz(tmp1, done);
+  } else
+#endif
+  {
+    __ eor(tmp1, store_addr, new_val);                     // tmp1 := store address ^ new value
+    __ lsr(tmp1, tmp1, G1HeapRegion::LogOfHRGrainBytes);   // tmp1 := ((store address ^ new value) >> LogOfHRGrainBytes)
+    __ cbz(tmp1, done);
+  }
+
   // Crosses regions, storing null?
   if (new_val_may_be_null) {
     __ cbz(new_val, done);

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -5754,13 +5754,35 @@ void MacroAssembler::adrp(Register reg1, const Address &dest, uint64_t &byte_off
 }
 
 void MacroAssembler::load_byte_map_base(Register reg) {
+#if INCLUDE_CDS
+  if (AOTCodeCache::is_on_for_dump()) {
+    address byte_map_base_adr = AOTRuntimeConstants::card_table_base_address();
+    lea(reg, ExternalAddress(byte_map_base_adr));
+    ldr(reg, Address(reg));
+    return;
+  }
+#endif
   CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
 
   // Strictly speaking the card table base isn't an address at all, and it might
   // even be negative. It is thus materialised as a constant.
   mov(reg, (uint64_t)ctbs->card_table_base_const());
 }
 
+void MacroAssembler::load_aotrc_address(Register reg, address a) {
+#if INCLUDE_CDS
+  assert(AOTRuntimeConstants::contains(a), "address out of range for data area");
+  if (AOTCodeCache::is_on_for_dump()) {
+    // all aotrc field addresses should be registered in the AOTCodeCache address table
+    lea(reg, ExternalAddress(a));
+  } else {
+    mov(reg, (uint64_t)a);
+  }
+#else
+  ShouldNotReachHere();
+#endif
+}
+
 void MacroAssembler::build_frame(int framesize) {
   assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR");
   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -1476,6 +1476,9 @@ class MacroAssembler: public Assembler {
   // Load the base of the cardtable byte map into reg.
   void load_byte_map_base(Register reg);
 
+  // Load a constant address in the AOT Runtime Constants area
+  void load_aotrc_address(Register reg, address a);
+
   // Prolog generator routines to support switch between x86 code and
   // generated ARM code
 

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@@ -32,6 +32,7 @@
 #include "c1/c1_ValueStack.hpp"
 #include "ci/ciArrayKlass.hpp"
 #include "ci/ciInstance.hpp"
+#include "code/aotCodeCache.hpp"
 #include "compiler/oopMap.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/gc_globals.hpp"
@@ -535,6 +536,15 @@ void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_cod
 
     case T_LONG: {
       assert(patch_code == lir_patch_none, "no patching handled here");
+#if INCLUDE_CDS
+      if (AOTCodeCache::is_on_for_dump()) {
+        address b = c->as_pointer();
+        if (AOTRuntimeConstants::contains(b)) {
+          __ load_aotrc_address(dest->as_register_lo(), b);
+          break;
+        }
+      }
+#endif
       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
       break;
     }

src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp

@@ -23,6 +23,7 @@
  */
 
 #include "asm/macroAssembler.inline.hpp"
+#include "code/aotCodeCache.hpp"
 #include "gc/g1/g1BarrierSet.hpp"
 #include "gc/g1/g1BarrierSetAssembler.hpp"
 #include "gc/g1/g1BarrierSetRuntime.hpp"
@@ -268,6 +269,16 @@ void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
   __ bind(done);
 }
 
+#if INCLUDE_CDS
+// return a register that differs from reg1, reg2, reg3 and reg4
+
+static Register pick_different_reg(Register reg1, Register reg2 = noreg, Register reg3= noreg, Register reg4 = noreg) {
+  RegSet available = (RegSet::of(rscratch1, rscratch2, rax, rbx) + rdx -
+                      RegSet::of(reg1, reg2, reg3, reg4));
+  return *(available.begin());
+}
+#endif // INCLUDE_CDS
+
 static void generate_post_barrier(MacroAssembler* masm,
                                   const Register store_addr,
                                   const Register new_val,
@@ -280,10 +291,32 @@ static void generate_post_barrier(MacroAssembler* masm,
 
   Label L_done;
   // Does store cross heap regions?
-  __ movptr(tmp1, store_addr);                                    // tmp1 := store address
-  __ xorptr(tmp1, new_val);                                       // tmp1 := store address ^ new value
-  __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes);               // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
-  __ jccb(Assembler::equal, L_done);
+#if INCLUDE_CDS
+  // AOT code needs to load the barrier grain shift from the aot
+  // runtime constants area in the code cache otherwise we can compile
+  // it as an immediate operand
+
+  if (AOTCodeCache::is_on_for_dump()) {
+    address grain_shift_addr = AOTRuntimeConstants::grain_shift_address();
+    Register save = pick_different_reg(rcx, tmp1, new_val, store_addr);
+    __ push(save);
+    __ movptr(save, store_addr);
+    __ xorptr(save, new_val);
+    __ push(rcx);
+    __ lea(rcx, ExternalAddress(grain_shift_addr));
+    __ movl(rcx, Address(rcx, 0));
+    __ shrptr(save);
+    __ pop(rcx);
+    __ pop(save);
+    __ jcc(Assembler::equal, L_done);
+  } else
+#endif // INCLUDE_CDS
+  {
+    __ movptr(tmp1, store_addr);                                    // tmp1 := store address
+    __ xorptr(tmp1, new_val);                                       // tmp1 := store address ^ new value
+    __ shrptr(tmp1, G1HeapRegion::LogOfHRGrainBytes);               // ((store address ^ new value) >> LogOfHRGrainBytes) == 0?
+    __ jccb(Assembler::equal, L_done);
+  }
 
   // Crosses regions, storing null?
   if (new_val_may_be_null) {

src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp

@@ -23,6 +23,7 @@
  */
 
 #include "asm/macroAssembler.inline.hpp"
+#include "code/aotCodeCache.hpp"
 #include "gc/shared/barrierSet.hpp"
 #include "gc/shared/cardTable.hpp"
 #include "gc/shared/cardTableBarrierSet.hpp"
@@ -111,7 +112,15 @@ void CardTableBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembl
   __ shrptr(end, CardTable::card_shift());
   __ subptr(end, addr); // end --> cards count
 
-  __ mov64(tmp, (intptr_t)ctbs->card_table_base_const());
+#if INCLUDE_CDS
+  if (AOTCodeCache::is_on_for_dump()) {
+    __ lea(tmp, ExternalAddress(AOTRuntimeConstants::card_table_base_address()));
+    __ movq(tmp, Address(tmp, 0));
+  } else
+#endif
+  {
+    __ mov64(tmp, (intptr_t)ctbs->card_table_base_const());
+  }
   __ addptr(addr, tmp);
 __ BIND(L_loop);
   __ movb(Address(addr, count, Address::times_1), 0);
@@ -121,7 +130,7 @@ __ BIND(L_loop);
 __ BIND(L_done);
 }
 
-void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst) {
+void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst, Register rscratch) {
   // Does a store check for the oop in register obj. The content of
   // register obj is destroyed afterwards.
   CardTableBarrierSet* ctbs = CardTableBarrierSet::barrier_set();
@@ -136,6 +145,13 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
   // never need to be relocated. On 64bit however the value may be too
   // large for a 32bit displacement.
   intptr_t byte_map_base = (intptr_t)ctbs->card_table_base_const();
+#if INCLUDE_CDS
+  if (AOTCodeCache::is_on_for_dump()) {
+    __ lea(rscratch, ExternalAddress(AOTRuntimeConstants::card_table_base_address()));
+    __ movq(rscratch, Address(rscratch, 0));
+    card_addr = Address(rscratch, obj, Address::times_1, 0);
+  } else
+#endif
   if (__ is_simm32(byte_map_base)) {
     card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
   } else {
@@ -174,10 +190,10 @@ void CardTableBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorS
   if (needs_post_barrier) {
     // flatten object address if needed
     if (!precise || (dst.index() == noreg && dst.disp() == 0)) {
-      store_check(masm, dst.base(), dst);
+      store_check(masm, dst.base(), dst, tmp2);
     } else {
       __ lea(tmp1, dst);
-      store_check(masm, tmp1, dst);
+      store_check(masm, tmp1, dst, tmp2);
     }
   }
 }

src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.hpp

@@ -33,7 +33,7 @@ class CardTableBarrierSetAssembler: public BarrierSetAssembler {
   virtual void gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
                                                Register addr, Register count) {}
 
-  void store_check(MacroAssembler* masm, Register obj, Address dst);
+  void store_check(MacroAssembler* masm, Register obj, Address dst, Register rscratch);
 
   virtual void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp);
 

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -10034,6 +10034,20 @@ void MacroAssembler::restore_legacy_gprs() {
   addq(rsp, 16 * wordSize);
 }
 
+void MacroAssembler::load_aotrc_address(Register reg, address a) {
+#if INCLUDE_CDS
+  assert(AOTRuntimeConstants::contains(a), "address out of range for data area");
+  if (AOTCodeCache::is_on_for_dump()) {
+    // all aotrc field addresses should be registered in the AOTCodeCache address table
+    lea(reg, ExternalAddress(a));
+  } else {
+    mov64(reg, (uint64_t)a);
+  }
+#else
+  ShouldNotReachHere();
+#endif
+}
+
 void MacroAssembler::setcc(Assembler::Condition comparison, Register dst) {
   if (VM_Version::supports_apx_f()) {
     esetzucc(comparison, dst);