community

Author: offamitkumar

src/hotspot/cpu/aarch64/frame_aarch64.cpp

@@ -828,7 +828,6 @@ void JavaFrameAnchor::make_walkable() {
   // already walkable?
   if (walkable()) return;
   vmassert(last_Java_sp() != nullptr, "not called from Java code?");
-  vmassert(last_Java_pc() == nullptr, "already walkable");
   _last_Java_pc = (address)_last_Java_sp[-1];
   vmassert(walkable(), "something went wrong");
 }

src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp

@@ -35,6 +35,53 @@
 
 // Inline functions for AArch64 frames:
 
+#if INCLUDE_JFR
+
+// Static helper routines
+
+inline address frame::interpreter_bcp(const intptr_t* fp) {
+  assert(fp != nullptr, "invariant");
+  return reinterpret_cast<address>(fp[frame::interpreter_frame_bcp_offset]);
+}
+
+inline address frame::interpreter_return_address(const intptr_t* fp) {
+  assert(fp != nullptr, "invariant");
+  return reinterpret_cast<address>(fp[frame::return_addr_offset]);
+}
+
+inline intptr_t* frame::interpreter_sender_sp(const intptr_t* fp) {
+  assert(fp != nullptr, "invariant");
+  return reinterpret_cast<intptr_t*>(fp[frame::interpreter_frame_sender_sp_offset]);
+}
+
+inline bool frame::is_interpreter_frame_setup_at(const intptr_t* fp, const void* sp) {
+  assert(fp != nullptr, "invariant");
+  assert(sp != nullptr, "invariant");
+  return sp <= fp + frame::interpreter_frame_initial_sp_offset;
+}
+
+inline intptr_t* frame::sender_sp(intptr_t* fp) {
+  assert(fp != nullptr, "invariant");
+  return fp + frame::sender_sp_offset;
+}
+
+inline intptr_t* frame::link(const intptr_t* fp) {
+  assert(fp != nullptr, "invariant");
+  return reinterpret_cast<intptr_t*>(fp[frame::link_offset]);
+}
+
+inline address frame::return_address(const intptr_t* sp) {
+  assert(sp != nullptr, "invariant");
+  return reinterpret_cast<address>(sp[-1]);
+}
+
+inline intptr_t* frame::fp(const intptr_t* sp) {
+  assert(sp != nullptr, "invariant");
+  return reinterpret_cast<intptr_t*>(sp[-2]);
+}
+
+#endif // INCLUDE_JFR
+
 // Constructors:
 
 inline frame::frame() {

src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp

@@ -41,6 +41,7 @@
 #include "prims/jvmtiExport.hpp"
 #include "prims/jvmtiThreadState.hpp"
 #include "runtime/basicLock.hpp"
+#include "runtime/continuationEntry.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/javaThread.hpp"
 #include "runtime/safepointMechanism.hpp"
@@ -456,11 +457,43 @@ void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {
   dispatch_base(state, table);
 }
 
+void InterpreterMacroAssembler::call_VM_with_sender_Java_fp_entry(address entry_point) {
+  mov(c_rarg0, rthread);
+  MacroAssembler::call_VM_leaf_base(entry_point, 0);
+}
+
+void InterpreterMacroAssembler::set_last_Java_frame_with_sender_fp(Register last_java_sp,
+                                                                   Register last_java_fp,
+                                                                   address last_java_pc,
+                                                                   Register scratch) {
+  assert_different_registers(last_java_fp, rfp);
+
+#if INCLUDE_JFR
+  Label L_ljf, L_valid_rfp;
+  cbnz(rfp, L_valid_rfp);
+  mov(scratch, 1);
+  str(scratch, Address(rthread, JavaThread::last_sender_Java_fp_offset()));
+  b(L_ljf);
+  bind(L_valid_rfp);
+  str(rfp, Address(rthread, JavaThread::last_sender_Java_fp_offset()));
+  bind(L_ljf);
+#endif
+
+  set_last_Java_frame(last_java_sp, last_java_fp, last_java_pc, scratch);
+}
+
+void InterpreterMacroAssembler::reset_last_Java_frame_with_sender_fp(Register fp_reg) {
+  // Restore the fp_reg.
+  ldr(fp_reg, Address(rthread, JavaThread::last_Java_fp_offset()));
+  reset_last_Java_frame(true);
+  JFR_ONLY(str(zr, Address(rthread, JavaThread::last_sender_Java_fp_offset()));)
+}
+
 // remove activation
 //
-// Apply stack watermark barrier.
 // Unlock the receiver if this is a synchronized method.
 // Unlock any Java monitors from synchronized blocks.
+// Apply stack watermark barrier.
 // Remove the activation from the stack.
 //
 // If there are locked Java monitors
@@ -470,30 +503,14 @@ void InterpreterMacroAssembler::dispatch_via(TosState state, address* table) {
 //       installs IllegalMonitorStateException
 //    Else
 //       no error processing
-void InterpreterMacroAssembler::remove_activation(
-        TosState state,
-        bool throw_monitor_exception,
-        bool install_monitor_exception,
-        bool notify_jvmdi) {
+void InterpreterMacroAssembler::remove_activation(TosState state,
+                                                  bool throw_monitor_exception,
+                                                  bool install_monitor_exception,
+                                                  bool notify_jvmdi) {
   // Note: Registers r3 xmm0 may be in use for the
   // result check if synchronized method
   Label unlocked, unlock, no_unlock;
 
-  // The below poll is for the stack watermark barrier. It allows fixing up frames lazily,
-  // that would normally not be safe to use. Such bad returns into unsafe territory of
-  // the stack, will call InterpreterRuntime::at_unwind.
-  Label slow_path;
-  Label fast_path;
-  safepoint_poll(slow_path, true /* at_return */, false /* acquire */, false /* in_nmethod */);
-  br(Assembler::AL, fast_path);
-  bind(slow_path);
-  push(state);
-  set_last_Java_frame(esp, rfp, (address)pc(), rscratch1);
-  super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
-  reset_last_Java_frame(true);
-  pop(state);
-  bind(fast_path);
-
   // get the value of _do_not_unlock_if_synchronized into r3
   const Address do_not_unlock_if_synchronized(rthread,
     in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
@@ -647,17 +664,70 @@ void InterpreterMacroAssembler::remove_activation(
     bind(no_reserved_zone_enabling);
   }
 
-  // restore sender esp
-  mov(esp, rscratch2);
-  // remove frame anchor
-  leave();
+  // For asynchronous profiling to work correctly, we must remove the
+  // activation frame _before_ we test the method return safepoint poll.
+  // This is equivalent to how it is done for compiled frames.
+  // Removing an interpreter activation frame from a sampling perspective means
+  // updating the frame link. But since we are unwinding the current frame,
+  // we must save the current rfp in a temporary register, this_fp, for use
+  // as the last java fp should we decide to unwind.
+  // The asynchronous profiler will only see the updated rfp, either using the
+  // CPU context or by reading the last_sender_Java_fp() field as part of the ljf.
+  const Register this_fp = rscratch2;
+  make_sender_fp_current(this_fp, rscratch1);
+
+  // The interpreter frame is now unwound from a sampling perspective,
+  // meaning it sees the sender frame as the current frame from this point onwards.
+
+  // The below poll is for the stack watermark barrier. It allows fixing up frames lazily,
+  // that would normally not be safe to use. Such bad returns into unsafe territory of
+  // the stack, will call InterpreterRuntime::at_unwind.
+  Label slow_path;
+  Label fast_path;
+  safepoint_poll(slow_path, this_fp, true /* at_return */, false /* acquire */, false /* in_nmethod */);
+  br(Assembler::AL, fast_path);
+  bind(slow_path);
+  save_bcp(this_fp); // need to save bcp but not restore it.
+  push(state);
+  set_last_Java_frame_with_sender_fp(esp, this_fp, (address)pc(), rscratch1);
+  call_VM_with_sender_Java_fp_entry(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind));
+  reset_last_Java_frame_with_sender_fp(this_fp);
+  pop(state);
+  bind(fast_path);
+
+  ldr(esp, Address(this_fp, frame::interpreter_frame_sender_sp_offset * wordSize));
+  ldr(lr, Address(this_fp, wordSize));
+  authenticate_return_address();
+  lea(sp, Address(this_fp, 2 * wordSize));
+
   // If we're returning to interpreted code we will shortly be
   // adjusting SP to allow some space for ESP.  If we're returning to
   // compiled code the saved sender SP was saved in sender_sp, so this
   // restores it.
   andr(sp, esp, -16);
 }
 
+void InterpreterMacroAssembler::make_sender_fp_current(Register save_this_fp, Register tmp) {
+  const Register return_addr = save_this_fp;
+  const Register continuation_return_pc = tmp;
+  const Register sender_sp = tmp;
+  ldr(return_addr, Address(rfp, wordSize)); // return address
+  // Load address of ContinuationEntry return pc
+  lea(continuation_return_pc, ExternalAddress(ContinuationEntry::return_pc_address()));
+  // Load the ContinuationEntry return pc
+  ldr(continuation_return_pc, Address(continuation_return_pc));
+  Label L_continuation, L_end;
+  cmp(continuation_return_pc, return_addr);
+  mov(save_this_fp, rfp); // Save current fp in temporary register.
+  br(Assembler::EQ, L_continuation);
+  ldr(rfp, Address(rfp)); // Update the frame link.
+  b(L_end);
+  bind(L_continuation);
+  lea(sender_sp, Address(rfp, frame::sender_sp_offset * wordSize));
+  ldr(rfp, Address(sender_sp, (int)(ContinuationEntry::size()))); // Update the frame link.
+  bind(L_end);
+}
+
 // Lock object
 //
 // Args:

src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -63,14 +63,27 @@ class InterpreterMacroAssembler: public MacroAssembler {
                            Register arg_1);
   void restore_after_resume(bool is_native);
 
+  void call_VM_with_sender_Java_fp_entry(address entry_point);
+
+  void set_last_Java_frame_with_sender_fp(Register last_java_sp,
+                                          Register last_java_fp,
+                                          address last_java_pc,
+                                          Register scratch);
+
+  void reset_last_Java_frame_with_sender_fp(Register fp_reg);
+
   void jump_to_entry(address entry);
 
   virtual void check_and_handle_popframe(Register java_thread);
   virtual void check_and_handle_earlyret(Register java_thread);
 
   // Interpreter-specific registers
   void save_bcp() {
-    str(rbcp, Address(rfp, frame::interpreter_frame_bcp_offset * wordSize));
+    save_bcp(rfp);
+  }
+
+  void save_bcp(Register fp_register) {
+    str(rbcp, Address(fp_register, frame::interpreter_frame_bcp_offset * wordSize));
   }
 
   void restore_bcp() {
@@ -225,6 +238,8 @@ class InterpreterMacroAssembler: public MacroAssembler {
                          bool install_monitor_exception = true,
                          bool notify_jvmdi = true);
 
+  void make_sender_fp_current(Register save_this_fp, Register tmp);
+
   // FIXME: Give us a valid frame at a null check.
   virtual void null_check(Register reg, int offset = -1) {
 // #ifdef ASSERT

src/hotspot/cpu/aarch64/javaFrameAnchor_aarch64.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -26,12 +26,13 @@
 #ifndef CPU_AARCH64_JAVAFRAMEANCHOR_AARCH64_HPP
 #define CPU_AARCH64_JAVAFRAMEANCHOR_AARCH64_HPP
 
-private:
+ private:
 
   // FP value associated with _last_Java_sp:
   intptr_t* volatile        _last_Java_fp;           // pointer is volatile not what it points to
+  JFR_ONLY(intptr_t* volatile _last_sender_Java_fp;) // specialized field for when JFR samples an interpreter frame
 
-public:
+ public:
   // Each arch must define reset, save, restore
   // These are used by objects that only care about:
   //  1 - initializing a new state (thread creation, javaCalls)
@@ -44,6 +45,7 @@
     OrderAccess::release();
     _last_Java_fp = nullptr;
     _last_Java_pc = nullptr;
+    JFR_ONLY(_last_sender_Java_fp = nullptr;)
   }
 
   void copy(JavaFrameAnchor* src) {
@@ -59,6 +61,7 @@
       OrderAccess::release();
     }
     _last_Java_fp = src->_last_Java_fp;
+    JFR_ONLY(_last_sender_Java_fp = src->_last_sender_Java_fp;)
     _last_Java_pc = src->_last_Java_pc;
     // Must be last so profiler will always see valid frame if has_last_frame() is true
     _last_Java_sp = src->_last_Java_sp;
@@ -72,16 +75,18 @@
 
   address last_Java_pc(void)                     { return _last_Java_pc; }
 
-private:
-
-  static ByteSize last_Java_fp_offset()          { return byte_offset_of(JavaFrameAnchor, _last_Java_fp); }
-
-public:
+ public:
 
   void set_last_Java_sp(intptr_t* sp)            { _last_Java_sp = sp; OrderAccess::release(); }
 
-  intptr_t*   last_Java_fp(void)                 { return _last_Java_fp; }
+  intptr_t*   last_Java_fp() const               { return _last_Java_fp; }
 
   void set_last_Java_fp(intptr_t* fp)            { _last_Java_fp = fp; }
 
+  JFR_ONLY(intptr_t* last_sender_Java_fp() const { return _last_sender_Java_fp; })
+
+  static ByteSize last_Java_fp_offset() { return byte_offset_of(JavaFrameAnchor, _last_Java_fp); }
+
+  JFR_ONLY(static ByteSize last_sender_Java_fp_offset() { return byte_offset_of(JavaFrameAnchor, _last_sender_Java_fp); })
+
 #endif // CPU_AARCH64_JAVAFRAMEANCHOR_AARCH64_HPP

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -554,6 +554,11 @@ address MacroAssembler::target_addr_for_insn_or_null(address insn_addr, unsigned
 }
 
 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod, Register tmp) {
+  safepoint_poll(slow_path, rfp, at_return, acquire, in_nmethod, tmp);
+}
+
+void MacroAssembler::safepoint_poll(Label& slow_path, Register fp_reg, bool at_return, bool acquire, bool in_nmethod, Register tmp) {
+  assert(fp_reg != tmp, "invariant");
   if (acquire) {
     lea(tmp, Address(rthread, JavaThread::polling_word_offset()));
     ldar(tmp, tmp);
@@ -563,7 +568,7 @@ void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acqui
   if (at_return) {
     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
     // we may safely use the sp instead to perform the stack watermark check.
-    cmp(in_nmethod ? sp : rfp, tmp);
+    cmp(in_nmethod ? sp : fp_reg, tmp);
     br(Assembler::HI, slow_path);
   } else {
     tbnz(tmp, log2i_exact(SafepointMechanism::poll_bit()), slow_path);

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -121,6 +121,8 @@ class MacroAssembler: public Assembler {
  virtual void check_and_handle_earlyret(Register java_thread);
 
   void safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod, Register tmp = rscratch1);
+  void safepoint_poll(Label& slow_path, Register fp_reg, bool at_return, bool acquire, bool in_nmethod, Register tmp = rscratch1);
+
   void rt_call(address dest, Register tmp = rscratch1);
 
   // Load Effective Address