fix issues with single stepping during method exit event

Author: plummercj

src/jdk.jdwp.agent/share/native/libjdwp/stepControl.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -171,18 +171,16 @@ initState(JNIEnv *env, jthread thread, StepRequest *step)
      * Initial values that may be changed below
      */
     step->fromLine = -1;
-    step->fromNative = JNI_FALSE;
+    step->notifyFramePopFailed = JNI_FALSE;
     step->frameExited = JNI_FALSE;
     step->fromStackDepth = getFrameCount(thread);
 
     if (step->fromStackDepth <= 0) {
         /*
-         * If there are no stack frames, treat the step as though
-         * from a native frame. This is most likely to occur at the
-         * beginning of a debug session, right after the VM_INIT event,
-         * so we need to do something intelligent.
+         * If there are no stack frames, there is nothing more to do here. If we are
+         * doing a step INTO, initEvents() will enable stepping. Otherwise it is
+         * not enabled because there is nothing to step OVER or OUT of.
          */
-        step->fromNative = JNI_TRUE;
         return JVMTI_ERROR_NONE;
     }
 
@@ -196,7 +194,13 @@ initState(JNIEnv *env, jthread thread, StepRequest *step)
     error = JVMTI_FUNC_PTR(gdata->jvmti,NotifyFramePop)
                 (gdata->jvmti, thread, 0);
     if (error == JVMTI_ERROR_OPAQUE_FRAME) {
-        step->fromNative = JNI_TRUE;
+        // OPAQUE_FRAME doesn't always mean native method. It's rare that it doesn't, and
+        // means that there is something about the frame's statw that prevents setting up
+        // a NotifyFramePop. One example is a frame that is in the process of returning,
+        // which can happen if we start single stepping after getting a MethodExit event.
+        // In either any case, we need to be aware that there will be no FramePop event
+        // when this frame exits.
+        step->notifyFramePopFailed = JNI_TRUE;
         error = JVMTI_ERROR_NONE;
         /* continue without error */
     } else if (error == JVMTI_ERROR_DUPLICATE) {
@@ -761,31 +765,28 @@ initEvents(jthread thread, StepRequest *step)
         }
 
     }
+    
     /*
-     * Initially enable stepping:
-     * 1) For step into, always
-     * 2) For step over, unless right after the VM_INIT.
-     *    Enable stepping for STEP_MIN or STEP_LINE with or without line numbers.
-     *    If the class is redefined then non EMCP methods may not have line
-     *    number info. So enable line stepping for non line number so that it
-     *    behaves like STEP_MIN/STEP_OVER.
-     * 3) For step out, only if stepping from native, except right after VM_INIT
-     *
-     * (right after VM_INIT, a step->over or out is identical to running
-     * forever)
+     * Enable step events if necessary. Note that right after VM_INIT, a
+     * step OVER or OUT is identical to running forever, so we only enable
+     * step events if fromStackDepth > 0.
      */
     switch (step->depth) {
         case JDWP_STEP_DEPTH(INTO):
             enableStepping(thread);
             break;
         case JDWP_STEP_DEPTH(OVER):
-            if (step->fromStackDepth > 0 && !step->fromNative ) {
+            // We need to always enable for OVER (except right after VM_INIT).
+            // If we are in a native method, that is the only way to find out
+            // that we have returned to a java method.
+            if (step->fromStackDepth > 0) {
               enableStepping(thread);
             }
             break;
         case JDWP_STEP_DEPTH(OUT):
-            if (step->fromNative &&
-                (step->fromStackDepth > 0)) {
+            // We rely on the FramePop event to tell us when we exit the current frame.
+            // If NotifyFramePop failed, then we need to enable stepping.
+            if (step->notifyFramePopFailed && (step->fromStackDepth > 0)) {
                 enableStepping(thread);
             }
             break;

src/jdk.jdwp.agent/share/native/libjdwp/stepControl.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -37,7 +37,7 @@ typedef struct {
     /* State */
     jboolean pending;
     jboolean frameExited;    /* for depth == STEP_OVER or STEP_OUT */
-    jboolean fromNative;
+    jboolean notifyFramePopFailed;
     jint fromStackDepth;     /* for all but STEP_INTO STEP_INSTRUCTION */
     jint fromLine;           /* for granularity == STEP_LINE */
     jmethodID method;   /* Where line table came from. */