[GR-68545] Ignore deopt nodes with deopt action "None" for deopt cycle detection.

PullRequest: graal/21783

Author: jchalou

compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/truffle/test/DeoptLoopDetectionTest.java

@@ -107,6 +107,48 @@ public Object execute(VirtualFrame frame) {
         }, "alwaysDeopt", CallTarget::call, 1);
     }
 
+    @Test
+    public void testAlwaysDeoptNoInvalidate() {
+        AssertionError expectedError = Assert.assertThrows(AssertionError.class, () -> assertDeoptLoop(new BaseRootNode() {
+            @CompilerDirectives.TruffleBoundary
+            static void boundaryMethod() {
+
+            }
+
+            @Override
+            public Object execute(VirtualFrame frame) {
+                int arg = (int) frame.getArguments()[0];
+                int threshold = TruffleCompilerOptions.DeoptCycleDetectionThreshold.getDefaultValue();
+                if (arg < threshold) {
+                    CompilerDirectives.transferToInterpreterAndInvalidate();
+                }
+                // call boundary method to prevent compiler from moving the following deoptimization
+                // up
+                boundaryMethod();
+                CompilerDirectives.transferToInterpreter();
+                return null;
+            }
+        }, "alwaysDeoptNoInvalidate", new Consumer<CallTarget>() {
+            int i;
+
+            @Override
+            public void accept(CallTarget callTarget) {
+                callTarget.call(i++);
+                if (i == TruffleCompilerOptions.DeoptCycleDetectionThreshold.getDefaultValue() + 1) {
+                    /*
+                     * Invalidate the target that was just deoptimized (but not invalidated) by
+                     * transferToInterpreter. The exact same compilation is then repeated in the
+                     * next iteration, but because deoptimize nodes with deoptimization action
+                     * "None" (like the one used for transferToInterpreter) don't trigger deopt loop
+                     * detection, no deopt loop should be detected.
+                     */
+                    ((OptimizedCallTarget) callTarget).invalidate("Force one more recompile");
+                }
+            }
+        }, 1));
+        Assert.assertEquals("No deopt loop detected after " + MAX_EXECUTIONS + " executions", expectedError.getMessage());
+    }
+
     @Test
     public void testLocalDeopt() {
         assertDeoptLoop(new BaseRootNode() {

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/ForceDeoptSpeculationPhase.java

@@ -87,7 +87,7 @@ protected void run(StructuredGraph graph, CoreProviders context) {
             if (node instanceof DynamicDeoptimizeNode) {
                 throw new GraalError("%s must be disabled to use this phase", DynamicDeoptimizeNode.class.getSimpleName());
             }
-            if (node instanceof DeoptimizeNode deopt) {
+            if (node instanceof DeoptimizeNode deopt && deopt.getAction() != DeoptimizationAction.None) {
                 if (deopt.getSpeculation().equals(NO_SPECULATION)) {
                     SpeculationLog.Speculation speculation = createSpeculation(graph, signature, deopt);
                     deopt.setSpeculation(speculation);
@@ -97,12 +97,14 @@ protected void run(StructuredGraph graph, CoreProviders context) {
                 JavaConstant spec = implicitNullCheck.getDeoptSpeculation();
                 SpeculationLog.Speculation speculation = spec != null ? context.getMetaAccess().decodeSpeculation(spec, speculationLog) : NO_SPECULATION;
                 if (speculation.equals(NO_SPECULATION)) {
-                    spec = context.getMetaAccess().encodeSpeculation(createSpeculation(graph, signature, implicitNullCheck));
                     JavaConstant actionAndReason = implicitNullCheck.getDeoptReasonAndAction();
-                    if (actionAndReason == null) {
-                        actionAndReason = context.getMetaAccess().encodeDeoptActionAndReason(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NullCheckException, 0);
+                    if (actionAndReason == null || context.getMetaAccess().decodeDeoptAction(actionAndReason) != DeoptimizationAction.None) {
+                        spec = context.getMetaAccess().encodeSpeculation(createSpeculation(graph, signature, implicitNullCheck));
+                        if (actionAndReason == null) {
+                            actionAndReason = context.getMetaAccess().encodeDeoptActionAndReason(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NullCheckException, 0);
+                        }
+                        implicitNullCheck.setImplicitDeoptimization(actionAndReason, spec);
                     }
-                    implicitNullCheck.setImplicitDeoptimization(actionAndReason, spec);
                 }
             }
         }