Fix adapting of loop exit probabilities.

Author: rmosaner

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/LoopTransformations.java

@@ -370,8 +370,6 @@ public static PreMainPostResult insertPrePostLoops(Loop loop) {
         AbstractMergeNode postMergeNode = postEndNode.merge();
         graph.getDebug().dump(DebugContext.VERY_DETAILED_LEVEL, graph, "After post loop duplication");
 
-        preLoopBegin.incrementSplits();
-        preLoopBegin.incrementSplits();
         preLoopBegin.setPreLoop();
         mainLoopBegin.setMainLoop();
         postLoopBegin.setPostLoop();
@@ -454,8 +452,18 @@ public static PreMainPostResult insertPrePostLoops(Loop loop) {
         assert mainLoopExitNode.predecessor() instanceof IfNode : Assertions.errorMessage(mainLoopExitNode);
         assert postLoopExitNode.predecessor() instanceof IfNode : Assertions.errorMessage(postLoopExitNode);
 
-        setSingleVisitedLoopFrequencySplitProbability(preLoopExitNode);
-        setSingleVisitedLoopFrequencySplitProbability(postLoopExitNode);
+        /*
+         * The bodies of pre and post loops are assumed to be executed just once. As the local loop
+         * frequency is calculated from the loop exit probabilities, it has to be taken into account
+         * how often the exit check is performed. If the loop is inverted, i.e., tail-counted, the
+         * exit will be taken at the end of the first body execution. Thus, the frequency of the
+         * exit check is 1. If the loop is head-counted, the exit check will be performed twice,
+         * which is reflected by a frequency of 2. This results in the correct relative frequency
+         * being propagated into the loop body.
+         */
+        final int prePostFrequency = loop.counted().isInverted() ? 1 : 2;
+        adaptCountedLoopExitProbability(preLoopExitNode, prePostFrequency);
+        adaptCountedLoopExitProbability(postLoopExitNode, prePostFrequency);
 
         if (graph.isAfterStage(StageFlag.VALUE_PROXY_REMOVAL)) {
             // The pre and post loops don't require safepoints at all
@@ -473,7 +481,8 @@ public static PreMainPostResult insertPrePostLoops(Loop loop) {
 
     /**
      * Inject a split probability for the (counted) loop check that will result in a loop frequency
-     * of 1 (in case this is the only loop exit).
+     * of 1 (in case this is the only loop exit). This implies that the loop body is expected to be
+     * never entered.
      */
     private static void setSingleVisitedLoopFrequencySplitProbability(AbstractBeginNode lex) {
         IfNode ifNode = ((IfNode) lex.predecessor());
@@ -482,11 +491,12 @@ private static void setSingleVisitedLoopFrequencySplitProbability(AbstractBeginN
     }
 
     /**
-     * Inject a new frequency for the condition dominating the given loop exit path. This
-     * calculation will act as if the given loop exit is the only exit of the loop.
+     * Inject a new branch probability for the condition dominating the given loop exit path. This
+     * probability is based on the local frequency of the exit check. This calculation will act as
+     * if the given loop exit is the only exit of the loop.
      */
-    public static void adaptCountedLoopExitProbability(AbstractBeginNode lex, double newFrequency) {
-        double probability = 1.0D - 1.0D / newFrequency;
+    public static void adaptCountedLoopExitProbability(AbstractBeginNode lex, double newExitCheckFrequency) {
+        double probability = 1.0D - 1.0D / newExitCheckFrequency;
         if (probability <= 0D) {
             setSingleVisitedLoopFrequencySplitProbability(lex);
             return;

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopBeginNode.java

@@ -210,7 +210,6 @@ public LoopBeginNode() {
         super(TYPE);
         loopOrigFrequency = 1;
         unswitches = 0;
-        splits = 0;
         loopEndsSafepointState = SafepointState.ENABLED;
         loopExitsSafepointState = SafepointState.ENABLED;
         guestLoopEndsSafepointState = SafepointState.ENABLED;
@@ -483,10 +482,6 @@ public EndNode forwardEnd() {
         return forwardEndAt(0);
     }
 
-    public void incrementSplits() {
-        splits++;
-    }
-
     public int peelings() {
         return peelings;
     }