8316396: Endless loop in C2 compilation triggered by AddNode::IdealIL

Reviewed-by: thartmann, kvn

Author: rwestrel

src/hotspot/share/opto/addnode.cpp

@@ -280,8 +280,26 @@ Node* AddNode::IdealIL(PhaseGVN* phase, bool can_reshape, BasicType bt) {
       assert( in1->in(2) != this && in2->in(2) != this,
               "dead loop in AddINode::Ideal" );
       Node* sub = SubNode::make(nullptr, nullptr, bt);
-      sub->init_req(1, phase->transform(AddNode::make(in1->in(1), in2->in(1), bt)));
-      sub->init_req(2, phase->transform(AddNode::make(in1->in(2), in2->in(2), bt)));
+      Node* sub_in1;
+      PhaseIterGVN* igvn = phase->is_IterGVN();
+      // During IGVN, if both inputs of the new AddNode are a tree of SubNodes, this same transformation will be applied
+      // to every node of the tree. Calling transform() causes the transformation to be applied recursively, once per
+      // tree node whether some subtrees are identical or not. Pushing to the IGVN worklist instead, causes the transform
+      // to be applied once per unique subtrees (because all uses of a subtree are updated with the result of the
+      // transformation). In case of a large tree, this can make a difference in compilation time.
+      if (igvn != nullptr) {
+        sub_in1 = igvn->register_new_node_with_optimizer(AddNode::make(in1->in(1), in2->in(1), bt));
+      } else {
+        sub_in1 = phase->transform(AddNode::make(in1->in(1), in2->in(1), bt));
+      }
+      Node* sub_in2;
+      if (igvn != nullptr) {
+        sub_in2 = igvn->register_new_node_with_optimizer(AddNode::make(in1->in(2), in2->in(2), bt));
+      } else {
+        sub_in2 = phase->transform(AddNode::make(in1->in(2), in2->in(2), bt));
+      }
+      sub->init_req(1, sub_in1);
+      sub->init_req(2, sub_in2);
       return sub;
     }
     // Convert "(a-b)+(b+c)" into "(a+c)"

test/hotspot/jtreg/compiler/c2/TestLargeTreeOfSubNodes.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2023, Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8316396
+ * @summary Endless loop in C2 compilation triggered by AddNode::IdealIL
+ * @run main/othervm  -XX:CompileCommand=compileonly,*TestLargeTreeOfSubNodes*::test -XX:-TieredCompilation -Xcomp TestLargeTreeOfSubNodes
+ */
+
+public class TestLargeTreeOfSubNodes {
+    public static long res = 0;
+
+    public static void test() {
+        int a = -1, b = 0;
+        for (int i = 0; i < 100; ++i) {
+            for (int j = 0; j < 10; ++j) {
+                for (int k = 0; k < 1; ++k) {
+                }
+                b -= a;
+                a += b;
+            }
+        }
+        res = a;
+    }
+
+    public static void main(String[] args) {
+        test();
+    }
+}