8330045: Enhance array handling

Backport-of: 727c2612766dd1737bf59fb0d9ba1e58a7b8c32b

Author: martinuy

src/hotspot/share/opto/subnode.cpp

@@ -1483,44 +1483,86 @@ Node *BoolNode::Ideal(PhaseGVN *phase, bool can_reshape) {
     return new BoolNode( ncmp, _test.negate() );
   }
 
-  // Change ((x & m) u<= m) or ((m & x) u<= m) to always true
-  // Same with ((x & m) u< m+1) and ((m & x) u< m+1)
+  // We use the following Lemmas/insights for the following two transformations (1) and (2):
+  //   x & y <=u y, for any x and y           (Lemma 1, masking always results in a smaller unsigned number)
+  //   y <u y + 1 is always true if y != -1   (Lemma 2, (uint)(-1 + 1) == (uint)(UINT_MAX + 1) which overflows)
+  //   y <u 0 is always false for any y       (Lemma 3, 0 == UINT_MIN and nothing can be smaller than that)
+  //
+  // (1a) Always:     Change ((x & m) <=u m  ) or ((m & x) <=u m  ) to always true   (true by Lemma 1)
+  // (1b) If m != -1: Change ((x & m) <u  m + 1) or ((m & x) <u  m + 1) to always true:
+  //    x & m <=u m          is always true   // (Lemma 1)
+  //    x & m <=u m <u m + 1 is always true   // (Lemma 2: m <u m + 1, if m != -1)
+  //
+  // A counter example for (1b), if we allowed m == -1:
+  //     (x & m)  <u m + 1
+  //     (x & -1) <u 0
+  //      x       <u 0
+  //   which is false for any x (Lemma 3)
   if (cop == Op_CmpU &&
       cmp1_op == Op_AndI) {
-    Node* bound = nullptr;
+    Node* m = nullptr;
     if (_test._test == BoolTest::le) {
-      bound = cmp2;
+      // (1a) "((x & m) <=u m)", cmp2 = m
+      m = cmp2;
     } else if (_test._test == BoolTest::lt &&
                cmp2->Opcode() == Op_AddI &&
                cmp2->in(2)->find_int_con(0) == 1) {
-      bound = cmp2->in(1);
+      // (1b) "(x & m) <u m + 1" and "(m & x) <u m + 1", cmp2 = m + 1
+      Node* rhs_m = cmp2->in(1);
+      const TypeInt* rhs_m_type = phase->type(rhs_m)->isa_int();
+      if (rhs_m_type->_lo > -1 || rhs_m_type->_hi < -1) {
+        // Exclude any case where m == -1 is possible.
+        m = rhs_m;
+      }
     }
-    if (cmp1->in(2) == bound || cmp1->in(1) == bound) {
+    if (cmp1->in(2) == m || cmp1->in(1) == m) {
       return ConINode::make(1);
     }
   }
 
-  // Change ((x & (m - 1)) u< m) into (m > 0)
-  // This is the off-by-one variant of the above
+  // (2) Change ((x & (m - 1)) <u m) or (((m - 1) & x) <u m) to (m >u 0)
+  // This is the off-by-one variant of the above.
+  //
+  // We now prove that this replacement is correct. This is the same as proving
+  //   "m >u 0" if and only if "x & (m - 1) <u m", i.e. "m >u 0 <=> x & (m - 1) <u m"
+  //
+  // We use (Lemma 1) and (Lemma 3) from above.
+  //
+  // Case "x & (m - 1) <u m => m >u 0":
+  //   We prove this by contradiction:
+  //     Assume m <=u 0 which is equivalent to m == 0:
+  //   and thus
+  //     x & (m - 1) <u m = 0               // m == 0
+  //     y           <u     0               // y = x & (m - 1)
+  //   by Lemma 3, this is always false, i.e. a contradiction to our assumption.
+  //
+  // Case "m >u 0 => x & (m - 1) <u m":
+  //   x & (m - 1) <=u (m - 1)              // (Lemma 1)
+  //   x & (m - 1) <=u (m - 1) <u m         // Using assumption m >u 0, no underflow of "m - 1"
+  //
+  //
+  // Note that the signed version of "m > 0":
+  //   m > 0 <=> x & (m - 1) <u m
+  // does not hold:
+  //   Assume m == -1 and x == -1:
+  //     x  & (m - 1) <u m
+  //     -1 & -2      <u -1
+  //     -2           <u -1
+  //     UINT_MAX - 1 <u UINT_MAX           // Signed to unsigned numbers
+  // which is true while
+  //   m > 0
+  // is false which is a contradiction.
   if (cop == Op_CmpU &&
       _test._test == BoolTest::lt &&
       cmp1_op == Op_AndI) {
-    Node* l = cmp1->in(1);
-    Node* r = cmp1->in(2);
-    for (int repeat = 0; repeat < 2; repeat++) {
-      bool match = r->Opcode() == Op_AddI && r->in(2)->find_int_con(0) == -1 &&
-                   r->in(1) == cmp2;
-      if (match) {
-        // arraylength known to be non-negative, so a (arraylength != 0) is sufficient,
-        // but to be compatible with the array range check pattern, use (arraylength u> 0)
-        Node* ncmp = cmp2->Opcode() == Op_LoadRange
-                     ? phase->transform(new CmpUNode(cmp2, phase->intcon(0)))
-                     : phase->transform(new CmpINode(cmp2, phase->intcon(0)));
-        return new BoolNode(ncmp, BoolTest::gt);
-      } else {
-        // commute and try again
-        l = cmp1->in(2);
-        r = cmp1->in(1);
+    Node* m = cmp2; // RHS: m
+    for (int add_idx = 1; add_idx <= 2; add_idx++) { // LHS: "(m + (-1)) & x" or "x & (m + (-1))"?
+      Node* maybe_m_minus_1 = cmp1->in(add_idx);
+      if (maybe_m_minus_1->Opcode() == Op_AddI &&
+          maybe_m_minus_1->in(2)->find_int_con(0) == -1 &&
+          maybe_m_minus_1->in(1) == m) {
+        Node* m_cmpu_0 = phase->transform(new CmpUNode(m, phase->intcon(0)));
+        return new BoolNode(m_cmpu_0, BoolTest::gt);
       }
     }
   }