TRegex: fix integer overflow in minPath and maxPath calculations

Author: djoooooe

regex/src/com.oracle.truffle.regex.test/src/com/oracle/truffle/regex/tregex/test/OracleDBTests.java

@@ -1614,6 +1614,7 @@ public void generatedTests() {
         test("(e?\\D[xg]){87,87}z", "",
                         "axaxeageagageaxeaxeaxageaxagageaxeaxagageagaxaxeagaxeaxagagaxeagageaxeaxeagageaxeaxagaxaxaxageageagageagaxaxaxageaxageaxeageaxaxaxaxaxagaxagageaxeageageageaxeaxeaxageaxaxeaxeagaxagageaxeageaxeaxaxeaxageaxaxeagaxageageaz",
                         0, false);
+        test("((b\\2{1400,1400})+|)*a", "", "a", 0, true, 0, 1, 0, 0, -1, -1);
         test("(a{1100,1100})\\1", "i", "a".repeat(2400), 0, true, 0, 2200, 0, 1100);
         test("[a]\\S{213,213}bcdz", "", "a".repeat(215) + ("bcxd" + "a".repeat(213)).repeat(3), 0, false);
 

regex/src/com.oracle.truffle.regex/src/com/oracle/truffle/regex/tregex/parser/ast/CalcASTPropsVisitor.java

@@ -55,6 +55,7 @@
 import com.oracle.truffle.regex.tregex.buffer.CompilationBuffer;
 import com.oracle.truffle.regex.tregex.parser.ast.visitors.DepthFirstTraversalRegexASTVisitor;
 import com.oracle.truffle.regex.tregex.string.Encodings;
+import com.oracle.truffle.regex.tregex.util.MathUtil;
 
 /**
  * This visitor computes various properties of {@link RegexAST} and its {@link RegexASTNode}s, in
@@ -184,7 +185,7 @@ protected void visit(BackReference backReference) {
         backReference.getParent().setHasBackReferences();
 
         int minWidth = 0;
-        if (ast.getFlavor().backreferencesToUnmatchedGroupsFail()) {
+        if (ast.getFlavor().backreferencesToUnmatchedGroupsFail() && !backReference.isNestedBackReference()) {
             /*
              * Calculate the back-reference's min and max path by checking the referenced group's
              * min and max width. This is useful only if
@@ -339,14 +340,14 @@ protected void leave(Group group) {
                  * summed up with min and max path of the group, so sequence.minPath - group.minPath
                  * is the sequence's "own" minPath
                  */
-                minPath = group.getMinPath() + ((minPath - group.getMinPath()) * (group.isOptionalQuantifier() ? 0 : group.getQuantifier().getMin()));
+                minPath = MathUtil.saturatingAdd(group.getMinPath(), MathUtil.saturatingMul(minPath - group.getMinPath(), group.isOptionalQuantifier() ? 0 : group.getQuantifier().getMin()));
                 if (group.getQuantifier().isInfiniteLoop()) {
                     flags |= RegexASTNode.FLAG_HAS_LOOPS;
                     // Just increase maxPath by one loop iteration; It's enough to determine
                     // whether a given sub-expression is fixed-width.
-                    maxPath = group.getMaxPath() + ((maxPath - group.getMaxPath()) * (group.getQuantifier().getMin() + 1));
+                    maxPath = MathUtil.saturatingAdd(group.getMaxPath(), MathUtil.saturatingMul(maxPath - group.getMaxPath(), MathUtil.saturatingInc(group.getQuantifier().getMin())));
                 } else {
-                    maxPath = group.getMaxPath() + ((maxPath - group.getMaxPath()) * group.getQuantifier().getMax());
+                    maxPath = MathUtil.saturatingAdd(group.getMaxPath(), MathUtil.saturatingMul(maxPath - group.getMaxPath(), group.getQuantifier().getMax()));
                 }
             }
         }
@@ -474,8 +475,8 @@ protected void leave(LookBehindAssertion assertion) {
                 if (laParent instanceof LookBehindAssertion) {
                     ast.getProperties().setNestedLookBehindAssertions();
                 }
-                minPath += laParent.getMinPath();
-                maxPath += laParent.getMaxPath();
+                minPath = MathUtil.saturatingAdd(minPath, laParent.getMinPath());
+                maxPath = MathUtil.saturatingAdd(maxPath, laParent.getMaxPath());
                 laParent = laParent.getSubTreeParent();
             }
             if (assertion.isLiteral()) {

regex/src/com.oracle.truffle.regex/src/com/oracle/truffle/regex/tregex/parser/ast/RegexASTNode.java

@@ -44,6 +44,7 @@
 import com.oracle.truffle.regex.tregex.buffer.CompilationBuffer;
 import com.oracle.truffle.regex.tregex.parser.ast.visitors.CopyVisitor;
 import com.oracle.truffle.regex.tregex.parser.ast.visitors.MarkLookBehindEntriesVisitor;
+import com.oracle.truffle.regex.tregex.util.MathUtil;
 import com.oracle.truffle.regex.tregex.util.json.Json;
 import com.oracle.truffle.regex.tregex.util.json.JsonConvertible;
 import com.oracle.truffle.regex.tregex.util.json.JsonObject;
@@ -501,6 +502,7 @@ public int getMinPath() {
     }
 
     public void setMinPath(int n) {
+        assert n >= 0;
         minPath = n;
     }
 
@@ -509,14 +511,15 @@ public void incMinPath() {
     }
 
     public void incMinPath(int n) {
-        minPath += n;
+        minPath = MathUtil.saturatingAdd(minPath, n);
     }
 
     public int getMaxPath() {
         return maxPath;
     }
 
     public void setMaxPath(int n) {
+        assert n >= 0;
         maxPath = n;
     }
 
@@ -525,7 +528,7 @@ public void incMaxPath() {
     }
 
     public void incMaxPath(int n) {
-        maxPath += n;
+        maxPath = MathUtil.saturatingAdd(maxPath, n);
     }
 
     public int getPrefixLengthMin() {

regex/src/com.oracle.truffle.regex/src/com/oracle/truffle/regex/tregex/util/MathUtil.java

@@ -57,4 +57,17 @@ public static int saturatingInc(int x) {
         }
         return x + 1;
     }
+
+    public static int saturatingAdd(int x, int y) {
+        int sum = x + y;
+        return sum < 0 ? Integer.MAX_VALUE : sum;
+    }
+
+    public static int saturatingMul(int x, int y) {
+        long r = (long) x * (long) y;
+        if ((int) r != r) {
+            return Integer.MAX_VALUE;
+        }
+        return (int) r;
+    }
 }