Catch-and-rethrow TooComplexToDeterminizeException within ESQL (#137024)

MattAlp · elasticsearchmachine · web-flow · commit c0342b01078f · 2025-10-26T12:03:37.000-04:00
---------

Co-authored-by: elasticsearchmachine &lt;infra-root+elasticsearchmachine@elastic.co&gt;
diff --git a/docs/changelog/137024.yaml b/docs/changelog/137024.yaml
@@ -0,0 +1,5 @@
+pr: 137024
+summary: Catch-and-rethrow `TooComplexToDeterminizeException` within ESQL
+area: ES|QL
+type: bug
+issues: []
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java
@@ -11,12 +11,21 @@
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
 
 public abstract class AbstractStringPattern implements StringPattern {
 
     private Automaton automaton;
 
-    public abstract Automaton createAutomaton(boolean ignoreCase);
+    public final Automaton createAutomaton(boolean ignoreCase) {
+        try {
+            return doCreateAutomaton(ignoreCase);
+        } catch (TooComplexToDeterminizeException e) {
+            throw new IllegalArgumentException("Pattern was too complex to determinize", e);
+        }
+    }
+
+    protected abstract Automaton doCreateAutomaton(boolean ignoreCase);
 
     private Automaton automaton() {
         if (automaton == null) {
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java
@@ -34,7 +34,7 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public Automaton createAutomaton(boolean ignoreCase) {
+    protected Automaton doCreateAutomaton(boolean ignoreCase) {
         int matchFlags = ignoreCase ? RegExp.CASE_INSENSITIVE : 0;
         return Operations.determinize(
             new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, matchFlags).toAutomaton(),
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePatternList.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePatternList.java
@@ -43,7 +43,7 @@ public List<RLikePattern> patternList() {
      * We create a single automaton that is the union of all individual automatons to improve performance
      */
     @Override
-    public Automaton createAutomaton(boolean ignoreCase) {
+    protected Automaton doCreateAutomaton(boolean ignoreCase) {
         List<Automaton> automatonList = patternList.stream().map(x -> x.createAutomaton(ignoreCase)).toList();
         Automaton result = Operations.union(automatonList);
         return Operations.determinize(result, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java
@@ -53,7 +53,7 @@ public String pattern() {
     }
 
     @Override
-    public Automaton createAutomaton(boolean ignoreCase) {
+    protected Automaton doCreateAutomaton(boolean ignoreCase) {
         return ignoreCase
             ? Operations.determinize(
                 new RegExp(luceneWildcardToRegExp(wildcard), RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, RegExp.CASE_INSENSITIVE)
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPatternList.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPatternList.java
@@ -51,7 +51,7 @@ public List<WildcardPattern> patternList() {
      * We create a single automaton that is the union of all individual automata to improve performance
      */
     @Override
-    public Automaton createAutomaton(boolean ignoreCase) {
+    protected Automaton doCreateAutomaton(boolean ignoreCase) {
         List<Automaton> automatonList = patternList.stream().map(x -> x.createAutomaton(ignoreCase)).toList();
         Automaton result = Operations.union(automatonList);
         return Operations.determinize(result, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/StringPatternTests.java
@@ -106,4 +106,12 @@ public void testRegexExactMatchWithEmptyMatch() {
         assertNotNull(exactMatchRLike("#|foo"));
         assertNotNull(exactMatchRLike("foo|#"));
     }
+
+    public void testTooComplexPattern() {
+        var e = expectThrows(IllegalArgumentException.class, () -> rlike("(a|b)*a(a|b){13}").createAutomaton(false));
+        assertEquals("Pattern was too complex to determinize", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class, () -> like("*a?????????????").createAutomaton(false));
+        assertEquals("Pattern was too complex to determinize", e.getMessage());
+    }
 }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -1579,6 +1579,11 @@ public enum Cap {
          */
         FORBID_LIMIT_BEFORE_INLINE_STATS(INLINE_STATS.enabled),
 
+        /**
+         * Catch-and-rethrow determinization complexity errors as 400s rather than 500s
+         */
+        HANDLE_DETERMINIZATION_COMPLEXITY,
+
         /**
          * Support for the TRANGE function
          */
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/AutomataMatch.java
@@ -11,6 +11,7 @@
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.ByteRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
 import org.apache.lucene.util.automaton.Transition;
 import org.apache.lucene.util.automaton.UTF32ToUTF8;
 import org.elasticsearch.compute.ann.Evaluator;
@@ -36,7 +37,13 @@ public static EvalOperator.ExpressionEvaluator.Factory toEvaluator(
          * ByteRunAutomaton has a way to convert utf32 to utf8, but if we used it
          * we couldn’t get a nice toDot - so we call UTF32ToUTF8 ourselves.
          */
-        Automaton automaton = Operations.determinize(new UTF32ToUTF8().convert(utf32Automaton), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        Automaton automaton;
+        try {
+            automaton = Operations.determinize(new UTF32ToUTF8().convert(utf32Automaton), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        } catch (TooComplexToDeterminizeException e) {
+            throw new IllegalArgumentException("Pattern was too complex to determinize", e);
+        }
+
         ByteRunAutomaton run = new ByteRunAutomaton(automaton, true);
         return new AutomataMatchEvaluator.Factory(source, field, run, toDot(automaton));
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
@@ -15,6 +15,7 @@
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
+import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.regex.Regex;
@@ -485,12 +486,18 @@ public NamedExpression visitQualifiedNamePattern(EsqlBaseParser.QualifiedNamePat
                 list.add(o instanceof Automaton a ? a : Automata.makeString(o.toString()));
             }
             // use the fast run variant
-            result = new UnresolvedNamePattern(
-                src,
-                new CharacterRunAutomaton(Operations.determinize(Operations.concatenate(list), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)),
-                patternString.toString(),
-                nameString.toString()
-            );
+            try {
+                result = new UnresolvedNamePattern(
+                    src,
+                    new CharacterRunAutomaton(
+                        Operations.determinize(Operations.concatenate(list), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT)
+                    ),
+                    patternString.toString(),
+                    nameString.toString()
+                );
+            } catch (TooComplexToDeterminizeException e) {
+                throw new ParsingException("Pattern was too complex to determinize", e);
+            }
 
             if (qualifier != null) {
                 throw qualifiersUnsupportedInPatterns(src, qualifiedCtx.getText());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
@@ -1368,6 +1368,13 @@ public void testLikeRLike() {
         );
     }
 
+    public void testIdentifierPatternTooComplex() {
+        // It is incredibly unlikely that we will see this limit hit in practice
+        // The repetition value 2450 was a ballpark estimate and validated experimentally
+        String explodingWildcard = "a*".repeat(2450);
+        expectError("FROM a | KEEP " + explodingWildcard, "Pattern was too complex to determinize");
+    }
+
     public void testEnrich() {
         assertEqualsIgnoringIds(
             new Enrich(
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml
@@ -616,3 +616,32 @@ setup:
       esql.query:
         body:
           query: 'FROM test | EVAL tag = name::text | KEEP name'
+---
+"like with overly complex pattern":
+  - requires:
+        capabilities:
+          - method: POST
+            path: /_query
+            parameters: [ method, path, parameters, capabilities ]
+            capabilities: [ handle_determinization_complexity ]
+        reason: "determinization errors not yet rethrown"
+  - do:
+      catch: /Pattern was too complex to determinize/
+      esql.query:
+        body:
+          query: 'FROM test | WHERE job LIKE "*a?????????????"'
+
+---
+"rlike with overly complex pattern":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [ method, path, parameters, capabilities ]
+          capabilities: [ handle_determinization_complexity ]
+      reason: "determinization errors not yet rethrown"
+  - do:
+      catch: /Pattern was too complex to determinize/
+      esql.query:
+        body:
+          query: 'FROM test | WHERE job RLIKE "(a|b)*a(a|b){13}"'

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ public void writeTo(StreamOutput out) throws IOException {`
`34`	`34`	`}`
`35`	`35`
`36`	`36`	`@Override`
`37`		`- public Automaton createAutomaton(boolean ignoreCase) {`
	`37`	`+ protected Automaton doCreateAutomaton(boolean ignoreCase) {`
`38`	`38`	`int matchFlags = ignoreCase ? RegExp.CASE_INSENSITIVE : 0;`
`39`	`39`	`return Operations.determinize(`
`40`	`40`	`new RegExp(regexpPattern, RegExp.ALL \| RegExp.DEPRECATED_COMPLEMENT, matchFlags).toAutomaton(),`
Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@ public String pattern() {`
`53`	`53`	`}`
`54`	`54`
`55`	`55`	`@Override`
`56`		`- public Automaton createAutomaton(boolean ignoreCase) {`
	`56`	`+ protected Automaton doCreateAutomaton(boolean ignoreCase) {`
`57`	`57`	`return ignoreCase`
`58`	`58`	`? Operations.determinize(`
`59`	`59`	`new RegExp(luceneWildcardToRegExp(wildcard), RegExp.ALL \| RegExp.DEPRECATED_COMPLEMENT, RegExp.CASE_INSENSITIVE)`
Original file line number	Diff line number	Diff line change
`@@ -106,4 +106,12 @@ public void testRegexExactMatchWithEmptyMatch() {`
`106`	`106`	`assertNotNull(exactMatchRLike("#\|foo"));`
`107`	`107`	`assertNotNull(exactMatchRLike("foo\|#"));`
`108`	`108`	`}`
	`109`	`+`
	`110`	`+ public void testTooComplexPattern() {`
	`111`	`+ var e = expectThrows(IllegalArgumentException.class, () -> rlike("(a\|b)*a(a\|b){13}").createAutomaton(false));`
	`112`	`+ assertEquals("Pattern was too complex to determinize", e.getMessage());`
	`113`	`+`
	`114`	`+ e = expectThrows(IllegalArgumentException.class, () -> like("*a?????????????").createAutomaton(false));`
	`115`	`+ assertEquals("Pattern was too complex to determinize", e.getMessage());`
	`116`	`+ }`
`109`	`117`	`}`