Integrate with latest from main, add UTs

Author: julian-elastic

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java

@@ -205,25 +205,24 @@ private Map<String, Object> runEsql(RestEsqlTestCase.RequestObjectBuilder reques
         }
     }
 
-    private <C, V> void assertResultMapForLike(
+    private <C, V> void assertResultMapWithCapabilities(
         boolean includeCCSMetadata,
         Map<String, Object> result,
         C columns,
         V values,
         boolean remoteOnly,
-        boolean requireLikeListCapability
+        List<String> fullResultCapabilities
     ) throws IOException {
-        List<String> requiredCapabilities = new ArrayList<>(List.of("like_on_index_fields"));
-        if (requireLikeListCapability) {
-            requiredCapabilities.add("like_list_on_index_fields");
-        }
         // the feature is completely supported if both local and remote clusters support it
-        boolean isSupported = capabilitiesSupportedNewAndOld(requiredCapabilities);
-
+        // otherwise we expect a partial result, and will not check the data
+        boolean isSupported = capabilitiesSupportedNewAndOld(fullResultCapabilities);
         if (isSupported) {
             assertResultMap(includeCCSMetadata, result, columns, values, remoteOnly);
         } else {
-            logger.info("-->  skipping data check for like index test, cluster does not support like index feature");
+            logger.info(
+                "-->  skipping data check for a test, cluster does not support all of [{}] capabilities",
+                String.join(",", fullResultCapabilities)
+            );
             // just verify that we did not get a partial result
             var clusters = result.get("_clusters");
             var reason = "unexpected partial results" + (clusters != null ? ": _clusters=" + clusters : "");
@@ -526,7 +525,7 @@ public void testLikeIndex() throws Exception {
             """, includeCCSMetadata);
         var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
         var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex));
-        assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
+        assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
     }
 
     public void testLikeIndexLegacySettingNoResults() throws Exception {
@@ -548,7 +547,7 @@ public void testLikeIndexLegacySettingNoResults() throws Exception {
             var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
             // we expect empty result, since the setting is false
             var values = List.of();
-            assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
+            assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
         }
     }
 
@@ -572,7 +571,7 @@ public void testLikeIndexLegacySettingResults() throws Exception {
             var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
             // we expect results, since the setting is false, but there is : in the LIKE query
             var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex));
-            assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
+            assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
         }
     }
 
@@ -586,7 +585,7 @@ public void testNotLikeIndex() throws Exception {
             """, includeCCSMetadata);
         var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
         var values = List.of(List.of(localDocs.size(), localIndex));
-        assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
+        assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
     }
 
     public void testLikeListIndex() throws Exception {
@@ -601,7 +600,14 @@ public void testLikeListIndex() throws Exception {
             """, includeCCSMetadata);
         var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
         var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex));
-        assertResultMapForLike(includeCCSMetadata, result, columns, values, false, true);
+        assertResultMapWithCapabilities(
+            includeCCSMetadata,
+            result,
+            columns,
+            values,
+            false,
+            List.of("like_on_index_fields", "like_list_on_index_fields")
+        );
     }
 
     public void testNotLikeListIndex() throws Exception {
@@ -615,7 +621,14 @@ public void testNotLikeListIndex() throws Exception {
             """, includeCCSMetadata);
         var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
         var values = List.of(List.of(localDocs.size(), localIndex));
-        assertResultMapForLike(includeCCSMetadata, result, columns, values, false, true);
+        assertResultMapWithCapabilities(
+            includeCCSMetadata,
+            result,
+            columns,
+            values,
+            false,
+            List.of("like_on_index_fields", "like_list_on_index_fields")
+        );
     }
 
     public void testNotLikeListKeyword() throws Exception {
@@ -639,7 +652,14 @@ public void testNotLikeListKeyword() throws Exception {
         if (localCount > 0) {
             values.add(List.of(localCount, localIndex));
         }
-        assertResultMapForLike(includeCCSMetadata, result, columns, values, false, true);
+        assertResultMapWithCapabilities(
+            includeCCSMetadata,
+            result,
+            columns,
+            values,
+            false,
+            List.of("like_on_index_fields", "like_list_on_index_fields")
+        );
     }
 
     public void testRLikeIndex() throws Exception {
@@ -652,7 +672,7 @@ public void testRLikeIndex() throws Exception {
             """, includeCCSMetadata);
         var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
         var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex));
-        assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
+        assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
     }
 
     public void testNotRLikeIndex() throws Exception {
@@ -665,7 +685,37 @@ public void testNotRLikeIndex() throws Exception {
             """, includeCCSMetadata);
         var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
         var values = List.of(List.of(localDocs.size(), localIndex));
-        assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false);
+        assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
+    }
+
+    public void testRLikeListIndex() throws Exception {
+        assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("rlike_with_list_of_patterns")));
+        boolean includeCCSMetadata = includeCCSMetadata();
+        Map<String, Object> result = run("""
+            FROM test-local-index,*:test-remote-index METADATA _index
+            | WHERE _index RLIKE (".*remote.*", ".*not-exist.*")
+            | STATS c = COUNT(*) BY _index
+            | SORT _index ASC
+            """, includeCCSMetadata);
+        var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
+        var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex));
+        // we depend on the code in like_on_index_fields to serialize an ExpressionQueryBuilder
+        assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
+    }
+
+    public void testNotRLikeListIndex() throws Exception {
+        assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("rlike_with_list_of_patterns")));
+        boolean includeCCSMetadata = includeCCSMetadata();
+        Map<String, Object> result = run("""
+            FROM test-local-index,*:test-remote-index METADATA _index
+            | WHERE _index NOT RLIKE (".*remote.*", ".*not-exist.*")
+            | STATS c = COUNT(*) BY _index
+            | SORT _index ASC
+            """, includeCCSMetadata);
+        var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword"));
+        var values = List.of(List.of(localDocs.size(), localIndex));
+        // we depend on the code in like_on_index_fields to serialize an ExpressionQueryBuilder
+        assertResultMapWithCapabilities(includeCCSMetadata, result, columns, values, false, List.of("like_on_index_fields"));
     }
 
     private RestClient remoteClusterClient() throws IOException {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -1237,7 +1237,7 @@ public enum Cap {
         /**
          * Support for the RLIKE operator with a list of regexes.
          */
-        RLIKE_WITH_LIST_OF_PATTERNS;
+        RLIKE_WITH_LIST_OF_PATTERNS,
 
         /**
          * FUSE command

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RLike.java

@@ -33,19 +33,27 @@ public class RLike extends RegexMatch<RLikePattern> {
         Use `RLIKE` to filter data based on string patterns using using
         <<regexp-syntax,regular expressions>>. `RLIKE` usually acts on a field placed on
         the left-hand side of the operator, but it can also act on a constant (literal)
-        expression. The right-hand side of the operator represents the pattern or a list of patterns.""", detailedDescription = """
-        Matching special characters (eg. `.`, `*`, `(`...) will require escaping.
-        The escape character is backslash `\\`. Since also backslash is a special character in string literals,
-        it will require further escaping.
+        expression. The right-hand side of the operator represents the pattern.""", detailedDescription = """
+         Matching special characters (eg. `.`, `*`, `(`...) will require escaping.
+         The escape character is backslash `\\`. Since also backslash is a special character in string literals,
+         it will require further escaping.
 
-        <<load-esql-example, file=string tag=rlikeEscapingSingleQuotes>>
+         <<load-esql-example, file=string tag=rlikeEscapingSingleQuotes>>
 
-        <<load-esql-example, file=where-like tag=rlikeListDocExample>>
+         To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"`
 
-        To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"`
+         <<load-esql-example, file=string tag=rlikeEscapingTripleQuotes>>
 
-        <<load-esql-example, file=string tag=rlikeEscapingTripleQuotes>>
-        """, operator = NAME, examples = @Example(file = "docs", tag = "rlike"))
+        ```{applies_to}
+         stack: ga 9.1
+         serverless: ga
+         ```
+         Both a single pattern or a list of patterns are supported. If a list of patterns is provided,
+         the expression will return true if any of the patterns match.
+
+         <<load-esql-example, file=where-like tag=rlikeListDocExample>>
+
+         """, operator = NAME, examples = @Example(file = "docs", tag = "rlike"))
     public RLike(
         Source source,
         @Param(name = "str", type = { "keyword", "text" }, description = "A literal value.") Expression value,

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RLikeList.java

@@ -7,23 +7,29 @@
 
 package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex;
 
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePatternList;
-import org.elasticsearch.xpack.esql.core.querydsl.query.EsqlAutomatonQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.io.stream.ExpressionQuery;
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates;
 import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
 
 import java.io.IOException;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 public class RLikeList extends RegexMatch<RLikePatternList> {
@@ -33,6 +39,30 @@ public class RLikeList extends RegexMatch<RLikePatternList> {
         RLikeList::new
     );
 
+    Supplier<Automaton> automatonSupplier = new Supplier<>() {
+        Automaton cached;
+
+        @Override
+        public Automaton get() {
+            if (cached == null) {
+                cached = pattern().createAutomaton(caseInsensitive());
+            }
+            return cached;
+        }
+    };
+
+    Supplier<CharacterRunAutomaton> characterRunAutomatonSupplier = new Supplier<>() {
+        CharacterRunAutomaton cached;
+
+        @Override
+        public CharacterRunAutomaton get() {
+            if (cached == null) {
+                cached = new CharacterRunAutomaton(automatonSupplier.get());
+            }
+            return cached;
+        }
+    };
+
     /**
      * The documentation for this function is in RLike, and shown to the users as `RLIKE` in the docs.
      */
@@ -97,15 +127,30 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand
     }
 
     private Query translateField(String targetFieldName) {
-        return new EsqlAutomatonQuery(source(), targetFieldName, pattern().createAutomaton(caseInsensitive()), getAutomatonDescription());
+        return new ExpressionQuery(source(), targetFieldName, this);
+    }
+
+    @Override
+    public org.apache.lucene.search.Query asLuceneQuery(
+        MappedFieldType fieldType,
+        MultiTermQuery.RewriteMethod constantScoreRewrite,
+        SearchExecutionContext context
+    ) {
+        return fieldType.automatonQuery(
+            automatonSupplier,
+            characterRunAutomatonSupplier,
+            constantScoreRewrite,
+            context,
+            getLuceneQueryDescription()
+        );
     }
 
     @Override
     protected NodeInfo<? extends Expression> info() {
         return NodeInfo.create(this, RLikeList::new, field(), pattern(), caseInsensitive());
     }
 
-    private String getAutomatonDescription() {
+    private String getLuceneQueryDescription() {
         // we use the information used to create the automaton to describe the query here
         String patternDesc = pattern().patternList().stream().map(RLikePattern::pattern).collect(Collectors.joining("\", \""));
         return "RLIKE(\"" + patternDesc + "\"), caseInsensitive=" + caseInsensitive();

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLikeList.java

@@ -147,7 +147,6 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand
 
     private boolean supportsPushdown(TransportVersion version) {
         return version == null || expressionTransportSupported(version);
-        return new EsqlAutomatonQuery(source(), targetFieldName, pattern().createAutomaton(caseInsensitive()), getAutomatonDescription());
     }
 
     @Override