ESQL: Push more ==s on text fields to lucene

docs/changelog/126641.yaml

@@ -0,0 +1,5 @@
+pr: 126641
+summary: Push more `==`s on text fields to lucene
+area: ES|QL
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -984,6 +984,21 @@ public boolean canUseSyntheticSourceDelegateForQuerying() {
                 && syntheticSourceDelegate.isIndexed();
         }
 
+        /**
+         * Returns true if the delegate sub-field can be used for querying only (ie. isIndexed must be true)
+         */
+        public boolean canUseSyntheticSourceDelegateForQueryingEquality(String str) {
+            if (syntheticSourceDelegate == null
+                // Can't push equality to an index if there isn't an index
+                || syntheticSourceDelegate.isIndexed() == false
+                // ESQL needs docs values to push equality
+                || syntheticSourceDelegate.hasDocValues() == false) {
+                return false;
+            }
+            // Can't push equality if the field we're checking for is so big we'd ignore it.
+            return str.length() < syntheticSourceDelegate.ignoreAbove();
+        }
+
         @Override
         public BlockLoader blockLoader(BlockLoaderContext blContext) {
             if (canUseSyntheticSourceDelegateForLoading()) {

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java

@@ -313,7 +313,7 @@ public void testProfile() throws IOException {
             @SuppressWarnings("unchecked")
             List<Map<String, Object>> operators = (List<Map<String, Object>>) p.get("operators");
             for (Map<String, Object> o : operators) {
-                sig.add(checkOperatorProfile(o));
+                sig.add(checkOperatorProfile(o, "*:*"));
             }
             String description = p.get("description").toString();
             switch (description) {
@@ -411,6 +411,55 @@ public void testProfileParsing() throws IOException {
         }
     }
 
+    public void testPushEqualityOnDefaults() throws IOException {
+        indexTimestampData(1);
+
+        RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | WHERE test == \"value1\"");
+        builder.profile(true);
+        Map<String, Object> result = runEsql(builder);
+        assertResultMap(
+            result,
+            getResultMatcher(result).entry("profile", matchesMap().entry("drivers", instanceOf(List.class))),
+            matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date"))
+                .item(matchesMap().entry("name", "test").entry("type", "text"))
+                .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword"))
+                .item(matchesMap().entry("name", "value").entry("type", "long")),
+            equalTo(List.of(List.of("2020-12-12T00:00:00.000Z", "value1", "value1", 1)))
+        );
+
+        @SuppressWarnings("unchecked")
+        List<Map<String, Object>> profiles = (List<Map<String, Object>>) ((Map<String, Object>) result.get("profile")).get("drivers");
+        for (Map<String, Object> p : profiles) {
+            fixTypesOnProfile(p);
+            assertThat(p, commonProfile());
+            List<String> sig = new ArrayList<>();
+            @SuppressWarnings("unchecked")
+            List<Map<String, Object>> operators = (List<Map<String, Object>>) p.get("operators");
+            for (Map<String, Object> o : operators) {
+                // The query here is the most important bit - we *do* push to lucene.
+                sig.add(checkOperatorProfile(o, "test.keyword:value1"));
+            }
+            String description = p.get("description").toString();
+            switch (description) {
+                case "data" -> assertMap(
+                    sig,
+                    matchesList().item("LuceneSourceOperator")
+                        .item("ValuesSourceReaderOperator")
+                        .item("ProjectOperator")
+                        .item("ExchangeSinkOperator")
+                );
+                case "node_reduce" -> assertThat(
+                    sig,
+                    either(matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator")).or(
+                        matchesList().item("ExchangeSourceOperator").item("AggregationOperator").item("ExchangeSinkOperator")
+                    )
+                );
+                case "final" -> assertMap(sig, matchesList().item("ExchangeSourceOperator").item("LimitOperator").item("OutputOperator"));
+                default -> throw new IllegalArgumentException("can't match " + description);
+            }
+        }
+    }
+
     @SuppressWarnings("unchecked")
     public void assertProcessMetadataForNextNode(Map<String, Object> nodeMetadata, Set<String> expectedNamesForNodes, int seenNodes) {
         assertEquals("M", nodeMetadata.get("ph"));
@@ -521,7 +570,7 @@ public void testInlineStatsProfile() throws IOException {
             @SuppressWarnings("unchecked")
             List<Map<String, Object>> operators = (List<Map<String, Object>>) p.get("operators");
             for (Map<String, Object> o : operators) {
-                sig.add(checkOperatorProfile(o));
+                sig.add(checkOperatorProfile(o, "*:*"));
             }
             signatures.add(sig);
         }
@@ -673,7 +722,7 @@ private void fixTypesOnProfile(Map<String, Object> profile) {
         profile.put("took_nanos", ((Number) profile.get("took_nanos")).longValue());
     }
 
-    private String checkOperatorProfile(Map<String, Object> o) {
+    private String checkOperatorProfile(Map<String, Object> o, String query) {
         String name = (String) o.get("operator");
         name = name.replaceAll("\\[.+", "");
         MapMatcher status = switch (name) {
@@ -687,7 +736,7 @@ private String checkOperatorProfile(Map<String, Object> o) {
                 .entry("pages_emitted", greaterThan(0))
                 .entry("rows_emitted", greaterThan(0))
                 .entry("process_nanos", greaterThan(0))
-                .entry("processed_queries", List.of("*:*"));
+                .entry("processed_queries", List.of(query));
             case "ValuesSourceReaderOperator" -> basicProfile().entry("readers_built", matchesMap().extraOk());
             case "AggregationOperator" -> matchesMap().entry("pages_processed", greaterThan(0))
                 .entry("rows_received", greaterThan(0))

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java

@@ -293,6 +293,11 @@ public byte[] max(String field, DataType dataType) {
         public boolean isSingleValue(String field) {
             return false;
         }
+
+        @Override
+        public boolean canUseEqualityOnSyntheticSourceDelegate(String name, String value) {
+            return false;
+        }
     }
 
     /**

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic-limited-raw.json

@@ -0,0 +1,42 @@
+{
+    "properties" : {
+        "emp_no" : {
+            "type" : "integer"
+        },
+        "first_name" : {
+            "type" : "keyword"
+        },
+        "gender" : {
+            "type" : "text"
+        },
+        "languages" : {
+            "type" : "byte"
+        },
+        "last_name" : {
+            "type" : "keyword"
+        },
+        "salary" : {
+            "type" : "integer"
+        },
+        "_meta_field": {
+            "type" : "keyword"
+        },
+        "hire_date": {
+          "type": "date"
+        },
+        "job": {
+            "type": "text",
+            "fields": {
+                "raw": {
+                  "type": "keyword",
+                  "ignore_above": 4
+                }
+            }
+        },
+        "long_noidx": {
+          "type": "long",
+          "index": false,
+          "doc_values": false
+        }
+    }
+}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java

@@ -31,7 +31,7 @@ public interface TranslationAware {
      * <p>and <b>not</b> this:</p>
      * <p>{@code Query childQuery = child.asQuery(handler);}</p>
      */
-    Query asQuery(TranslatorHandler handler);
+    Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler);
 
     /**
      * Subinterface for expressions that can only process single values (and null out on MVs).

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java

@@ -153,7 +153,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         return queryBuilder != null ? new TranslationAwareExpressionQuery(source(), queryBuilder) : translate(handler);
     }
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.Rewriteable;
 import org.elasticsearch.xpack.esql.core.util.Holder;
+import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
@@ -76,7 +77,9 @@ public FullTextFunctionsRewritable rewrite(QueryRewriteContext ctx) throws IOExc
             Holder<Boolean> updated = new Holder<>(false);
             LogicalPlan newPlan = plan.transformExpressionsDown(FullTextFunction.class, f -> {
                 QueryBuilder builder = f.queryBuilder(), initial = builder;
-                builder = builder == null ? f.asQuery(TranslatorHandler.TRANSLATOR_HANDLER).toQueryBuilder() : builder;
+                builder = builder == null
+                    ? f.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER).toQueryBuilder()
+                    : builder;
                 try {
                     builder = builder.rewrite(ctx);
                 } catch (IOException e) {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java

@@ -184,7 +184,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         var fa = LucenePushdownPredicates.checkIsFieldAttribute(ipField);
         Check.isTrue(Expressions.foldable(matches), "Expected foldable matches, but got [{}]", matches);
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java

@@ -183,7 +183,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         if (left().foldable()) {
             checkSpatialRelatesFunction(left(), queryRelation());
             return translate(handler, right(), left());

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java

@@ -144,7 +144,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         LucenePushdownPredicates.checkIsPushableAttribute(str);
         var fieldName = handler.nameOf(str instanceof FieldAttribute fa ? fa.exactAttribute() : str);
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java

@@ -112,7 +112,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         var fa = LucenePushdownPredicates.checkIsFieldAttribute(field());
         // TODO: see whether escaping is needed
         return new RegexQuery(source(), handler.nameOf(fa.exactAttribute()), pattern().asJavaRegex(), caseInsensitive());

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java

@@ -141,7 +141,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         LucenePushdownPredicates.checkIsPushableAttribute(str);
         var fieldName = handler.nameOf(str instanceof FieldAttribute fa ? fa.exactAttribute() : str);
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java

@@ -124,7 +124,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         var field = field();
         LucenePushdownPredicates.checkIsPushableAttribute(field);
         return translateField(handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field));

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java

@@ -220,7 +220,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         return translate(handler);
     }
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQueryPredicate.java

@@ -100,7 +100,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         return new MultiMatchQuery(source(), query(), fields(), this);
     }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java

@@ -90,8 +90,13 @@ && right() instanceof TranslationAware rightAware
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
-        return boolQuery(source(), handler.asQuery(left()), handler.asQuery(right()), this instanceof And);
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
+        return boolQuery(
+            source(),
+            handler.asQuery(pushdownPredicates, left()),
+            handler.asQuery(pushdownPredicates, right()),
+            this instanceof And
+        );
     }
 
     public static Query boolQuery(Source source, Query left, Query right, boolean isAnd) {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java

@@ -105,7 +105,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
-        return handler.asQuery(field()).negate(source());
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
+        return handler.asQuery(pushdownPredicates, field()).negate(source());
     }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java

@@ -80,7 +80,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         return new ExistsQuery(source(), handler.nameOf(field()));
     }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java

@@ -81,7 +81,7 @@ protected static boolean isTranslatable(Expression field, LucenePushdownPredicat
     }
 
     @Override
-    public Query asQuery(TranslatorHandler handler) {
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
         return new NotQuery(source(), new ExistsQuery(source(), handler.nameOf(field())));
     }
 }

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java

@@ -11,13 +11,18 @@
 import org.elasticsearch.common.time.DateUtils;
 import org.elasticsearch.compute.ann.Evaluator;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable;
+import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
+import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates;
+import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
 
 import java.time.ZoneId;
 import java.util.Map;
@@ -120,6 +125,31 @@ public Equals(Source source, Expression left, Expression right, ZoneId zoneId) {
         );
     }
 
+    @Override
+    public boolean translatable(LucenePushdownPredicates pushdownPredicates) {
+        if (right() instanceof Literal lit) {
+            if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
+                if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, ((BytesRef) lit.value()).utf8ToString())) {
+                    return true;
+                }
+            }
+        }
+        return super.translatable(pushdownPredicates);
+    }
+
+    @Override
+    public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
+        if (right() instanceof Literal lit) {
+            if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) {
+                String value = ((BytesRef) lit.value()).utf8ToString();
+                if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, value)) {
+                    return new EqualsSyntheticSourceDelegate(source(), handler.nameOf(fa), value);
+                }
+            }
+        }
+        return super.asQuery(pushdownPredicates, handler);
+    }
+
     @Override
     public String getWriteableName() {
         return ENTRY.name;