diff --git a/docs/changelog/127355.yaml b/docs/changelog/127355.yaml new file mode 100644 index 0000000000000..28ead562ed8c5 --- /dev/null +++ b/docs/changelog/127355.yaml @@ -0,0 +1,5 @@ +pr: 127355 +summary: '`text ==` and `text !=` pushdown' +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index 272f884addc4c..518b097c6604f 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -7,10 +7,12 @@ package org.elasticsearch.xpack.esql.qa.single_node; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.TestClustersThreadFilter; @@ -27,6 +29,7 @@ import java.util.Locale; import java.util.Map; import java.util.regex.Pattern; +import java.util.stream.Stream; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; @@ -48,50 +51,161 @@ public class PushQueriesIT extends ESRestTestCase { @ClassRule public static ElasticsearchCluster cluster = Clusters.testCluster(); - public void testPushEqualityOnDefaults() throws IOException { + @ParametersFactory(argumentFormatting = "%1s") + public static List args() { + return Stream.of("auto", "text", "match_only_text", "semantic_text").map(s -> new Object[] { s }).toList(); + } + + private final String type; + + public PushQueriesIT(String type) { + this.type = type; + } + + public void testEquality() throws IOException { String value = "v".repeat(between(0, 256)); - testPushQuery(value, """ + String esqlQuery = """ FROM test | WHERE test == "%value" - """, "*:*", true, true); + """; + String luceneQuery = switch (type) { + case "text", "auto" -> "#test.keyword:%value -_ignored:test.keyword"; + case "match_only_text" -> "*:*"; + case "semantic_text" -> "FieldExistsQuery [field=_primary_term]"; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + boolean filterInCompute = switch (type) { + case "text", "auto" -> false; + case "match_only_text", "semantic_text" -> true; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + testPushQuery(value, esqlQuery, luceneQuery, filterInCompute, true); } - public void testPushEqualityOnDefaultsTooBigToPush() throws IOException { + public void testEqualityTooBigToPush() throws IOException { String value = "a".repeat(between(257, 1000)); - testPushQuery(value, """ + String esqlQuery = """ FROM test | WHERE test == "%value" - """, "*:*", true, true); + """; + String luceneQuery = switch (type) { + case "text", "auto", "match_only_text" -> "*:*"; + case "semantic_text" -> "FieldExistsQuery [field=_primary_term]"; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + testPushQuery(value, esqlQuery, luceneQuery, true, true); + } + + /** + * Turns into an {@code IN} which isn't currently pushed. + */ + public void testEqualityOrTooBig() throws IOException { + String value = "v".repeat(between(0, 256)); + String tooBig = "a".repeat(between(257, 1000)); + String esqlQuery = """ + FROM test + | WHERE test == "%value" OR test == "%tooBig" + """.replace("%tooBig", tooBig); + String luceneQuery = switch (type) { + case "text", "auto", "match_only_text" -> "*:*"; + case "semantic_text" -> "FieldExistsQuery [field=_primary_term]"; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + testPushQuery(value, esqlQuery, luceneQuery, true, true); + } + + public void testEqualityOrOther() throws IOException { + String value = "v".repeat(between(0, 256)); + String esqlQuery = """ + FROM test + | WHERE test == "%value" OR foo == 2 + """; + String luceneQuery = switch (type) { + case "text", "auto" -> "(#test.keyword:%value -_ignored:test.keyword) foo:[2 TO 2]"; + case "match_only_text" -> "*:*"; + case "semantic_text" -> "FieldExistsQuery [field=_primary_term]"; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + boolean filterInCompute = switch (type) { + case "text", "auto" -> false; + case "match_only_text", "semantic_text" -> true; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + testPushQuery(value, esqlQuery, luceneQuery, filterInCompute, true); } - public void testPushInequalityOnDefaults() throws IOException { + public void testEqualityAndOther() throws IOException { String value = "v".repeat(between(0, 256)); - testPushQuery(value, """ + String esqlQuery = """ + FROM test + | WHERE test == "%value" AND foo == 1 + """; + String luceneQuery = switch (type) { + case "text", "auto" -> "#test.keyword:%value -_ignored:test.keyword #foo:[1 TO 1]"; + case "match_only_text" -> "foo:[1 TO 1]"; + case "semantic_text" -> + /* + * single_value_match is here because there are extra documents hiding in the index + * that don't have the `foo` field. + */ + "#foo:[1 TO 1] #single_value_match(foo)"; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + boolean filterInCompute = switch (type) { + case "text", "auto" -> false; + case "match_only_text", "semantic_text" -> true; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + testPushQuery(value, esqlQuery, luceneQuery, filterInCompute, true); + } + + public void testInequality() throws IOException { + String value = "v".repeat(between(0, 256)); + String esqlQuery = """ FROM test | WHERE test != "%different_value" - """, "*:*", true, true); + """; + String luceneQuery = switch (type) { + case "text", "auto" -> "(-test.keyword:%different_value #*:*) _ignored:test.keyword"; + case "match_only_text" -> "*:*"; + case "semantic_text" -> "FieldExistsQuery [field=_primary_term]"; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + testPushQuery(value, esqlQuery, luceneQuery, true, true); } - public void testPushInequalityOnDefaultsTooBigToPush() throws IOException { + public void testInequalityTooBigToPush() throws IOException { String value = "a".repeat(between(257, 1000)); - testPushQuery(value, """ + String esqlQuery = """ FROM test | WHERE test != "%value" - """, "*:*", true, false); + """; + String luceneQuery = switch (type) { + case "text", "auto", "match_only_text" -> "*:*"; + case "semantic_text" -> "FieldExistsQuery [field=_primary_term]"; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + testPushQuery(value, esqlQuery, luceneQuery, true, false); } - public void testPushCaseInsensitiveEqualityOnDefaults() throws IOException { + public void testCaseInsensitiveEquality() throws IOException { String value = "a".repeat(between(0, 256)); - testPushQuery(value, """ + String esqlQuery = """ FROM test | WHERE TO_LOWER(test) == "%value" - """, "*:*", true, true); + """; + String luceneQuery = switch (type) { + case "text", "auto", "match_only_text" -> "*:*"; + case "semantic_text" -> "FieldExistsQuery [field=_primary_term]"; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + testPushQuery(value, esqlQuery, luceneQuery, true, true); } private void testPushQuery(String value, String esqlQuery, String luceneQuery, boolean filterInCompute, boolean found) throws IOException { indexValue(value); - String differentValue = randomValueOtherThan(value, () -> randomAlphaOfLength(value.length() == 0 ? 1 : value.length())); + String differentValue = randomValueOtherThan(value, () -> randomAlphaOfLength(value.isEmpty() ? 1 : value.length())); String replacedQuery = esqlQuery.replaceAll("%value", value).replaceAll("%different_value", differentValue); RestEsqlTestCase.RequestObjectBuilder builder = requestObjectBuilder().query(replacedQuery + "\n| KEEP test"); @@ -148,15 +262,43 @@ private void testPushQuery(String value, String esqlQuery, String luceneQuery, b } private void indexValue(String value) throws IOException { + try { + // Delete the index if it has already been created. + client().performRequest(new Request("DELETE", "test")); + } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() != 404) { + throw e; + } + } + Request createIndex = new Request("PUT", "test"); - createIndex.setJsonEntity(""" + String json = """ { "settings": { "index": { "number_of_shards": 1 } - } - }"""); + }"""; + if (false == "auto".equals(type)) { + json += """ + , + "mappings": { + "properties": { + "test": { + "type": "%type", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + }""".replace("%type", type); + } + json += "}"; + createIndex.setJsonEntity(json); Response createResponse = client().performRequest(createIndex); assertThat( entityToMap(createResponse.getEntity(), XContentType.JSON), @@ -167,7 +309,7 @@ private void indexValue(String value) throws IOException { bulk.addParameter("refresh", ""); bulk.setJsonEntity(String.format(Locale.ROOT, """ {"create":{"_index":"test"}} - {"test":"%s"} + {"test":"%s","foo":1} """, value)); Response bulkResponse = client().performRequest(bulk); assertThat(entityToMap(bulkResponse.getEntity(), XContentType.JSON), matchesMap().entry("errors", false).extraOk()); @@ -190,4 +332,10 @@ private static String checkOperatorProfile(Map o, String query) protected String getTestRestCluster() { return cluster.getHttpAddresses(); } + + @Override + protected boolean preserveClusterUponCompletion() { + // Preserve the cluser to speed up the semantic_text tests + return true; + } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv index 222cc42d27cde..ee3873c339ada 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv @@ -3,3 +3,4 @@ 2023-10-23T13:55:01.544Z,Connected to 10.1.0.1 2023-10-23T13:55:01.545Z,[Connected to 10.1.0.1, More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100] 2023-10-23T13:55:01.546Z,More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100 +2023-10-23T13:55:01.547Z,[More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100,Second than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java index 893ed26a1ab42..1a2fd81db4b6c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java @@ -7,20 +7,34 @@ package org.elasticsearch.xpack.esql.capabilities; +import org.elasticsearch.compute.lucene.LuceneTopNSourceOperator; +import org.elasticsearch.compute.operator.FilterOperator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.planner.TranslatorHandler; /** - * Expressions implementing this interface can get called on data nodes to provide an Elasticsearch/Lucene query. + * Expressions implementing this interface are asked provide an + * Elasticsearch/Lucene query as part of the data node optimizations. */ public interface TranslationAware { /** - * Indicates whether the expression can be translated or not. - * Usually checks whether the expression arguments are actual fields that exist in Lucene. + * Can this instance be translated or not? Usually checks whether the + * expression arguments are actual fields that exist in Lucene. See {@link Translatable} + * for precisely what can be signaled from this method. */ - boolean translatable(LucenePushdownPredicates pushdownPredicates); + Translatable translatable(LucenePushdownPredicates pushdownPredicates); + + /** + * Is an {@link Expression} translatable? + */ + static TranslationAware.Translatable translatable(Expression exp, LucenePushdownPredicates lucenePushdownPredicates) { + if (exp instanceof TranslationAware aware) { + return aware.translatable(lucenePushdownPredicates); + } + return TranslationAware.Translatable.NO; + } /** * Translates the implementing expression into a Query. @@ -42,4 +56,112 @@ interface SingleValueTranslationAware extends TranslationAware { */ Expression singleValueField(); } + + /** + * How is this expression translatable? + */ + enum Translatable { + /** + * Not translatable at all. Calling {@link TranslationAware#asQuery} is an error. + * The expression will stay in the query plan and be filtered via a {@link FilterOperator}. + * Imagine {@code kwd == "a"} when {@code kwd} is configured without a search index. + */ + NO(FinishedTranslatable.NO), + /** + * Entirely translatable into a lucene query. Calling {@link TranslationAware#asQuery} + * will produce a query that matches all documents matching this expression and + * only documents matching this expression. Imagine {@code kwd == "a"} + * when {@code kwd} has a search index and doc values - which is the + * default configuration. This will entirely remove the clause from the + * {@code WHERE}, removing the entire {@link FilterOperator} if it's empty. Sometimes + * this allows us to push the entire top-n operation to lucene with + * a {@link LuceneTopNSourceOperator}. + */ + YES(FinishedTranslatable.YES), + /** + * Translation requires a recheck. Calling {@link TranslationAware#asQuery} will + * produce a query that matches all documents matching this expression but might + * match more documents that do not match the expression. This will cause us to + * push a query to lucene and keep the query in the query plan, + * rechecking it via a {@link FilterOperator}. This can never push the entire + * top-n to Lucene, but it's still quite a lot better than the full scan from + * {@link #NO}. + *

+ * Imagine {@code kwd == "a"} where {@code kwd} has a search index but doesn't + * have doc values. In that case we can find candidate matches in lucene but + * can't tell if those docs are single-valued. If they are multivalued they'll + * still match the query but won't match the expression. Thus, the double-checking. + * Technically we could just check for single-valued-ness in + * this case, but it's simpler to + *

+ */ + RECHECK(FinishedTranslatable.RECHECK), + /** + * The same as {@link #YES}, but if this expression is negated it turns into {@link #RECHECK}. + * This comes up when pushing {@code NOT(text == "a")} to {@code text.keyword} which can + * have ignored fields. + */ + YES_BUT_RECHECK_NEGATED(FinishedTranslatable.YES); + + private final FinishedTranslatable finish; + + Translatable(FinishedTranslatable finish) { + this.finish = finish; + } + + /** + * Translate into a {@link FinishedTranslatable} which never + * includes {@link #YES_BUT_RECHECK_NEGATED}. + */ + public FinishedTranslatable finish() { + return finish; + } + + public Translatable negate() { + if (this == YES_BUT_RECHECK_NEGATED) { + return RECHECK; + } + return this; + } + + /** + * Merge two {@link TranslationAware#translatable} results. + */ + public Translatable merge(Translatable rhs) { + return switch (this) { + case NO -> NO; + case YES -> switch (rhs) { + case NO -> NO; + case YES -> YES; + case RECHECK -> RECHECK; + case YES_BUT_RECHECK_NEGATED -> YES_BUT_RECHECK_NEGATED; + }; + case RECHECK -> switch (rhs) { + case NO -> NO; + case YES, RECHECK, YES_BUT_RECHECK_NEGATED -> RECHECK; + }; + case YES_BUT_RECHECK_NEGATED -> switch (rhs) { + case NO -> NO; + case YES, YES_BUT_RECHECK_NEGATED -> YES_BUT_RECHECK_NEGATED; + case RECHECK -> RECHECK; + }; + }; + } + + } + + enum FinishedTranslatable { + /** + * See {@link Translatable#YES}. + */ + YES, + /** + * See {@link Translatable#NO}. + */ + NO, + /** + * See {@link Translatable#RECHECK}. + */ + RECHECK; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 2d86d7a604b36..8fdb305804c04 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -156,9 +156,9 @@ public boolean equals(Object obj) { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { // In isolation, full text functions are pushable to source. We check if there are no disjunctions in Or conditions - return true; + return Translatable.YES; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java index 7ffae77483882..547022e0fbde5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java @@ -179,8 +179,8 @@ protected NodeInfo info() { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { - return pushdownPredicates.isPushableFieldAttribute(ipField) && Expressions.foldable(matches); + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableFieldAttribute(ipField) && Expressions.foldable(matches) ? Translatable.YES : Translatable.NO; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java index cfb317234b1d1..437973705f343 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/BinarySpatialFunction.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.lucene.spatial.CoordinateEncoder; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -273,12 +274,14 @@ protected Geometry fromBytesRef(BytesRef bytesRef) { /** * Push-down to Lucene is only possible if one field is an indexed spatial field, and the other is a constant spatial or string column. */ - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { + public TranslationAware.Translatable translatable(LucenePushdownPredicates pushdownPredicates) { // The use of foldable here instead of SpatialEvaluatorFieldKey.isConstant is intentional to match the behavior of the // Lucene pushdown code in EsqlTranslationHandler::SpatialRelatesTranslator // We could enhance both places to support ReferenceAttributes that refer to constants, but that is a larger change return isPushableSpatialAttribute(left(), pushdownPredicates) && right().foldable() - || isPushableSpatialAttribute(right(), pushdownPredicates) && left().foldable(); + || isPushableSpatialAttribute(right(), pushdownPredicates) && left().foldable() + ? TranslationAware.Translatable.YES + : TranslationAware.Translatable.NO; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java index f9164a0768209..4af5230e9d328 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java @@ -181,7 +181,7 @@ protected void processPointDocValuesAndSource( } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { return super.translatable(pushdownPredicates); // only for the explicit Override, as only this subclass implements TranslationAware } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java index 5b8dc911b7c6c..52dedcb670372 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java @@ -139,8 +139,8 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { - return pushdownPredicates.isPushableAttribute(str) && suffix.foldable(); + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableAttribute(str) && suffix.foldable() ? Translatable.YES : Translatable.NO; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java index 0950cc4a3a8ec..43c4705d1c750 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java @@ -107,8 +107,8 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { - return pushdownPredicates.isPushableFieldAttribute(field()); + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableFieldAttribute(field()) ? Translatable.YES : Translatable.NO; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java index 518d920ab4ee2..f457bacf44268 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java @@ -136,8 +136,8 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { - return pushdownPredicates.isPushableAttribute(str) && prefix.foldable(); + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableAttribute(str) && prefix.foldable() ? Translatable.YES : Translatable.NO; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java index 1dd2460ea72a1..3f90f9b70766f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java @@ -119,8 +119,8 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { - return pushdownPredicates.isPushableAttribute(field()); + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableAttribute(field()) ? Translatable.YES : Translatable.NO; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java index 30ea63d0e473d..6427449add799 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java @@ -215,8 +215,8 @@ public boolean equals(Object obj) { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { - return pushdownPredicates.isPushableAttribute(value) && lower.foldable() && upper.foldable(); + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableAttribute(value) && lower.foldable() && upper.foldable() ? Translatable.YES : Translatable.NO; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java index e457d3523e9c4..ebe17d325ccf7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java @@ -82,11 +82,8 @@ protected boolean isCommutative() { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { - return left() instanceof TranslationAware leftAware - && leftAware.translatable(pushdownPredicates) - && right() instanceof TranslationAware rightAware - && rightAware.translatable(pushdownPredicates); + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return TranslationAware.translatable(left(), pushdownPredicates).merge(TranslationAware.translatable(right(), pushdownPredicates)); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java index 4fc43e4c3a9cf..9f14e291fe7a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java @@ -100,8 +100,8 @@ static Expression negate(Expression exp) { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { - return field() instanceof TranslationAware aware && aware.translatable(pushdownPredicates); + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return TranslationAware.translatable(field(), pushdownPredicates).negate(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java index 7b183f44c4aae..5c7492b0a64dd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java @@ -75,7 +75,7 @@ public UnaryScalarFunction negate() { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { return IsNull.isTranslatable(field(), pushdownPredicates); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java index 9e393ddb05e9a..b41b20b673b86 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java @@ -72,12 +72,14 @@ public UnaryScalarFunction negate() { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { return isTranslatable(field(), pushdownPredicates); } - protected static boolean isTranslatable(Expression field, LucenePushdownPredicates pushdownPredicates) { - return LucenePushdownPredicates.isPushableTextFieldAttribute(field) || pushdownPredicates.isPushableFieldAttribute(field); + protected static Translatable isTranslatable(Expression field, LucenePushdownPredicates pushdownPredicates) { + return LucenePushdownPredicates.isPushableTextFieldAttribute(field) || pushdownPredicates.isPushableFieldAttribute(field) + ? Translatable.YES + : Translatable.NO; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index 75f47615bad6e..01f57342da1f0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -128,11 +128,11 @@ public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { if (right() instanceof Literal lit) { - if (false && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) { + if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) { if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, ((BytesRef) lit.value()).utf8ToString())) { - return true; + return Translatable.YES_BUT_RECHECK_NEGATED; } } } @@ -142,8 +142,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { @Override public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { if (right() instanceof Literal lit) { - // Disabled because it cased a bug with !=. Fix incoming shortly. - if (false && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) { + if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) { String value = ((BytesRef) lit.value()).utf8ToString(); if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, value)) { String name = handler.nameOf(fa); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index 542c696fd3521..69ef99ba04d15 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -328,16 +328,16 @@ public static String formatIncompatibleTypesMessage(DataType leftType, DataType } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { if (right().foldable()) { if (pushdownPredicates.isPushableFieldAttribute(left())) { - return true; + return Translatable.YES; } if (LucenePushdownPredicates.isPushableMetadataAttribute(left())) { - return this instanceof Equals || this instanceof NotEquals; + return this instanceof Equals || this instanceof NotEquals ? Translatable.YES : Translatable.NO; } } - return false; + return Translatable.NO; } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index 3f5d20f794e4c..c38d7abd7243e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -461,8 +461,8 @@ static boolean process(BitSet nulls, BitSet mvs, BytesRef lhs, BytesRef[] rhs) { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { - return pushdownPredicates.isPushableAttribute(value) && Expressions.foldable(list()); + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableAttribute(value) && Expressions.foldable(list()) ? Translatable.YES : Translatable.NO; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java index 8eda96236f504..ccd2f5bbbad5b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java @@ -102,8 +102,8 @@ public Boolean fold(FoldContext ctx) { } @Override - public boolean translatable(LucenePushdownPredicates pushdownPredicates) { - return pushdownPredicates.isPushableFieldAttribute(left()) && right().foldable(); + public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + return pushdownPredicates.isPushableFieldAttribute(left()) && right().foldable() ? Translatable.YES : Translatable.NO; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java index 13f18f4b2bf4d..1e976ca2e6263 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java @@ -12,6 +12,7 @@ import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeMap; @@ -42,8 +43,8 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.esql.capabilities.TranslationAware.translatable; import static org.elasticsearch.xpack.esql.expression.predicate.Predicates.splitAnd; -import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushToSource; import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.getAliasReplacedBy; /** @@ -106,7 +107,8 @@ private FilterExec rewrite( } return comparison; }); - if (rewritten.equals(filterExec.condition()) == false && canPushToSource(rewritten, lucenePushdownPredicates)) { + if (rewritten.equals(filterExec.condition()) == false + && translatable(rewritten, lucenePushdownPredicates).finish() == TranslationAware.FinishedTranslatable.YES) { return new FilterExec(filterExec.source(), esQueryExec, rewritten); } return filterExec; @@ -156,7 +158,8 @@ private PhysicalPlan rewriteBySplittingFilter( // Find and rewrite any binary comparisons that involve a distance function and a literal var rewritten = rewriteDistanceFilters(ctx, resExp, distances); // If all pushable StDistance functions were found and re-written, we need to re-write the FILTER/EVAL combination - if (rewritten.equals(resExp) == false && canPushToSource(rewritten, lucenePushdownPredicates)) { + if (rewritten.equals(resExp) == false + && translatable(rewritten, lucenePushdownPredicates).finish() == TranslationAware.FinishedTranslatable.YES) { pushable.add(rewritten); } else { nonPushable.add(exp); @@ -183,7 +186,8 @@ private PhysicalPlan rewriteBySplittingFilter( private Map getPushableDistances(List aliases, LucenePushdownPredicates lucenePushdownPredicates) { Map distances = new LinkedHashMap<>(); aliases.forEach(alias -> { - if (alias.child() instanceof StDistance distance && distance.translatable(lucenePushdownPredicates)) { + if (alias.child() instanceof StDistance distance + && distance.translatable(lucenePushdownPredicates).finish() == TranslationAware.FinishedTranslatable.YES) { distances.put(alias.id(), distance); } else if (alias.child() instanceof ReferenceAttribute ref && distances.containsKey(ref.id())) { StDistance distance = distances.get(ref.id()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 8d0c88d9c399d..1f8341c4768d2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeMap; @@ -36,6 +35,7 @@ import java.util.List; import static java.util.Arrays.asList; +import static org.elasticsearch.xpack.esql.capabilities.TranslationAware.translatable; import static org.elasticsearch.xpack.esql.expression.predicate.Predicates.splitAnd; import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER; @@ -57,7 +57,14 @@ private static PhysicalPlan planFilterExec(FilterExec filterExec, EsQueryExec qu List pushable = new ArrayList<>(); List nonPushable = new ArrayList<>(); for (Expression exp : splitAnd(filterExec.condition())) { - (canPushToSource(exp, pushdownPredicates) ? pushable : nonPushable).add(exp); + switch (translatable(exp, pushdownPredicates).finish()) { + case NO -> nonPushable.add(exp); + case YES -> pushable.add(exp); + case RECHECK -> { + pushable.add(exp); + nonPushable.add(exp); + } + } } return rewrite(pushdownPredicates, filterExec, queryExec, pushable, nonPushable, List.of()); } @@ -74,7 +81,14 @@ private static PhysicalPlan planFilterExec( List nonPushable = new ArrayList<>(); for (Expression exp : splitAnd(filterExec.condition())) { Expression resExp = exp.transformUp(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)); - (canPushToSource(resExp, pushdownPredicates) ? pushable : nonPushable).add(exp); + switch (translatable(resExp, pushdownPredicates).finish()) { + case NO -> nonPushable.add(exp); + case YES -> pushable.add(exp); + case RECHECK -> { + nonPushable.add(exp); + nonPushable.add(exp); + } + } } // Replace field references with their actual field attributes pushable.replaceAll(e -> e.transformDown(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r))); @@ -202,18 +216,4 @@ else if ((other instanceof GreaterThan || other instanceof GreaterThanOrEqual) } return changed ? CollectionUtils.combine(others, bcs, ranges) : pushable; } - - /** - * Check if the given expression can be pushed down to the source. - * This version of the check is called when we do not have SearchStats available. It assumes no exact subfields for TEXT fields, - * and makes the indexed/doc-values check using the isAggregatable flag only, which comes from field-caps, represents the field state - * over the entire cluster (is not node specific), and has risks for indexed=false/doc_values=true fields. - */ - public static boolean canPushToSource(Expression exp) { - return canPushToSource(exp, LucenePushdownPredicates.DEFAULT); - } - - static boolean canPushToSource(Expression exp, LucenePushdownPredicates lucenePushdownPredicates) { - return exp instanceof TranslationAware aware && aware.translatable(lucenePushdownPredicates); - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java index d7abc58bc7300..ed97ec5553fcc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java @@ -10,6 +10,7 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeMap; @@ -33,7 +34,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; -import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushToSource; +import static org.elasticsearch.xpack.esql.capabilities.TranslationAware.translatable; import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType.COUNT; import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER; @@ -107,7 +108,10 @@ private Tuple, List> pushableStats( // That's because stats pushdown only works for 1 agg function (without BY); but in that case, filters // are extracted into a separate filter node upstream from the aggregation (and hopefully pushed into // the EsQueryExec separately). - if (canPushToSource(count.filter()) == false) { + if (translatable( + count.filter(), + LucenePushdownPredicates.DEFAULT + ) != TranslationAware.Translatable.YES) { return null; // can't push down } var countFilter = TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, count.filter()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index f77532285c379..3593b90c9662f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; @@ -64,8 +65,8 @@ import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE; +import static org.elasticsearch.xpack.esql.capabilities.TranslationAware.translatable; import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.FILTER; -import static org.elasticsearch.xpack.esql.optimizer.rules.physical.local.PushFiltersToSource.canPushToSource; import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER; public class PlannerUtils { @@ -245,7 +246,9 @@ static QueryBuilder detectFilter(PhysicalPlan plan, Predicate fieldName) boolean matchesField = refsBuilder.removeIf(e -> fieldName.test(e.name())); // the expression only contains the target reference // and the expression is pushable (functions can be fully translated) - if (matchesField && refsBuilder.isEmpty() && canPushToSource(exp)) { + if (matchesField + && refsBuilder.isEmpty() + && translatable(exp, LucenePushdownPredicates.DEFAULT).finish() == TranslationAware.FinishedTranslatable.YES) { matches.add(exp); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index 3af78787808f0..912af6663a9dc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -17,7 +17,9 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.compute.lucene.LuceneSourceOperator; import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.FilterOperator; import org.elasticsearch.compute.operator.Warnings; import org.elasticsearch.compute.querydsl.query.SingleValueMatchQuery; import org.elasticsearch.index.mapper.IgnoredFieldMapper; @@ -50,6 +52,9 @@ * for now we're going to always wrap so we can always push. When we find cases * where double checking is better we'll try that. *

+ *

+ * NOTE: This will only work with {@code text} fields. + *

*/ public class SingleValueQuery extends Query { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -60,7 +65,7 @@ public class SingleValueQuery extends Query { private final Query next; private final String field; - private final boolean useSyntheticSourceDelegate; + private final UseSyntheticSourceDelegate useSyntheticSourceDelegate; /** * Build. @@ -71,6 +76,10 @@ public class SingleValueQuery extends Query { * we often want to use its delegate. */ public SingleValueQuery(Query next, String field, boolean useSyntheticSourceDelegate) { + this(next, field, useSyntheticSourceDelegate ? UseSyntheticSourceDelegate.YES : UseSyntheticSourceDelegate.NO); + } + + public SingleValueQuery(Query next, String field, UseSyntheticSourceDelegate useSyntheticSourceDelegate) { super(next.source()); this.next = next; this.field = field; @@ -79,9 +88,11 @@ public SingleValueQuery(Query next, String field, boolean useSyntheticSourceDele @Override protected AbstractBuilder asBuilder() { - return useSyntheticSourceDelegate - ? new SyntheticSourceDelegateBuilder(next.toQueryBuilder(), field, next.source()) - : new Builder(next.toQueryBuilder(), field, next.source()); + return switch (useSyntheticSourceDelegate) { + case NO -> new Builder(next.toQueryBuilder(), field, next.source()); + case YES -> new SyntheticSourceDelegateBuilder(next.toQueryBuilder(), field, next.source()); + case YES_NEGATED -> new NegatedSyntheticSourceDelegateBuilder(next.toQueryBuilder(), field, next.source()); + }; } @Override @@ -91,7 +102,11 @@ protected String innerToString() { @Override public SingleValueQuery negate(Source source) { - return new SingleValueQuery(next.negate(source), field, useSyntheticSourceDelegate); + return new SingleValueQuery(next.negate(source), field, switch (useSyntheticSourceDelegate) { + case NO -> UseSyntheticSourceDelegate.NO; + case YES -> UseSyntheticSourceDelegate.YES_NEGATED; + case YES_NEGATED -> UseSyntheticSourceDelegate.YES; + }); } @Override @@ -188,6 +203,28 @@ protected final boolean doEquals(AbstractBuilder other) { protected final int doHashCode() { return Objects.hash(next, field); } + + protected final org.apache.lucene.search.Query simple(MappedFieldType ft, SearchExecutionContext context) throws IOException { + SingleValueMatchQuery singleValueQuery = new SingleValueMatchQuery( + context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH), + Warnings.createWarnings( + DriverContext.WarningsMode.COLLECT, + source().source().getLineNumber(), + source().source().getColumnNumber(), + source().text() + ), + "single-value function encountered multi-value" + ); + org.apache.lucene.search.Query rewrite = singleValueQuery.rewrite(context.searcher()); + if (rewrite instanceof MatchAllDocsQuery) { + // nothing to filter + return next().toQuery(context); + } + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(next().toQuery(context), BooleanClause.Occur.FILTER); + builder.add(rewrite, BooleanClause.Occur.FILTER); + return builder.build(); + } } /** @@ -227,25 +264,7 @@ protected final org.apache.lucene.search.Query doToQuery(SearchExecutionContext if (ft == null) { return new MatchNoDocsQuery("missing field [" + field() + "]"); } - SingleValueMatchQuery singleValueQuery = new SingleValueMatchQuery( - context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH), - Warnings.createWarnings( - DriverContext.WarningsMode.COLLECT, - source().source().getLineNumber(), - source().source().getColumnNumber(), - source().text() - ), - "single-value function encountered multi-value" - ); - org.apache.lucene.search.Query rewrite = singleValueQuery.rewrite(context.searcher()); - if (rewrite instanceof MatchAllDocsQuery) { - // nothing to filter - return next().toQuery(context); - } - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.add(next().toQuery(context), BooleanClause.Occur.FILTER); - builder.add(rewrite, BooleanClause.Occur.FILTER); - return builder.build(); + return simple(ft, context); } @Override @@ -255,7 +274,7 @@ protected AbstractBuilder rewrite(QueryBuilder next) { } /** - * Builds a {@code bool} query combining the "next" query, a {@link SingleValueMatchQuery}, + * Builds a {@code bool} query ANDing the "next" query, a {@link SingleValueMatchQuery}, * and a {@link TermQuery} making sure we didn't ignore any values. Three total queries. * This is only used if the "next" query matches fields that would not be ignored. Read all * the paragraphs below to understand it. It's tricky! @@ -344,6 +363,92 @@ protected AbstractBuilder rewrite(QueryBuilder next) { } } + /** + * Builds a query matching either ignored values OR the union of {@code next} query + * and {@link SingleValueMatchQuery}. Three total queries. This is used to generate + * candidate matches for queries like {@code NOT(a == "b")} where some values of {@code a} + * are not indexed. In fact, let's use that as an example. + *

+ * In that case you use a query for {@code a != "b"} as the "next" query. Then + * this query will find all documents where {@code a} is single valued and + * {@code == "b"} AND all documents that have ignored some values of {@code a}. + * This produces candidate matches for {@code NOT(a == "b")}. + * It'll find documents like: + *

+ *
    + *
  • "a"
    • + *
  • ignored_value
    • + *
  • ["a", ignored_value]
    • + *
  • [ignored_value1, ignored_value2]
    • + *
  • ["b", ignored_field]
    • + *
+ *

+ * The first and second of those should match {@code NOT(a == "b")}. + * The last three should be rejected. So! When using this query you must + * push this query to the {@link LuceneSourceOperator} and + * retain it in the {@link FilterOperator}. + *

+ *

+ * This will not find: + *

+ *
    + *
  • "b"
    • + *
+ *

+ * And that's also great! These can't match {@code NOT(a == "b")} + *

+ */ + public static class NegatedSyntheticSourceDelegateBuilder extends AbstractBuilder { + NegatedSyntheticSourceDelegateBuilder(QueryBuilder next, String field, Source source) { + super(next, field, source); + } + + @Override + public String getWriteableName() { + throw new UnsupportedOperationException("Not serialized"); + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("negated_" + ENTRY.name); + builder.field("field", field() + ":synthetic_source_delegate"); + builder.field("next", next(), params); + builder.field("source", source().toString()); + builder.endObject(); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + throw new UnsupportedOperationException("Not serialized"); + } + + @Override + protected final org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) throws IOException { + MappedFieldType ft = context.getFieldType(field()); + if (ft == null) { + return new MatchNoDocsQuery("missing field [" + field() + "]"); + } + ft = ((TextFieldMapper.TextFieldType) ft).syntheticSourceDelegate(); + org.apache.lucene.search.Query svNext = simple(ft, context); + + org.apache.lucene.search.Query ignored = new TermQuery(new org.apache.lucene.index.Term(IgnoredFieldMapper.NAME, ft.name())); + ignored = ignored.rewrite(context.searcher()); + if (ignored instanceof MatchNoDocsQuery) { + return svNext; + } + + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(svNext, BooleanClause.Occur.SHOULD); + builder.add(ignored, BooleanClause.Occur.SHOULD); + return builder.build(); + } + + @Override + protected AbstractBuilder rewrite(QueryBuilder next) { + return new Builder(next, field(), source()); + } + } + /** * Write a {@link Source} including the text in it. */ @@ -364,4 +469,10 @@ static Source readOldSource(StreamInput in) throws IOException { String text = in.readString(); return new Source(new Location(line, charPositionInLine), text); } + + public enum UseSyntheticSourceDelegate { + NO, + YES, + YES_NEGATED; + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java index e4bec34f5bfb7..d0a68f4986efd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -113,7 +114,7 @@ public void testLuceneQuery_AllLiterals_NonTranslatable() { new Literal(Source.EMPTY, "test", DataType.KEYWORD) ); - assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false)); + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.NO)); } public void testLuceneQuery_NonFoldableSuffix_NonTranslatable() { @@ -123,7 +124,7 @@ public void testLuceneQuery_NonFoldableSuffix_NonTranslatable() { new FieldAttribute(Source.EMPTY, "field", new EsField("suffix", DataType.KEYWORD, Map.of(), true)) ); - assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false)); + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.NO)); } public void testLuceneQuery_NonFoldableSuffix_Translatable() { @@ -133,7 +134,7 @@ public void testLuceneQuery_NonFoldableSuffix_Translatable() { new Literal(Source.EMPTY, "a*b?c\\", DataType.KEYWORD) ); - assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(true)); + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.YES)); var query = function.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java index 8321ccee79fa0..bbbb717f03eb6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -73,7 +74,7 @@ public void testLuceneQuery_AllLiterals_NonTranslatable() { new Literal(Source.EMPTY, "test", DataType.KEYWORD) ); - assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false)); + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.NO)); } public void testLuceneQuery_NonFoldablePrefix_NonTranslatable() { @@ -83,7 +84,7 @@ public void testLuceneQuery_NonFoldablePrefix_NonTranslatable() { new FieldAttribute(Source.EMPTY, "field", new EsField("prefix", DataType.KEYWORD, Map.of(), true)) ); - assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(false)); + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.NO)); } public void testLuceneQuery_NonFoldablePrefix_Translatable() { @@ -93,7 +94,7 @@ public void testLuceneQuery_NonFoldablePrefix_Translatable() { new Literal(Source.EMPTY, "a*b?c\\", DataType.KEYWORD) ); - assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(true)); + assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(TranslationAware.Translatable.YES)); var query = function.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index d8da50585bdfa..ab43ee6c90c10 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -59,6 +59,7 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -7795,7 +7796,6 @@ public void testReductionPlanForAggs() { } public void testEqualsPushdownToDelegate() { - assumeFalse("disabled from bug", true); var optimized = optimizedPlan(physicalPlan(""" FROM test | WHERE job == "v" @@ -7824,6 +7824,33 @@ public void testEqualsPushdownToDelegateTooBig() { as(limit2.child(), FilterExec.class); } + public void testNotEqualsPushdownToDelegate() { + var optimized = optimizedPlan(physicalPlan(""" + FROM test + | WHERE job != "v" + """, testDataLimitedRaw), SEARCH_STATS_SHORT_DELEGATES); + var limit = as(optimized, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var extract = as(project.child(), FieldExtractExec.class); + var limit2 = as(extract.child(), LimitExec.class); + var filter = as(limit2.child(), FilterExec.class); + var extract2 = as(filter.child(), FieldExtractExec.class); + var query = as(extract2.child(), EsQueryExec.class); + assertThat( + query.query(), + equalTo( + new BoolQueryBuilder().filter( + new SingleValueQuery( + new NotQuery(Source.EMPTY, new EqualsSyntheticSourceDelegate(Source.EMPTY, "job", "v")), + "job", + SingleValueQuery.UseSyntheticSourceDelegate.YES_NEGATED + ).toQueryBuilder() + ) + ) + ); + } + /* * LimitExec[1000[INTEGER]] * \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, hire_date{f}#9, job{f}#10, job.raw{f}#11, langua diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryNegateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryNegateTests.java index d79c4a0a518ec..232cd9b804b18 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryNegateTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryNegateTests.java @@ -25,7 +25,15 @@ public void testNot() { var sv = new SingleValueQuery(new MatchAll(Source.EMPTY), "foo", useSyntheticSourceDelegate); assertThat( sv.negate(Source.EMPTY), - equalTo(new SingleValueQuery(new NotQuery(Source.EMPTY, new MatchAll(Source.EMPTY)), "foo", useSyntheticSourceDelegate)) + equalTo( + new SingleValueQuery( + new NotQuery(Source.EMPTY, new MatchAll(Source.EMPTY)), + "foo", + useSyntheticSourceDelegate + ? SingleValueQuery.UseSyntheticSourceDelegate.YES_NEGATED + : SingleValueQuery.UseSyntheticSourceDelegate.NO + ) + ) ); }