From 5406c666f23343fe9a191bc2599b5cfd75ca59ab Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 10 Apr 2025 13:58:51 -0400 Subject: [PATCH 01/17] ESQL: Push more `==`s on text fields to lucene If you do: ``` | WHERE text_field == "cat" ``` we can't push to the text field because it's search index is for individual words. But most text fields have a `.keyword` sub field and we *can* query it's index. EXCEPT! It's normal for these fields to have `ignore_above` in their mapping. In that case we don't push to the field. Very sad. With this change we can push down `==`, but only when the right hand side is shorter than the `ignore_above`. This has pretty much infinite speed gain. An example using a million documents: ``` Before: "took" : 391, After: "took" : 4, ``` But this is going from totally un-indexed linear scans to totally indexed. You can make the "Before" number as high as you want by loading more data. --- .../index/mapper/TextFieldMapper.java | 15 +++++ .../xpack/esql/qa/single_node/RestEsqlIT.java | 57 ++++++++++++++++-- .../xpack/esql/EsqlTestUtils.java | 5 ++ .../resources/mapping-basic-limited-raw.json | 42 +++++++++++++ .../esql/capabilities/TranslationAware.java | 2 +- .../function/fulltext/FullTextFunction.java | 2 +- .../fulltext/QueryBuilderResolver.java | 5 +- .../function/scalar/ip/CIDRMatch.java | 2 +- .../spatial/SpatialRelatesFunction.java | 2 +- .../function/scalar/string/EndsWith.java | 2 +- .../function/scalar/string/RLike.java | 2 +- .../function/scalar/string/StartsWith.java | 2 +- .../function/scalar/string/WildcardLike.java | 2 +- .../esql/expression/predicate/Range.java | 2 +- .../fulltext/MultiMatchQueryPredicate.java | 2 +- .../predicate/logical/BinaryLogic.java | 9 ++- .../expression/predicate/logical/Not.java | 4 +- .../expression/predicate/nulls/IsNotNull.java | 2 +- .../expression/predicate/nulls/IsNull.java | 2 +- .../predicate/operator/comparison/Equals.java | 30 ++++++++++ .../EqualsSyntheticSourceDelegate.java | 59 +++++++++++++++++++ .../comparison/EsqlBinaryComparison.java | 2 +- .../predicate/operator/comparison/In.java | 8 +-- .../comparison/InsensitiveEquals.java | 2 +- .../local/LucenePushdownPredicates.java | 12 ++++ .../physical/local/PushFiltersToSource.java | 13 ++-- .../physical/local/PushStatsToSource.java | 2 +- .../xpack/esql/planner/PlannerUtils.java | 5 +- .../xpack/esql/planner/TranslatorHandler.java | 5 +- .../xpack/esql/stats/SearchContextStats.java | 18 ++++++ .../xpack/esql/stats/SearchStats.java | 6 ++ .../function/fulltext/MatchErrorTests.java | 3 +- .../function/fulltext/MatchTests.java | 3 +- .../function/fulltext/QueryStringTests.java | 3 +- .../function/scalar/string/EndsWithTests.java | 2 +- .../scalar/string/StartsWithTests.java | 2 +- .../optimizer/PhysicalPlanOptimizerTests.java | 52 +++++++++++++++- .../xpack/esql/stats/DisabledSearchStats.java | 5 ++ 38 files changed, 349 insertions(+), 44 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic-limited-raw.json create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsSyntheticSourceDelegate.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 453a289af888a..36ffa774aaef9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -984,6 +984,21 @@ public boolean canUseSyntheticSourceDelegateForQuerying() { && syntheticSourceDelegate.isIndexed(); } + /** + * Returns true if the delegate sub-field can be used for querying only (ie. isIndexed must be true) + */ + public boolean canUseSyntheticSourceDelegateForQueryingEquality(String str) { + if (syntheticSourceDelegate == null + // Can't push equality to an index if there isn't an index + || syntheticSourceDelegate.isIndexed() == false + // ESQL needs docs values to push equality + || syntheticSourceDelegate.hasDocValues() == false) { + return false; + } + // Can't push equality if the field we're checking for is so big we'd ignore it. + return str.length() < syntheticSourceDelegate.ignoreAbove(); + } + @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (canUseSyntheticSourceDelegateForLoading()) { diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 1b44536eed508..6d8dc1cd02c41 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -313,7 +313,7 @@ public void testProfile() throws IOException { @SuppressWarnings("unchecked") List> operators = (List>) p.get("operators"); for (Map o : operators) { - sig.add(checkOperatorProfile(o)); + sig.add(checkOperatorProfile(o, "*:*")); } String description = p.get("description").toString(); switch (description) { @@ -411,6 +411,55 @@ public void testProfileParsing() throws IOException { } } + public void testPushEqualityOnDefaults() throws IOException { + indexTimestampData(1); + + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | WHERE test == \"value1\""); + builder.profile(true); + Map result = runEsql(builder); + assertResultMap( + result, + getResultMatcher(result).entry("profile", matchesMap().entry("drivers", instanceOf(List.class))), + matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) + .item(matchesMap().entry("name", "test").entry("type", "text")) + .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) + .item(matchesMap().entry("name", "value").entry("type", "long")), + equalTo(List.of(List.of("2020-12-12T00:00:00.000Z", "value1", "value1", 1))) + ); + + @SuppressWarnings("unchecked") + List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); + for (Map p : profiles) { + fixTypesOnProfile(p); + assertThat(p, commonProfile()); + List sig = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> operators = (List>) p.get("operators"); + for (Map o : operators) { + // The query here is the most important bit - we *do* push to lucene. + sig.add(checkOperatorProfile(o, "test.keyword:value1")); + } + String description = p.get("description").toString(); + switch (description) { + case "data" -> assertMap( + sig, + matchesList().item("LuceneSourceOperator") + .item("ValuesSourceReaderOperator") + .item("ProjectOperator") + .item("ExchangeSinkOperator") + ); + case "node_reduce" -> assertThat( + sig, + either(matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator")).or( + matchesList().item("ExchangeSourceOperator").item("AggregationOperator").item("ExchangeSinkOperator") + ) + ); + case "final" -> assertMap(sig, matchesList().item("ExchangeSourceOperator").item("LimitOperator").item("OutputOperator")); + default -> throw new IllegalArgumentException("can't match " + description); + } + } + } + @SuppressWarnings("unchecked") public void assertProcessMetadataForNextNode(Map nodeMetadata, Set expectedNamesForNodes, int seenNodes) { assertEquals("M", nodeMetadata.get("ph")); @@ -521,7 +570,7 @@ public void testInlineStatsProfile() throws IOException { @SuppressWarnings("unchecked") List> operators = (List>) p.get("operators"); for (Map o : operators) { - sig.add(checkOperatorProfile(o)); + sig.add(checkOperatorProfile(o, "*:*")); } signatures.add(sig); } @@ -673,7 +722,7 @@ private void fixTypesOnProfile(Map profile) { profile.put("took_nanos", ((Number) profile.get("took_nanos")).longValue()); } - private String checkOperatorProfile(Map o) { + private String checkOperatorProfile(Map o, String query) { String name = (String) o.get("operator"); name = name.replaceAll("\\[.+", ""); MapMatcher status = switch (name) { @@ -687,7 +736,7 @@ private String checkOperatorProfile(Map o) { .entry("pages_emitted", greaterThan(0)) .entry("rows_emitted", greaterThan(0)) .entry("process_nanos", greaterThan(0)) - .entry("processed_queries", List.of("*:*")); + .entry("processed_queries", List.of(query)); case "ValuesSourceReaderOperator" -> basicProfile().entry("readers_built", matchesMap().extraOk()); case "AggregationOperator" -> matchesMap().entry("pages_processed", greaterThan(0)) .entry("rows_received", greaterThan(0)) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 6402745d36005..cdb736f2820c5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -293,6 +293,11 @@ public byte[] max(String field, DataType dataType) { public boolean isSingleValue(String field) { return false; } + + @Override + public boolean canUseEqualityOnSyntheticSourceDelegate(String name, String value) { + return false; + } } /** diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic-limited-raw.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic-limited-raw.json new file mode 100644 index 0000000000000..f27690c0abdfa --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-basic-limited-raw.json @@ -0,0 +1,42 @@ +{ + "properties" : { + "emp_no" : { + "type" : "integer" + }, + "first_name" : { + "type" : "keyword" + }, + "gender" : { + "type" : "text" + }, + "languages" : { + "type" : "byte" + }, + "last_name" : { + "type" : "keyword" + }, + "salary" : { + "type" : "integer" + }, + "_meta_field": { + "type" : "keyword" + }, + "hire_date": { + "type": "date" + }, + "job": { + "type": "text", + "fields": { + "raw": { + "type": "keyword", + "ignore_above": 4 + } + } + }, + "long_noidx": { + "type": "long", + "index": false, + "doc_values": false + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java index 8ef528b6668ab..893ed26a1ab42 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java @@ -31,7 +31,7 @@ public interface TranslationAware { *

and not this:

*

{@code Query childQuery = child.asQuery(handler);}

*/ - Query asQuery(TranslatorHandler handler); + Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler); /** * Subinterface for expressions that can only process single values (and null out on MVs). diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 6e60bb2496e13..5c17ee2dd4935 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -153,7 +153,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { return queryBuilder != null ? new TranslationAwareExpressionQuery(source(), queryBuilder) : translate(handler); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java index baad72155c7cc..b8cb199dd7888 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryBuilderResolver.java @@ -13,6 +13,7 @@ import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.xpack.esql.core.util.Holder; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.planner.TranslatorHandler; @@ -76,7 +77,9 @@ public FullTextFunctionsRewritable rewrite(QueryRewriteContext ctx) throws IOExc Holder updated = new Holder<>(false); LogicalPlan newPlan = plan.transformExpressionsDown(FullTextFunction.class, f -> { QueryBuilder builder = f.queryBuilder(), initial = builder; - builder = builder == null ? f.asQuery(TranslatorHandler.TRANSLATOR_HANDLER).toQueryBuilder() : builder; + builder = builder == null + ? f.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER).toQueryBuilder() + : builder; try { builder = builder.rewrite(ctx); } catch (IOException e) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java index aae01ab774efa..7ffae77483882 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/CIDRMatch.java @@ -184,7 +184,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { var fa = LucenePushdownPredicates.checkIsFieldAttribute(ipField); Check.isTrue(Expressions.foldable(matches), "Expected foldable matches, but got [{}]", matches); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java index 295116a5e99c2..90677f116bd4e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java @@ -183,7 +183,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { if (left().foldable()) { checkSpatialRelatesFunction(left(), queryRelation()); return translate(handler, right(), left()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java index b3d50d7b572fb..5b8dc911b7c6c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java @@ -144,7 +144,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { LucenePushdownPredicates.checkIsPushableAttribute(str); var fieldName = handler.nameOf(str instanceof FieldAttribute fa ? fa.exactAttribute() : str); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java index 0138f2350a36a..0950cc4a3a8ec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java @@ -112,7 +112,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { var fa = LucenePushdownPredicates.checkIsFieldAttribute(field()); // TODO: see whether escaping is needed return new RegexQuery(source(), handler.nameOf(fa.exactAttribute()), pattern().asJavaRegex(), caseInsensitive()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java index 9ab552576dbbb..518d920ab4ee2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java @@ -141,7 +141,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { LucenePushdownPredicates.checkIsPushableAttribute(str); var fieldName = handler.nameOf(str instanceof FieldAttribute fa ? fa.exactAttribute() : str); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java index 22cb0895e4392..1dd2460ea72a1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java @@ -124,7 +124,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { var field = field(); LucenePushdownPredicates.checkIsPushableAttribute(field); return translateField(handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java index e523bf4ddfb42..30ea63d0e473d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java @@ -220,7 +220,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { return translate(handler); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQueryPredicate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQueryPredicate.java index eb6c04be58218..01bfdbe2833fa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQueryPredicate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/fulltext/MultiMatchQueryPredicate.java @@ -100,7 +100,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { return new MultiMatchQuery(source(), query(), fields(), this); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java index f145c6a994b76..e457d3523e9c4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/BinaryLogic.java @@ -90,8 +90,13 @@ && right() instanceof TranslationAware rightAware } @Override - public Query asQuery(TranslatorHandler handler) { - return boolQuery(source(), handler.asQuery(left()), handler.asQuery(right()), this instanceof And); + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { + return boolQuery( + source(), + handler.asQuery(pushdownPredicates, left()), + handler.asQuery(pushdownPredicates, right()), + this instanceof And + ); } public static Query boolQuery(Source source, Query left, Query right, boolean isAnd) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java index 8d11916a24038..4fc43e4c3a9cf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/logical/Not.java @@ -105,7 +105,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { - return handler.asQuery(field()).negate(source()); + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { + return handler.asQuery(pushdownPredicates, field()).negate(source()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java index 93f7df023d89c..7b183f44c4aae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java @@ -80,7 +80,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { return new ExistsQuery(source(), handler.nameOf(field())); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java index bbd47e27e652f..9e393ddb05e9a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNull.java @@ -81,7 +81,7 @@ protected static boolean isTranslatable(Expression field, LucenePushdownPredicat } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { return new NotQuery(source(), new ExistsQuery(source(), handler.nameOf(field()))); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index 3e158e7c8f1db..0b943fc8a989c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -11,13 +11,18 @@ import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; +import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import java.time.ZoneId; import java.util.Map; @@ -120,6 +125,31 @@ public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { ); } + @Override + public boolean translatable(LucenePushdownPredicates pushdownPredicates) { + if (right() instanceof Literal lit) { + if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) { + if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, ((BytesRef) lit.value()).utf8ToString())) { + return true; + } + } + } + return super.translatable(pushdownPredicates); + } + + @Override + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { + if (right() instanceof Literal lit) { + if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) { + String value = ((BytesRef) lit.value()).utf8ToString(); + if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, value)) { + return new EqualsSyntheticSourceDelegate(source(), handler.nameOf(fa), value); + } + } + } + return super.asQuery(pushdownPredicates, handler); + } + @Override public String getWriteableName() { return ENTRY.name; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsSyntheticSourceDelegate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsSyntheticSourceDelegate.java new file mode 100644 index 0000000000000..1409c93a96b83 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsSyntheticSourceDelegate.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.index.query.BaseTermQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class EqualsSyntheticSourceDelegate extends Query { + private final String fieldName; + private final String value; + + public EqualsSyntheticSourceDelegate(Source source, String fieldName, String value) { + super(source); + this.fieldName = fieldName; + this.value = value; + } + + @Override + protected QueryBuilder asBuilder() { + return new Builder(fieldName, value); + } + + @Override + protected String innerToString() { + return fieldName + "(delegate):" + value; + } + + private class Builder extends BaseTermQueryBuilder { + private Builder(String name, String value) { + super(name, value); + } + + @Override + protected org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) { + TextFieldMapper.TextFieldType ft = (TextFieldMapper.TextFieldType) context.getFieldType(fieldName); + return ft.syntheticSourceDelegate().termQuery(value, context); + } + + @Override + public String getWriteableName() { + return "equals_synthetic_source_delegate"; + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index d9d5aa985ded1..542c696fd3521 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -356,7 +356,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { * input to the operation. */ @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { Check.isTrue( right().foldable(), "Line {}:{}: Comparisons against fields are not (currently) supported; offender [{}] in [{}]", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index 6bd57a44c6131..d7279e18c1a8d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -466,11 +466,11 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { - return translate(handler); + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { + return translate(pushdownPredicates, handler); } - private Query translate(TranslatorHandler handler) { + private Query translate(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { logger.trace("Attempting to generate lucene query for IN expression"); TypedAttribute attribute = LucenePushdownPredicates.checkIsPushableAttribute(value()); @@ -483,7 +483,7 @@ private Query translate(TranslatorHandler handler) { // delegates to BinaryComparisons translator to ensure consistent handling of date and time values // TODO: // Query query = BinaryComparisons.translate(new Equals(in.source(), in.value(), rhs), handler); - Query query = handler.asQuery(new Equals(source(), value(), rhs)); + Query query = handler.asQuery(pushdownPredicates, new Equals(source(), value(), rhs)); if (query instanceof TermQuery) { terms.add(((TermQuery) query).value()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java index 3a9ca27637db8..9643655556274 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java @@ -102,7 +102,7 @@ public boolean translatable(LucenePushdownPredicates pushdownPredicates) { } @Override - public Query asQuery(TranslatorHandler handler) { + public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { checkInsensitiveComparison(); return translate(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index f7fad1cdb984c..7843f8a6cfe04 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -47,6 +47,8 @@ public interface LucenePushdownPredicates { */ boolean isIndexed(FieldAttribute attr); + boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute attr, String value); + /** * We see fields as pushable if either they are aggregatable or they are indexed. * This covers non-indexed cases like AbstractScriptFieldType which hard-coded isAggregatable to true, @@ -116,6 +118,11 @@ public boolean isIndexed(FieldAttribute attr) { // TODO: This is the original behaviour, but is it correct? In FieldType isAggregatable usually only means hasDocValues return attr.field().isAggregatable(); } + + @Override + public boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute attr, String value) { + return false; + } }; /** @@ -141,6 +148,11 @@ public boolean isIndexedAndHasDocValues(FieldAttribute attr) { public boolean isIndexed(FieldAttribute attr) { return stats.isIndexed(attr.name()); } + + @Override + public boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute attr, String value) { + return stats.canUseEqualityOnSyntheticSourceDelegate(attr.field().getName(), value); + } }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 67dc69efb2bc7..8d0c88d9c399d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -53,12 +53,13 @@ protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext } private static PhysicalPlan planFilterExec(FilterExec filterExec, EsQueryExec queryExec, LocalPhysicalOptimizerContext ctx) { + LucenePushdownPredicates pushdownPredicates = LucenePushdownPredicates.from(ctx.searchStats()); List pushable = new ArrayList<>(); List nonPushable = new ArrayList<>(); for (Expression exp : splitAnd(filterExec.condition())) { - (canPushToSource(exp, LucenePushdownPredicates.from(ctx.searchStats())) ? pushable : nonPushable).add(exp); + (canPushToSource(exp, pushdownPredicates) ? pushable : nonPushable).add(exp); } - return rewrite(filterExec, queryExec, pushable, nonPushable, List.of()); + return rewrite(pushdownPredicates, filterExec, queryExec, pushable, nonPushable, List.of()); } private static PhysicalPlan planFilterExec( @@ -67,16 +68,17 @@ private static PhysicalPlan planFilterExec( EsQueryExec queryExec, LocalPhysicalOptimizerContext ctx ) { + LucenePushdownPredicates pushdownPredicates = LucenePushdownPredicates.from(ctx.searchStats()); AttributeMap aliasReplacedBy = getAliasReplacedBy(evalExec); List pushable = new ArrayList<>(); List nonPushable = new ArrayList<>(); for (Expression exp : splitAnd(filterExec.condition())) { Expression resExp = exp.transformUp(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)); - (canPushToSource(resExp, LucenePushdownPredicates.from(ctx.searchStats())) ? pushable : nonPushable).add(exp); + (canPushToSource(resExp, pushdownPredicates) ? pushable : nonPushable).add(exp); } // Replace field references with their actual field attributes pushable.replaceAll(e -> e.transformDown(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r))); - return rewrite(filterExec, queryExec, pushable, nonPushable, evalExec.fields()); + return rewrite(pushdownPredicates, filterExec, queryExec, pushable, nonPushable, evalExec.fields()); } static AttributeMap getAliasReplacedBy(EvalExec evalExec) { @@ -90,6 +92,7 @@ static AttributeMap getAliasReplacedBy(EvalExec evalExec) { } private static PhysicalPlan rewrite( + LucenePushdownPredicates pushdownPredicates, FilterExec filterExec, EsQueryExec queryExec, List pushable, @@ -99,7 +102,7 @@ private static PhysicalPlan rewrite( // Combine GT, GTE, LT and LTE in pushable to Range if possible List newPushable = combineEligiblePushableToRange(pushable); if (newPushable.size() > 0) { // update the executable with pushable conditions - Query queryDSL = TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(newPushable)); + Query queryDSL = TRANSLATOR_HANDLER.asQuery(pushdownPredicates, Predicates.combineAnd(newPushable)); QueryBuilder planQuery = queryDSL.toQueryBuilder(); Queries.Clause combiningQueryClauseType = queryExec.hasScoring() ? Queries.Clause.MUST : Queries.Clause.FILTER; var query = Queries.combine(combiningQueryClauseType, asList(queryExec.query(), planQuery)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java index e3954688aed41..3dab1e8b31eac 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushStatsToSource.java @@ -106,7 +106,7 @@ private Tuple, List> pushableStats( if (canPushToSource(count.filter()) == false) { return null; // can't push down } - var countFilter = TRANSLATOR_HANDLER.asQuery(count.filter()); + var countFilter = TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, count.filter()); query = Queries.combine(Queries.Clause.MUST, asList(countFilter.toQueryBuilder(), query)); } return new EsStatsQueryExec.Stat(fieldName, COUNT, query); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index ec87016ccba31..98202dca42b03 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.plan.QueryPlan; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Filter; @@ -220,7 +221,9 @@ static QueryBuilder detectFilter(PhysicalPlan plan, Predicate fieldName) } } if (matches.isEmpty() == false) { - requestFilters.add(TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(matches)).toQueryBuilder()); + requestFilters.add( + TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, Predicates.combineAnd(matches)).toQueryBuilder() + ); } }); }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java index f7f09d36a4296..81b1c021e5aeb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; /** @@ -29,9 +30,9 @@ public final class TranslatorHandler { private TranslatorHandler() {} - public Query asQuery(Expression e) { + public Query asQuery(LucenePushdownPredicates predicates, Expression e) { if (e instanceof TranslationAware ta) { - Query query = ta.asQuery(this); + Query query = ta.asQuery(predicates, this); return ta instanceof TranslationAware.SingleValueTranslationAware sv ? wrapFunctionQuery(sv.singleValueField(), query) : query; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java index d4048c790defb..57ad2d0275a88 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java @@ -292,6 +292,24 @@ private boolean detectSingleValue(IndexReader r, MappedFieldType fieldType, Stri return false; } + @Override + public boolean canUseEqualityOnSyntheticSourceDelegate(String name, String value) { + for (SearchExecutionContext ctx : contexts) { + MappedFieldType type = ctx.getFieldType(name); + if (type == null) { + return false; + } + if (type instanceof TextFieldMapper.TextFieldType t) { + if (t.canUseSyntheticSourceDelegateForQueryingEquality(value) == false) { + return false; + } + } else { + return false; + } + } + return true; + } + private interface DocCountTester { Boolean test(LeafReader leafReader) throws IOException; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchStats.java index ca24bd54ee67c..e00de98178832 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchStats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchStats.java @@ -37,6 +37,8 @@ public interface SearchStats { boolean isSingleValue(String field); + boolean canUseEqualityOnSyntheticSourceDelegate(String name, String value); + /** * When there are no search stats available, for example when there are no search contexts, we have static results. */ @@ -92,5 +94,9 @@ public boolean isSingleValue(String field) { return true; } + @Override + public boolean canUseEqualityOnSyntheticSourceDelegate(String name, String value) { + return false; + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchErrorTests.java index a1f4792094615..a60a522b86aa4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchErrorTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.hamcrest.Matcher; import java.util.List; @@ -40,7 +41,7 @@ protected Expression build(Source source, List args) { // We need to add the QueryBuilder to the match expression, as it is used to implement equals() and hashCode() and // thus test the serialization methods. But we can only do this if the parameters make sense . if (args.get(0) instanceof FieldAttribute && args.get(1).foldable()) { - QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(match).toQueryBuilder(); + QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, match).toQueryBuilder(); match.replaceQueryBuilder(queryBuilder); } return match; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java index f790e120786dc..6993f7583dd02 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MatchTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import java.util.ArrayList; import java.util.List; @@ -80,7 +81,7 @@ protected Expression build(Source source, List args) { // We need to add the QueryBuilder to the match expression, as it is used to implement equals() and hashCode() and // thus test the serialization methods. But we can only do this if the parameters make sense . if (args.get(0) instanceof FieldAttribute && args.get(1).foldable()) { - QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(match).toQueryBuilder(); + QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, match).toQueryBuilder(); match.replaceQueryBuilder(queryBuilder); } return match; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java index 131d9e2d6157e..71c0debc13fcc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryStringTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import java.util.ArrayList; import java.util.List; @@ -79,7 +80,7 @@ protected Expression build(Source source, List args) { // We need to add the QueryBuilder to the match expression, as it is used to implement equals() and hashCode() and // thus test the serialization methods. But we can only do this if the parameters make sense . if (args.get(0).foldable()) { - QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(qstr).toQueryBuilder(); + QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, qstr).toQueryBuilder(); qstr.replaceQueryBuilder(queryBuilder); } return qstr; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java index a5fe9d7c78b68..e4bec34f5bfb7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java @@ -135,7 +135,7 @@ public void testLuceneQuery_NonFoldableSuffix_Translatable() { assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(true)); - var query = function.asQuery(TranslatorHandler.TRANSLATOR_HANDLER); + var query = function.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER); assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "*a\\*b\\?c\\\\"))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java index 06d2757766060..8321ccee79fa0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java @@ -95,7 +95,7 @@ public void testLuceneQuery_NonFoldablePrefix_Translatable() { assertThat(function.translatable(LucenePushdownPredicates.DEFAULT), equalTo(true)); - var query = function.asQuery(TranslatorHandler.TRANSLATOR_HANDLER); + var query = function.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER); assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "a\\*b\\?c\\\\*"))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 19e449d04f6ba..7881f0d21e584 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -84,6 +84,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EqualsSyntheticSourceDelegate; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; @@ -215,6 +216,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase { private PhysicalPlanOptimizer physicalPlanOptimizer; private Mapper mapper; private TestDataSource testData; + private TestDataSource testDataLimitedRaw; private int allFieldRowSize; // TODO: Move this into testDataSource so tests that load other indexes can also assert on this private TestDataSource airports; private TestDataSource airportsNoDocValues; // Test when spatial field is indexed but has no doc values @@ -232,10 +234,10 @@ public class PhysicalPlanOptimizerTests extends ESTestCase { private final Configuration config; - private record TestDataSource(Map mapping, EsIndex index, Analyzer analyzer, SearchStats stats) {} + public record TestDataSource(Map mapping, EsIndex index, Analyzer analyzer, SearchStats stats) {} @ParametersFactory(argumentFormatting = PARAM_FORMATTING) - public static List readScriptSpec() { + public static List params() { return settings().stream().map(t -> { var settings = Settings.builder().loadFromMap(t.v2()).build(); return new Object[] { t.v1(), configuration(new QueryPragmas(settings)) }; @@ -260,6 +262,7 @@ public void init() { var enrichResolution = setupEnrichResolution(); // Most tests used data from the test index, so we load it here, and use it in the plan() function. this.testData = makeTestDataSource("test", "mapping-basic.json", functionRegistry, enrichResolution); + this.testDataLimitedRaw = makeTestDataSource("test", "mapping-basic-limited-raw.json", functionRegistry, enrichResolution); allFieldRowSize = testData.mapping.values() .stream() .mapToInt( @@ -7788,6 +7791,37 @@ public void testReductionPlanForAggs() { assertThat(reductionAggs.estimatedRowSize(), equalTo(58)); // double and keyword } + public void testEqualsPushdownToDelegate() { + var optimized = optimizedPlan(physicalPlan(""" + FROM test + | WHERE job == "v" + """, testDataLimitedRaw), SEARCH_STATS_SHORT_DELEGATES); + var limit = as(optimized, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var extract = as(project.child(), FieldExtractExec.class); + var query = as(extract.child(), EsQueryExec.class); + // NOCOMMIT the single value query should target the synthetic source delegate. + assertThat( + query.query(), + equalTo(new SingleValueQuery(new EqualsSyntheticSourceDelegate(Source.EMPTY, "job", "v"), "job.raw").toQueryBuilder()) + ); + + } + + public void testEqualsPushdownToDelegateTooBig() { + var optimized = optimizedPlan(physicalPlan(""" + FROM test + | WHERE job == "too_long" + """, testDataLimitedRaw), SEARCH_STATS_SHORT_DELEGATES); + var limit = as(optimized, LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + var project = as(exchange.child(), ProjectExec.class); + var extract = as(project.child(), FieldExtractExec.class); + var limit2 = as(extract.child(), LimitExec.class); + as(limit2.child(), FilterExec.class); + } + @SuppressWarnings("SameParameterValue") private static void assertFilterCondition( Filter filter, @@ -7963,7 +7997,7 @@ private PhysicalPlan physicalPlan(String query) { return physicalPlan(query, testData); } - private PhysicalPlan physicalPlan(String query, TestDataSource dataSource) { + public PhysicalPlan physicalPlan(String query, TestDataSource dataSource) { return physicalPlan(query, dataSource, true); } @@ -8001,4 +8035,16 @@ private QueryBuilder sv(QueryBuilder builder, String fieldName) { protected List filteredWarnings() { return withDefaultLimitWarning(super.filteredWarnings()); } + + private static final SearchStats SEARCH_STATS_SHORT_DELEGATES = new EsqlTestUtils.TestSearchStats() { + @Override + public boolean hasExactSubfield(String field) { + return false; + } + + @Override + public boolean canUseEqualityOnSyntheticSourceDelegate(String name, String value) { + return value.length() < 4; + } + }; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/DisabledSearchStats.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/DisabledSearchStats.java index fce05b07a6a42..60848c4ae5d00 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/DisabledSearchStats.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/DisabledSearchStats.java @@ -61,4 +61,9 @@ public byte[] max(String field, DataType dataType) { public boolean isSingleValue(String field) { return false; } + + @Override + public boolean canUseEqualityOnSyntheticSourceDelegate(String name, String value) { + return false; + } } From c6bb22870ba27fef1a234ef921842717feabbffa Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 10 Apr 2025 16:07:18 -0400 Subject: [PATCH 02/17] Update docs/changelog/126641.yaml --- docs/changelog/126641.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/126641.yaml diff --git a/docs/changelog/126641.yaml b/docs/changelog/126641.yaml new file mode 100644 index 0000000000000..d99977d981acd --- /dev/null +++ b/docs/changelog/126641.yaml @@ -0,0 +1,5 @@ +pr: 126641 +summary: Push more `==`s on text fields to lucene +area: ES|QL +type: enhancement +issues: [] From a5e2206e730001d3b4ba8cc92048e2f8e9728fdd Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 11 Apr 2025 16:27:00 -0400 Subject: [PATCH 03/17] Fix off by one --- .../index/mapper/TextFieldMapper.java | 2 +- .../xpack/esql/qa/single_node/RestEsqlIT.java | 78 ++++++++++++++++++- 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 36ffa774aaef9..7293b5ca6c6ad 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -996,7 +996,7 @@ public boolean canUseSyntheticSourceDelegateForQueryingEquality(String str) { return false; } // Can't push equality if the field we're checking for is so big we'd ignore it. - return str.length() < syntheticSourceDelegate.ignoreAbove(); + return str.length() <= syntheticSourceDelegate.ignoreAbove(); } @Override diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 235f038811b3b..774f05e6318c5 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -414,7 +414,10 @@ public void testProfileParsing() throws IOException { public void testPushEqualityOnDefaults() throws IOException { indexTimestampData(1); - RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | WHERE test == \"value1\""); + String value = "v".repeat(between(0, 256)); + indexValue(value); + + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | WHERE test == \"" + value + "\""); builder.profile(true); Map result = runEsql(builder); assertResultMap( @@ -424,7 +427,7 @@ public void testPushEqualityOnDefaults() throws IOException { .item(matchesMap().entry("name", "test").entry("type", "text")) .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) .item(matchesMap().entry("name", "value").entry("type", "long")), - equalTo(List.of(List.of("2020-12-12T00:00:00.000Z", "value1", "value1", 1))) + equalTo(List.of(Arrays.asList(null, value, value, null))) ); @SuppressWarnings("unchecked") @@ -437,7 +440,7 @@ public void testPushEqualityOnDefaults() throws IOException { List> operators = (List>) p.get("operators"); for (Map o : operators) { // The query here is the most important bit - we *do* push to lucene. - sig.add(checkOperatorProfile(o, "test.keyword:value1")); + sig.add(checkOperatorProfile(o, "test.keyword:" + value)); } String description = p.get("description").toString(); switch (description) { @@ -460,6 +463,73 @@ public void testPushEqualityOnDefaults() throws IOException { } } + public void testPushEqualityOnDefaultsTooBigToPush() throws IOException { + indexTimestampData(1); + + String value = "a".repeat(between(257, 1000)); + indexValue(value); + + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | WHERE test == \"" + value + "\""); + builder.profile(true); + Map result = runEsql(builder); + assertResultMap( + result, + getResultMatcher(result).entry("profile", matchesMap().entry("drivers", instanceOf(List.class))), + matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) + .item(matchesMap().entry("name", "test").entry("type", "text")) + .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) + .item(matchesMap().entry("name", "value").entry("type", "long")), + equalTo(List.of(Arrays.asList(null, value, null, null))) + ); + + @SuppressWarnings("unchecked") + List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); + for (Map p : profiles) { + fixTypesOnProfile(p); + assertThat(p, commonProfile()); + List sig = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> operators = (List>) p.get("operators"); + for (Map o : operators) { + // The query here is the most important bit - we do *not* push to lucene. + sig.add(checkOperatorProfile(o, "*:*")); + } + String description = p.get("description").toString(); + switch (description) { + case "data" -> assertMap( + sig, + matchesList().item("LuceneSourceOperator") + .item("ValuesSourceReaderOperator") + .item("FilterOperator") + .item("LimitOperator") + .item("ValuesSourceReaderOperator") + .item("ProjectOperator") + .item("ExchangeSinkOperator") + ); + case "node_reduce" -> assertThat( + sig, + either(matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator")).or( + matchesList().item("ExchangeSourceOperator").item("AggregationOperator").item("ExchangeSinkOperator") + ) + ); + case "final" -> assertMap(sig, matchesList().item("ExchangeSourceOperator").item("LimitOperator").item("OutputOperator")); + default -> throw new IllegalArgumentException("can't match " + description); + } + } + } + + private void indexValue(String value) throws IOException { + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("filter_path", "errors"); + bulk.addParameter("refresh", ""); + bulk.setJsonEntity(String.format(""" + {"create":{"_index":"%s"}} + {"test":"%s"} + """, testIndexName(), value)); + Response response = client().performRequest(bulk); + Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); + } + @SuppressWarnings("unchecked") public void assertProcessMetadataForNextNode(Map nodeMetadata, Set expectedNamesForNodes, int seenNodes) { assertEquals("M", nodeMetadata.get("ph")); @@ -748,7 +818,7 @@ private String checkOperatorProfile(Map o, String query) { case "ExchangeSourceOperator" -> matchesMap().entry("pages_waiting", 0) .entry("pages_emitted", greaterThan(0)) .entry("rows_emitted", greaterThan(0)); - case "ProjectOperator", "EvalOperator" -> basicProfile(); + case "ProjectOperator", "EvalOperator", "FilterOperator" -> basicProfile(); case "LimitOperator" -> matchesMap().entry("pages_processed", greaterThan(0)) .entry("limit", 1000) .entry("limit_remaining", 999) From 9bd4b89bd5b93d2da1aacb0fa9c158c355bba124 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 14 Apr 2025 13:40:27 -0400 Subject: [PATCH 04/17] Proper delegate --- .../predicate/operator/comparison/Equals.java | 5 +- .../xpack/esql/planner/TranslatorHandler.java | 6 +- .../query}/EqualsSyntheticSourceDelegate.java | 2 +- .../esql/querydsl/query/SingleValueQuery.java | 146 ++++++++++++++---- .../operator/comparison/InTests.java | 3 +- .../optimizer/PhysicalPlanOptimizerTests.java | 6 +- .../query/SingleValueQueryNegateTests.java | 10 +- .../querydsl/query/SingleValueQueryTests.java | 11 +- 8 files changed, 139 insertions(+), 50 deletions(-) rename x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/{expression/predicate/operator/comparison => querydsl/query}/EqualsSyntheticSourceDelegate.java (96%) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index 0b943fc8a989c..d00d276ce228b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -23,6 +23,8 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.planner.TranslatorHandler; +import org.elasticsearch.xpack.esql.querydsl.query.EqualsSyntheticSourceDelegate; +import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import java.time.ZoneId; import java.util.Map; @@ -143,7 +145,8 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) { String value = ((BytesRef) lit.value()).utf8ToString(); if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, value)) { - return new EqualsSyntheticSourceDelegate(source(), handler.nameOf(fa), value); + String name = handler.nameOf(fa); + return new SingleValueQuery(new EqualsSyntheticSourceDelegate(source(), name, value), name, true); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java index 81b1c021e5aeb..4b7af5bf49de8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java @@ -40,9 +40,13 @@ public Query asQuery(LucenePushdownPredicates predicates, Expression e) { } private static Query wrapFunctionQuery(Expression field, Query query) { + if (query instanceof SingleValueQuery) { + // Already wrapped + return query; + } if (field instanceof FieldAttribute fa) { fa = fa.getExactInfo().hasExact() ? fa.exactAttribute() : fa; - return new SingleValueQuery(query, fa.name()); + return new SingleValueQuery(query, fa.name(), false); } if (field instanceof MetadataAttribute) { return query; // MetadataAttributes are always single valued diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsSyntheticSourceDelegate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java similarity index 96% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsSyntheticSourceDelegate.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java index 1409c93a96b83..def564e21310a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsSyntheticSourceDelegate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; +package org.elasticsearch.xpack.esql.querydsl.query; import org.elasticsearch.TransportVersion; import org.elasticsearch.index.mapper.TextFieldMapper; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index 819d2733d8686..fd0215021b55e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -20,6 +20,7 @@ import org.elasticsearch.compute.operator.Warnings; import org.elasticsearch.compute.querydsl.query.SingleValueMatchQuery; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -57,16 +58,28 @@ public class SingleValueQuery extends Query { private final Query next; private final String field; + private final boolean useSyntheticSourceDelegate; - public SingleValueQuery(Query next, String field) { + /** + * Build. + * @param next the query whose documents we should use for single-valued fields + * @param field the name of the field whose values to check + * @param useSyntheticSourceDelegate Should we check the field's synthetic source delegate (true) + * or it's values itself? If the field is a {@code text} field + * we often want to use its delegate. + */ + public SingleValueQuery(Query next, String field, boolean useSyntheticSourceDelegate) { super(next.source()); this.next = next; this.field = field; + this.useSyntheticSourceDelegate = useSyntheticSourceDelegate; } @Override - protected Builder asBuilder() { - return new Builder(next.toQueryBuilder(), field, next.source()); + protected AbstractBuilder asBuilder() { + return useSyntheticSourceDelegate + ? new SyntheticSourceDelegateBuilder(next.toQueryBuilder(), field, next.source()) + : new Builder(next.toQueryBuilder(), field, next.source()); } @Override @@ -76,7 +89,7 @@ protected String innerToString() { @Override public SingleValueQuery negate(Source source) { - return new SingleValueQuery(next.negate(source), field); + return new SingleValueQuery(next.negate(source), field, useSyntheticSourceDelegate); } @Override @@ -85,26 +98,28 @@ public boolean equals(Object o) { return false; } SingleValueQuery other = (SingleValueQuery) o; - return Objects.equals(next, other.next) && Objects.equals(field, other.field); + return Objects.equals(next, other.next) + && Objects.equals(field, other.field) + && useSyntheticSourceDelegate == other.useSyntheticSourceDelegate; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), next, field); + return Objects.hash(super.hashCode(), next, field, useSyntheticSourceDelegate); } - public static class Builder extends AbstractQueryBuilder { + public abstract static class AbstractBuilder extends AbstractQueryBuilder { private final QueryBuilder next; private final String field; private final Source source; - Builder(QueryBuilder next, String field, Source source) { + AbstractBuilder(QueryBuilder next, String field, Source source) { this.next = next; this.field = field; this.source = source; } - Builder(StreamInput in) throws IOException { + AbstractBuilder(StreamInput in) throws IOException { super(in); this.next = in.readNamedWriteable(QueryBuilder.class); this.field = in.readString(); @@ -126,7 +141,7 @@ public static class Builder extends AbstractQueryBuilder { } @Override - protected void doWriteTo(StreamOutput out) throws IOException { + protected final void doWriteTo(StreamOutput out) throws IOException { out.writeNamedWriteable(next); out.writeString(field); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) { @@ -148,28 +163,11 @@ public Source source() { return source; } - @Override - public String getWriteableName() { - return ENTRY.name; - } - - @Override - protected void doXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(ENTRY.name); - builder.field("field", field); - builder.field("next", next, params); - builder.field("source", source.toString()); - builder.endObject(); - } - - @Override - public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_8_11_X; // the first version of ESQL - } + protected abstract MappedFieldType mappedFieldType(SearchExecutionContext context); @Override - protected org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) throws IOException { - MappedFieldType ft = context.getFieldType(field); + protected final org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) throws IOException { + MappedFieldType ft = mappedFieldType(context); if (ft == null) { return new MatchNoDocsQuery("missing field [" + field + "]"); } @@ -194,8 +192,10 @@ protected org.apache.lucene.search.Query doToQuery(SearchExecutionContext contex return builder.build(); } + protected abstract AbstractBuilder rewrite(QueryBuilder next); + @Override - protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + protected final QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { QueryBuilder rewritten = next.rewrite(queryRewriteContext); if (rewritten instanceof MatchNoneQueryBuilder) { return rewritten; @@ -203,20 +203,98 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws if (rewritten == next) { return this; } - return new Builder(rewritten, field, source); + return rewrite(rewritten); } @Override - protected boolean doEquals(Builder other) { + protected final boolean doEquals(AbstractBuilder other) { return next.equals(other.next) && field.equals(other.field); } @Override - protected int doHashCode() { + protected final int doHashCode() { return Objects.hash(next, field); } } + public static class Builder extends AbstractBuilder { + Builder(QueryBuilder next, String field, Source source) { + super(next, field, source); + } + + Builder(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(ENTRY.name); + builder.field("field", field()); + builder.field("next", next(), params); + builder.field("source", source().toString()); + builder.endObject(); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.V_8_11_X; // the first version of ESQL + } + + @Override + protected MappedFieldType mappedFieldType(SearchExecutionContext context) { + return context.getFieldType(field()); + } + + @Override + protected AbstractBuilder rewrite(QueryBuilder next) { + return new Builder(next, field(), source()); + } + } + + public static class SyntheticSourceDelegateBuilder extends AbstractBuilder { + SyntheticSourceDelegateBuilder(QueryBuilder next, String field, Source source) { + super(next, field, source); + } + + SyntheticSourceDelegateBuilder(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + throw new UnsupportedOperationException(); + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(ENTRY.name); + builder.field("field", field() + ":synthetic_source_delegate"); + builder.field("next", next(), params); + builder.field("source", source().toString()); + builder.endObject(); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + throw new UnsupportedOperationException(); + } + + @Override + protected MappedFieldType mappedFieldType(SearchExecutionContext context) { + return ((TextFieldMapper.TextFieldType) context.getFieldType(field())).syntheticSourceDelegate(); + } + + @Override + protected AbstractBuilder rewrite(QueryBuilder next) { + return new Builder(next, field(), source()); + } + } + /** * Write a {@link Source} including the text in it. */ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java index e26b6ae616865..aed08b32bb6d3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import org.junit.AfterClass; @@ -93,7 +94,7 @@ public void testConvertedNull() { new FieldAttribute(Source.EMPTY, "field", new EsField("suffix", DataType.KEYWORD, Map.of(), true)), Arrays.asList(ONE, new Literal(Source.EMPTY, null, randomFrom(DataType.types())), THREE) ); - var query = in.asQuery(TranslatorHandler.TRANSLATOR_HANDLER); + var query = in.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER); assertEquals(new TermsQuery(EMPTY, "field", Set.of(1, 3)), query); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index a5fe11d815a13..d04a36f39cd19 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -85,7 +85,6 @@ import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EqualsSyntheticSourceDelegate; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; @@ -135,6 +134,7 @@ import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; +import org.elasticsearch.xpack.esql.querydsl.query.EqualsSyntheticSourceDelegate; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import org.elasticsearch.xpack.esql.querydsl.query.SpatialRelatesQuery; import org.elasticsearch.xpack.esql.session.Configuration; @@ -7802,12 +7802,10 @@ public void testEqualsPushdownToDelegate() { var project = as(exchange.child(), ProjectExec.class); var extract = as(project.child(), FieldExtractExec.class); var query = as(extract.child(), EsQueryExec.class); - // NOCOMMIT the single value query should target the synthetic source delegate. assertThat( query.query(), - equalTo(new SingleValueQuery(new EqualsSyntheticSourceDelegate(Source.EMPTY, "job", "v"), "job.raw").toQueryBuilder()) + equalTo(new SingleValueQuery(new EqualsSyntheticSourceDelegate(Source.EMPTY, "job", "v"), "job", true).toQueryBuilder()) ); - } public void testEqualsPushdownToDelegateTooBig() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryNegateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryNegateTests.java index 2545a93b326ab..d79c4a0a518ec 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryNegateTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryNegateTests.java @@ -21,12 +21,16 @@ */ public class SingleValueQueryNegateTests extends ESTestCase { public void testNot() { - var sv = new SingleValueQuery(new MatchAll(Source.EMPTY), "foo"); - assertThat(sv.negate(Source.EMPTY), equalTo(new SingleValueQuery(new NotQuery(Source.EMPTY, new MatchAll(Source.EMPTY)), "foo"))); + boolean useSyntheticSourceDelegate = randomBoolean(); + var sv = new SingleValueQuery(new MatchAll(Source.EMPTY), "foo", useSyntheticSourceDelegate); + assertThat( + sv.negate(Source.EMPTY), + equalTo(new SingleValueQuery(new NotQuery(Source.EMPTY, new MatchAll(Source.EMPTY)), "foo", useSyntheticSourceDelegate)) + ); } public void testNotNot() { - var sv = new SingleValueQuery(new MatchAll(Source.EMPTY), "foo"); + var sv = new SingleValueQuery(new MatchAll(Source.EMPTY), "foo", randomBoolean()); assertThat(sv.negate(Source.EMPTY).negate(Source.EMPTY), equalTo(sv)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java index 95444c9b2423f..a48984bb310f2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java @@ -69,7 +69,7 @@ public SingleValueQueryTests(Setup setup) { } public void testMatchAll() throws IOException { - testCase(new SingleValueQuery(new MatchAll(Source.EMPTY), "foo").asBuilder(), this::runCase); + testCase(new SingleValueQuery(new MatchAll(Source.EMPTY), "foo", false).asBuilder(), this::runCase); } public void testMatchSome() throws IOException { @@ -100,14 +100,14 @@ public void testRewritesToMatchNone() throws IOException { public void testNotMatchAll() throws IOException { testCase( - new SingleValueQuery(new MatchAll(Source.EMPTY), "foo").negate(Source.EMPTY).asBuilder(), + new SingleValueQuery(new MatchAll(Source.EMPTY), "foo", false).negate(Source.EMPTY).asBuilder(), (fieldValues, count) -> assertThat(count, equalTo(0)) ); } public void testNotMatchNone() throws IOException { testCase( - new SingleValueQuery(new MatchAll(Source.EMPTY).negate(Source.EMPTY), "foo").negate(Source.EMPTY).asBuilder(), + new SingleValueQuery(new MatchAll(Source.EMPTY).negate(Source.EMPTY), "foo", false).negate(Source.EMPTY).asBuilder(), this::runCase ); } @@ -115,7 +115,8 @@ public void testNotMatchNone() throws IOException { public void testNotMatchSome() throws IOException { int max = between(1, 100); testCase( - new SingleValueQuery(new RangeQuery(Source.EMPTY, "i", null, false, max, false, null), "foo").negate(Source.EMPTY).asBuilder(), + new SingleValueQuery(new RangeQuery(Source.EMPTY, "i", null, false, max, false, null), "foo", false).negate(Source.EMPTY) + .asBuilder(), (fieldValues, count) -> runCase(fieldValues, count, max, 100) ); } @@ -161,7 +162,7 @@ private void runCase(List> fieldValues, int count) { runCase(fieldValues, count, null, null); } - private void testCase(SingleValueQuery.Builder builder, TestCase testCase) throws IOException { + private void testCase(SingleValueQuery.AbstractBuilder builder, TestCase testCase) throws IOException { MapperService mapper = createMapperService(mapping(setup::mapping)); try (Directory d = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), d)) { List> fieldValues = setup.build(iw); From 42b7a20fb8dfd7f98faf40daf469fd6c4c190c20 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 14 Apr 2025 14:37:18 -0400 Subject: [PATCH 05/17] Fix delegate for ignored fields --- .../xpack/esql/CsvTestsDataLoader.java | 4 +- .../src/main/resources/data/mv_text.csv | 4 + .../src/main/resources/mapping-mv_text.json | 16 +++ .../src/main/resources/string.csv-spec | 12 +++ .../esql/querydsl/query/SingleValueQuery.java | 100 +++++++++++------- 5 files changed, 96 insertions(+), 40 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-mv_text.json diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 68b0818284f05..2abe77fe08c89 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -139,6 +139,7 @@ public class CsvTestsDataLoader { private static final TestDataset BOOKS = new TestDataset("books").withSetting("books-settings.json"); private static final TestDataset SEMANTIC_TEXT = new TestDataset("semantic_text").withInferenceEndpoint(true); private static final TestDataset LOGS = new TestDataset("logs"); + private static final TestDataset MV_TEXT = new TestDataset("mv_text"); public static final Map CSV_DATASET_MAP = Map.ofEntries( Map.entry(EMPLOYEES.indexName, EMPLOYEES), @@ -196,7 +197,8 @@ public class CsvTestsDataLoader { Map.entry(ADDRESSES.indexName, ADDRESSES), Map.entry(BOOKS.indexName, BOOKS), Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT), - Map.entry(LOGS.indexName, LOGS) + Map.entry(LOGS.indexName, LOGS), + Map.entry(MV_TEXT.indexName, MV_TEXT) ); private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json"); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv new file mode 100644 index 0000000000000..1e24cf85881f7 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv @@ -0,0 +1,4 @@ +@timestamp:date ,message:text +2023-10-23T13:55:01.543Z,[Connected to 10.1.0.1, Banana] +2023-10-23T13:55:01.544Z,Connected to 10.1.0.1 +2023-10-23T13:55:01.545Z,[Connected to 10.1.0.1, More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-mv_text.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-mv_text.json new file mode 100644 index 0000000000000..f6cf8d9cfe1df --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-mv_text.json @@ -0,0 +1,16 @@ +{ + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "text", + "fields": { + "raw": { + "type": "keyword", + "ignore_above": 100 + } + } + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index f291df4c3e4dc..e16a213532b58 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -2308,3 +2308,15 @@ message:keyword foo ( bar // end::rlikeEscapingTripleQuotes-result[] ; + +mvStringEquals +FROM mv_text +| WHERE message == "Connected to 10.1.0.1" +| KEEP @timestamp, message +; +warning:Line 2:9: evaluation of [message == \"Connected to 10.1.0.1\"] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + + @timestamp:date | message:text +2023-10-23T13:55:01.544Z|Connected to 10.1.0.1 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index fd0215021b55e..1076e77ce5b8e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -11,6 +11,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.TermQuery; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -19,6 +20,7 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.Warnings; import org.elasticsearch.compute.querydsl.query.SingleValueMatchQuery; +import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -30,6 +32,7 @@ import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Location; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.fulltext.Term; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; @@ -163,35 +166,6 @@ public Source source() { return source; } - protected abstract MappedFieldType mappedFieldType(SearchExecutionContext context); - - @Override - protected final org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) throws IOException { - MappedFieldType ft = mappedFieldType(context); - if (ft == null) { - return new MatchNoDocsQuery("missing field [" + field + "]"); - } - SingleValueMatchQuery singleValueQuery = new SingleValueMatchQuery( - context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH), - Warnings.createWarnings( - DriverContext.WarningsMode.COLLECT, - source.source().getLineNumber(), - source.source().getColumnNumber(), - source.text() - ), - "single-value function encountered multi-value" - ); - org.apache.lucene.search.Query rewrite = singleValueQuery.rewrite(context.searcher()); - if (rewrite instanceof MatchAllDocsQuery) { - // nothing to filter - return next.toQuery(context); - } - BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.add(next.toQuery(context), BooleanClause.Occur.FILTER); - builder.add(rewrite, BooleanClause.Occur.FILTER); - return builder.build(); - } - protected abstract AbstractBuilder rewrite(QueryBuilder next); @Override @@ -246,8 +220,30 @@ public TransportVersion getMinimalSupportedVersion() { } @Override - protected MappedFieldType mappedFieldType(SearchExecutionContext context) { - return context.getFieldType(field()); + protected final org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) throws IOException { + MappedFieldType ft = context.getFieldType(field()); + if (ft == null) { + return new MatchNoDocsQuery("missing field [" + field() + "]"); + } + SingleValueMatchQuery singleValueQuery = new SingleValueMatchQuery( + context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH), + Warnings.createWarnings( + DriverContext.WarningsMode.COLLECT, + source().source().getLineNumber(), + source().source().getColumnNumber(), + source().text() + ), + "single-value function encountered multi-value" + ); + org.apache.lucene.search.Query rewrite = singleValueQuery.rewrite(context.searcher()); + if (rewrite instanceof MatchAllDocsQuery) { + // nothing to filter + return next().toQuery(context); + } + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(next().toQuery(context), BooleanClause.Occur.FILTER); + builder.add(rewrite, BooleanClause.Occur.FILTER); + return builder.build(); } @Override @@ -261,13 +257,9 @@ public static class SyntheticSourceDelegateBuilder extends AbstractBuilder { super(next, field, source); } - SyntheticSourceDelegateBuilder(StreamInput in) throws IOException { - super(in); - } - @Override public String getWriteableName() { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("Not serialized"); } @Override @@ -281,12 +273,42 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep @Override public TransportVersion getMinimalSupportedVersion() { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("Not serialized"); } @Override - protected MappedFieldType mappedFieldType(SearchExecutionContext context) { - return ((TextFieldMapper.TextFieldType) context.getFieldType(field())).syntheticSourceDelegate(); + protected final org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) throws IOException { + MappedFieldType ft = context.getFieldType(field()); + if (ft == null) { + return new MatchNoDocsQuery("missing field [" + field() + "]"); + } + ft = ((TextFieldMapper.TextFieldType) ft).syntheticSourceDelegate(); + + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(next().toQuery(context), BooleanClause.Occur.FILTER); + + org.apache.lucene.search.Query singleValueQuery = new SingleValueMatchQuery( + context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH), + Warnings.createWarnings( + DriverContext.WarningsMode.COLLECT, + source().source().getLineNumber(), + source().source().getColumnNumber(), + source().text() + ), + "single-value function encountered multi-value" + ); + singleValueQuery = singleValueQuery.rewrite(context.searcher()); + if (singleValueQuery instanceof MatchAllDocsQuery == false) { + builder.add(singleValueQuery, BooleanClause.Occur.FILTER); + } + + org.apache.lucene.search.Query ignored = new TermQuery(new org.apache.lucene.index.Term(IgnoredFieldMapper.NAME, ft.name())); + ignored = ignored.rewrite(context.searcher()); + if (ignored instanceof MatchNoDocsQuery == false) { + builder.add(ignored, BooleanClause.Occur.MUST_NOT); + } + + return builder.build(); } @Override From a2455e97b562991a6787148655899d3ebb4ad51d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 14 Apr 2025 14:39:53 -0400 Subject: [PATCH 06/17] fmt --- .../xpack/esql/querydsl/query/SingleValueQuery.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index 1076e77ce5b8e..d07eb2096eb92 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -32,7 +32,6 @@ import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Location; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.expression.function.fulltext.Term; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; From 267632ba22c083a5985432db014769b945659440 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 14 Apr 2025 18:46:27 +0000 Subject: [PATCH 07/17] [CI] Auto commit changes from spotless --- .../xpack/esql/querydsl/query/SingleValueQuery.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index 1076e77ce5b8e..d07eb2096eb92 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -32,7 +32,6 @@ import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Location; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.expression.function.fulltext.Term; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; From 254158b338a96e6c2f4182694a818421d90771bc Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 14 Apr 2025 14:55:27 -0400 Subject: [PATCH 08/17] test! --- .../querydsl/query/SingleValueQueryTests.java | 61 +++++++++++++++---- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java index a48984bb310f2..458cb6d5a6bb9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.esql.core.querydsl.query.MatchAll; import org.elasticsearch.xpack.esql.core.querydsl.query.RangeQuery; +import org.elasticsearch.xpack.esql.core.querydsl.query.TermQuery; import org.elasticsearch.xpack.esql.core.tree.Source; import java.io.IOException; @@ -46,12 +47,14 @@ interface Setup { XContentBuilder mapping(XContentBuilder builder) throws IOException; List> build(RandomIndexWriter iw) throws IOException; + + boolean useSyntheticSourceDelegate(); } @ParametersFactory public static List params() { List params = new ArrayList<>(); - for (String fieldType : new String[] { "long", "integer", "short", "byte", "double", "float", "keyword" }) { + for (String fieldType : new String[] { "long", "integer", "short", "byte", "double", "float", "keyword", "text" }) { for (boolean multivaluedField : new boolean[] { true, false }) { for (boolean allowEmpty : new boolean[] { true, false }) { params.add(new Object[] { new StandardSetup(fieldType, multivaluedField, allowEmpty, 100) }); @@ -69,45 +72,54 @@ public SingleValueQueryTests(Setup setup) { } public void testMatchAll() throws IOException { - testCase(new SingleValueQuery(new MatchAll(Source.EMPTY), "foo", false).asBuilder(), this::runCase); + testCase(new SingleValueQuery(new MatchAll(Source.EMPTY), "foo", setup.useSyntheticSourceDelegate()).asBuilder(), this::runCase); } public void testMatchSome() throws IOException { int max = between(1, 100); testCase( - new SingleValueQuery.Builder(new RangeQueryBuilder("i").lt(max), "foo", Source.EMPTY), + new SingleValueQuery( + new RangeQuery(Source.EMPTY, "i", null, false, max, false, randomZone()), + "foo", + setup.useSyntheticSourceDelegate() + ).asBuilder(), (fieldValues, count) -> runCase(fieldValues, count, null, max) ); } public void testSubPhrase() throws IOException { - testCase(new SingleValueQuery.Builder(new MatchPhraseQueryBuilder("str", "fox jumped"), "foo", Source.EMPTY), this::runCase); + SingleValueQuery.AbstractBuilder builder = setup.useSyntheticSourceDelegate() + ? new SingleValueQuery.SyntheticSourceDelegateBuilder(new MatchPhraseQueryBuilder("str", "fox jumped"), "foo", Source.EMPTY) + : new SingleValueQuery.Builder(new MatchPhraseQueryBuilder("str", "fox jumped"), "foo", Source.EMPTY); + testCase(builder, this::runCase); } public void testMatchNone() throws IOException { testCase( - new SingleValueQuery.Builder(new MatchNoneQueryBuilder(), "foo", Source.EMPTY), + new SingleValueQuery(new MatchAll(Source.EMPTY).negate(Source.EMPTY), "foo", setup.useSyntheticSourceDelegate()).asBuilder(), (fieldValues, count) -> assertThat(count, equalTo(0)) ); } public void testRewritesToMatchNone() throws IOException { testCase( - new SingleValueQuery.Builder(new TermQueryBuilder("missing", 0), "foo", Source.EMPTY), + new SingleValueQuery(new TermQuery(Source.EMPTY, "missing", 0), "foo", setup.useSyntheticSourceDelegate()).asBuilder(), (fieldValues, count) -> assertThat(count, equalTo(0)) ); } public void testNotMatchAll() throws IOException { testCase( - new SingleValueQuery(new MatchAll(Source.EMPTY), "foo", false).negate(Source.EMPTY).asBuilder(), + new SingleValueQuery(new MatchAll(Source.EMPTY), "foo", setup.useSyntheticSourceDelegate()).negate(Source.EMPTY).asBuilder(), (fieldValues, count) -> assertThat(count, equalTo(0)) ); } public void testNotMatchNone() throws IOException { testCase( - new SingleValueQuery(new MatchAll(Source.EMPTY).negate(Source.EMPTY), "foo", false).negate(Source.EMPTY).asBuilder(), + new SingleValueQuery(new MatchAll(Source.EMPTY).negate(Source.EMPTY), "foo", setup.useSyntheticSourceDelegate()).negate( + Source.EMPTY + ).asBuilder(), this::runCase ); } @@ -115,8 +127,11 @@ public void testNotMatchNone() throws IOException { public void testNotMatchSome() throws IOException { int max = between(1, 100); testCase( - new SingleValueQuery(new RangeQuery(Source.EMPTY, "i", null, false, max, false, null), "foo", false).negate(Source.EMPTY) - .asBuilder(), + new SingleValueQuery( + new RangeQuery(Source.EMPTY, "i", null, false, max, false, null), + "foo", + setup.useSyntheticSourceDelegate() + ).negate(Source.EMPTY).asBuilder(), (fieldValues, count) -> runCase(fieldValues, count, max, 100) ); } @@ -186,7 +201,13 @@ private record StandardSetup(String fieldType, boolean multivaluedField, boolean public XContentBuilder mapping(XContentBuilder builder) throws IOException { builder.startObject("i").field("type", "long").endObject(); builder.startObject("str").field("type", "text").endObject(); - return builder.startObject("foo").field("type", fieldType).endObject(); + builder.startObject("foo").field("type", fieldType); + if (fieldType.equals("text")) { + builder.startObject("fields"); + builder.startObject("raw").field("type", "keyword").field("ignore_above", 256).endObject(); + builder.endObject(); + } + return builder.endObject(); } @Override @@ -200,6 +221,11 @@ public List> build(RandomIndexWriter iw) throws IOException { return fieldValues; } + @Override + public boolean useSyntheticSourceDelegate() { + return fieldType.equals("text"); + } + private List values(int i) { // i == 10 forces at least one multivalued field when we're configured for multivalued fields boolean makeMultivalued = multivaluedField && (i == 10 || randomBoolean()); @@ -226,7 +252,7 @@ private Object randomValue() { case "byte" -> randomByte(); case "double" -> randomDouble(); case "float" -> randomFloat(); - case "keyword" -> randomAlphaOfLength(5); + case "keyword", "text" -> randomAlphaOfLength(5); default -> throw new UnsupportedOperationException(); }; } @@ -253,6 +279,12 @@ private List docFor(int i, Iterable values) { fields.add(new KeywordField("foo", v.toString(), Field.Store.NO)); } } + case "text" -> { + for (Object v : values) { + fields.add(new TextField("foo", v.toString(), Field.Store.NO)); + fields.add(new KeywordField("foo.raw", v.toString(), Field.Store.NO)); + } + } default -> throw new UnsupportedOperationException(); } return fields; @@ -280,5 +312,10 @@ public List> build(RandomIndexWriter iw) throws IOException { } return fieldValues; } + + @Override + public boolean useSyntheticSourceDelegate() { + return randomBoolean(); + } } } From 1b34eb27a1788ad553283f15b71e2fd6c08d1a43 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Mon, 14 Apr 2025 19:04:41 +0000 Subject: [PATCH 09/17] [CI] Auto commit changes from spotless --- .../xpack/esql/querydsl/query/SingleValueQueryTests.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java index 458cb6d5a6bb9..6d5bb69502c27 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQueryTests.java @@ -22,12 +22,9 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; -import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.MatchPhraseQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.esql.core.querydsl.query.MatchAll; import org.elasticsearch.xpack.esql.core.querydsl.query.RangeQuery; From 600257acc0cce1cfe79cd2da5715b7e6a6fee4e2 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 14 Apr 2025 15:05:47 -0400 Subject: [PATCH 10/17] Explain --- .../esql/querydsl/query/SingleValueQuery.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index d07eb2096eb92..39032e4fa61c1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -190,6 +190,9 @@ protected final int doHashCode() { } } + /** + * Builds a {@code bool} query combining the "next" query and a {@link SingleValueMatchQuery}. + */ public static class Builder extends AbstractBuilder { Builder(QueryBuilder next, String field, Source source) { super(next, field, source); @@ -251,6 +254,29 @@ protected AbstractBuilder rewrite(QueryBuilder next) { } } + /** + * Builds a {@code bool} query combining the "next" query, a {@link SingleValueMatchQuery}, + * and a {@link TermQuery} making sure we didn't ignore any values. + *

+ * This is used in the case when you do {@code text_field == "foo"} and {@code text_field} + * has a {@code keyword} sub-field. See, {@code text} typed fields can't do our equality - + * they only do matching. But {@code keyword} fields *can* do the equality. In this case + * the "next" query is a {@link TermQuery} like {@code text_field.raw:foo}. + *

+ *

+ * But there's a big wrinkle! If you index a field longer than {@code ignore_above} into + * {@code text_field.raw} field then it'll drop its value on the floor. So the + * {@link SingleValueMatchQuery} isn't enough to emulate {@code ==}. You have to remove + * any matches that ignored a field. Luckilly we have {@link IgnoredFieldMapper}! We can + * do a {@link TermQuery} like {@code NOT(_ignored:text_field.raw)} to filter those out. + *

+ *

+ * You may be asking "how would the first {@code text_field.raw:foo} query work if the + * value we're searching for is very long? In that case we never use this query at all. + * We have to delegate the filtering to the compute engine. No fancy lucene searches in + * that case. + *

+ */ public static class SyntheticSourceDelegateBuilder extends AbstractBuilder { SyntheticSourceDelegateBuilder(QueryBuilder next, String field, Source source) { super(next, field, source); From 885598f39997202bd10ea827824f7fb2900ed211 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 14 Apr 2025 17:43:05 -0400 Subject: [PATCH 11/17] Fix test --- .../org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 774f05e6318c5..9f4e8c9f8b9ba 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -440,7 +440,7 @@ public void testPushEqualityOnDefaults() throws IOException { List> operators = (List>) p.get("operators"); for (Map o : operators) { // The query here is the most important bit - we *do* push to lucene. - sig.add(checkOperatorProfile(o, "test.keyword:" + value)); + sig.add(checkOperatorProfile(o, "#test.keyword:" + value + " -_ignored:test.keyword")); } String description = p.get("description").toString(); switch (description) { From c85680a763b5226d1f860e6e341a5eb28df72d0c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 15 Apr 2025 13:20:25 -0400 Subject: [PATCH 12/17] fixup --- .../esql/expression/function/fulltext/MultiMatchTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MultiMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MultiMatchTests.java index dcf8d94834efa..e14abda61ca63 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MultiMatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/MultiMatchTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import java.util.List; import java.util.function.Supplier; @@ -38,7 +39,7 @@ protected Expression build(Source source, List args) { // We need to add the QueryBuilder to the multi_match expression, as it is used to implement equals() and hashCode() and // thus test the serialization methods. But we can only do this if the parameters make sense . if (mm.query().foldable() && mm.fields().stream().allMatch(field -> field instanceof FieldAttribute)) { - QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(mm).toQueryBuilder(); + QueryBuilder queryBuilder = TRANSLATOR_HANDLER.asQuery(LucenePushdownPredicates.DEFAULT, mm).toQueryBuilder(); mm.replaceQueryBuilder(queryBuilder); } return mm; From 28c3b5b006f46943341b70a1bf3f44d7b95e1d2f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 15 Apr 2025 17:32:58 -0400 Subject: [PATCH 13/17] Update --- .../xpack/esql/qa/single_node/RestEsqlIT.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 0e9c7aebff678..810ce4a86ad80 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -422,7 +422,12 @@ public void testPushEqualityOnDefaults() throws IOException { Map result = runEsql(builder); assertResultMap( result, - getResultMatcher(result).entry("profile", matchesMap().entry("drivers", instanceOf(List.class))), + getResultMatcher(result).entry( + "profile", + matchesMap().entry("drivers", instanceOf(List.class)) + .entry("planning", matchesMap().extraOk()) + .entry("query", matchesMap().extraOk()) + ), matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) .item(matchesMap().entry("name", "test").entry("type", "text")) .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) @@ -474,7 +479,12 @@ public void testPushEqualityOnDefaultsTooBigToPush() throws IOException { Map result = runEsql(builder); assertResultMap( result, - getResultMatcher(result).entry("profile", matchesMap().entry("drivers", instanceOf(List.class))), + getResultMatcher(result).entry( + "profile", + matchesMap().entry("drivers", instanceOf(List.class)) + .entry("planning", matchesMap().extraOk()) + .entry("query", matchesMap().extraOk()) + ), matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) .item(matchesMap().entry("name", "test").entry("type", "text")) .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) From 5ddb10cbc164fcb5589619e7ded98f9d888ab15d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 16 Apr 2025 15:18:27 -0400 Subject: [PATCH 14/17] WIP --- .../xpack/esql/qa/single_node/RestEsqlIT.java | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 29923ca60da13..b16231b812233 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -528,6 +528,66 @@ public void testPushEqualityOnDefaultsTooBigToPush() throws IOException { } } + public void testPushCaseInsensitiveEqualityOnDefaults() throws IOException { + indexTimestampData(1); + + String value = "a".repeat(between(0, 256)); + indexValue(value); + + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | WHERE TO_LOWER(test) == \"" + value + "\""); + builder.profile(true); + Map result = runEsql(builder); + assertResultMap( + result, + getResultMatcher(result).entry( + "profile", + matchesMap().entry("drivers", instanceOf(List.class)) + .entry("planning", matchesMap().extraOk()) + .entry("query", matchesMap().extraOk()) + ), + matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) + .item(matchesMap().entry("name", "test").entry("type", "text")) + .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) + .item(matchesMap().entry("name", "value").entry("type", "long")), + equalTo(List.of(Arrays.asList(null, value, null, null))) + ); + + @SuppressWarnings("unchecked") + List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); + for (Map p : profiles) { + fixTypesOnProfile(p); + assertThat(p, commonProfile()); + List sig = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> operators = (List>) p.get("operators"); + for (Map o : operators) { + // The query here is the most important bit - we do *not* push to lucene. + sig.add(checkOperatorProfile(o, "*:*")); + } + String description = p.get("description").toString(); + switch (description) { + case "data" -> assertMap( + sig, + matchesList().item("LuceneSourceOperator") + .item("ValuesSourceReaderOperator") + .item("FilterOperator") + .item("LimitOperator") + .item("ValuesSourceReaderOperator") + .item("ProjectOperator") + .item("ExchangeSinkOperator") + ); + case "node_reduce" -> assertThat( + sig, + either(matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator")).or( + matchesList().item("ExchangeSourceOperator").item("AggregationOperator").item("ExchangeSinkOperator") + ) + ); + case "final" -> assertMap(sig, matchesList().item("ExchangeSourceOperator").item("LimitOperator").item("OutputOperator")); + default -> throw new IllegalArgumentException("can't match " + description); + } + } + } + private void indexValue(String value) throws IOException { Request bulk = new Request("POST", "/_bulk"); bulk.addParameter("filter_path", "errors"); @@ -806,7 +866,7 @@ private void fixTypesOnProfile(Map profile) { private String checkOperatorProfile(Map o, String query) { String name = (String) o.get("operator"); - name = name.replaceAll("\\[.+", ""); + name = name.replaceAll("\\[(.|\n)+", ""); MapMatcher status = switch (name) { case "LuceneSourceOperator" -> matchesMap().entry("processed_slices", greaterThan(0)) .entry("processed_shards", List.of(testIndexName() + ":0")) From 4c082e99de7f7ea3cfce2cf6ac67966b8dd1fa98 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 18 Apr 2025 11:05:30 -0400 Subject: [PATCH 15/17] Move test --- .../esql/qa/single_node/PushQueriesIT.java | 168 ++++++++++++++ .../xpack/esql/qa/single_node/RestEsqlIT.java | 205 +----------------- .../xpack/esql/qa/rest/RestEsqlTestCase.java | 3 +- 3 files changed, 178 insertions(+), 198 deletions(-) create mode 100644 x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java new file mode 100644 index 0000000000000..6d264c83c4239 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.single_node; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.test.ListMatcher; +import org.elasticsearch.test.MapMatcher; +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.esql.AssertWarnings; +import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import static org.elasticsearch.test.ListMatcher.matchesList; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.entityToMap; +import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.requestObjectBuilder; +import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.runEsql; +import static org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT.commonProfile; +import static org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT.fixTypesOnProfile; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.startsWith; + +/** + * Tests for pushing queries to lucene. + */ +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class PushQueriesIT extends ESRestTestCase { + @ClassRule + public static ElasticsearchCluster cluster = Clusters.testCluster(); + + public void testPushEqualityOnDefaults() throws IOException { + String value = "v".repeat(between(0, 256)); + testPushQuery(value, """ + FROM test + | WHERE test == "%value" + """, "#test.keyword:%value -_ignored:test.keyword", false); + } + + public void testPushEqualityOnDefaultsTooBigToPush() throws IOException { + String value = "a".repeat(between(257, 1000)); + testPushQuery(value, """ + FROM test + | WHERE test == "%value" + """, "*:*", true); + } + + public void testPushCaseInsensitiveEqualityOnDefaults() throws IOException { + String value = "a".repeat(between(0, 256)); + testPushQuery(value, """ + FROM test + | WHERE TO_LOWER(test) == "%value" + """, "*:*", true); + } + + private void testPushQuery(String value, String esqlQuery, String luceneQuery, boolean filterInCompute) throws IOException { + indexValue(value); + + RestEsqlTestCase.RequestObjectBuilder builder = requestObjectBuilder().query( + esqlQuery.replaceAll("%value", value) + "\n| KEEP test" + ); + builder.profile(true); + Map result = runEsql(builder, new AssertWarnings.NoWarnings(), RestEsqlTestCase.Mode.SYNC); + assertResultMap( + result, + getResultMatcher(result).entry( + "profile", + matchesMap().entry("drivers", instanceOf(List.class)) + .entry("planning", matchesMap().extraOk()) + .entry("query", matchesMap().extraOk()) + ), + matchesList().item(matchesMap().entry("name", "test").entry("type", "text")), + equalTo(List.of(List.of(value))) + ); + + @SuppressWarnings("unchecked") + List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); + for (Map p : profiles) { + fixTypesOnProfile(p); + assertThat(p, commonProfile()); + List sig = new ArrayList<>(); + @SuppressWarnings("unchecked") + List> operators = (List>) p.get("operators"); + for (Map o : operators) { + sig.add(checkOperatorProfile(o, luceneQuery.replaceAll("%value", value))); + } + String description = p.get("description").toString(); + switch (description) { + case "data" -> { + ListMatcher matcher = matchesList().item("LuceneSourceOperator").item("ValuesSourceReaderOperator"); + if (filterInCompute) { + matcher = matcher.item("FilterOperator").item("LimitOperator"); + } + matcher = matcher.item("ProjectOperator").item("ExchangeSinkOperator"); + assertMap(sig, matcher); + } + case "node_reduce" -> assertMap(sig, matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator")); + case "final" -> assertMap( + sig, + matchesList().item("ExchangeSourceOperator").item("LimitOperator").item("ProjectOperator").item("OutputOperator") + ); + default -> throw new IllegalArgumentException("can't match " + description); + } + } + } + + private void indexValue(String value) throws IOException { + Request createIndex = new Request("PUT", "test"); + createIndex.setJsonEntity(""" + { + "settings": { + "index": { + "number_of_shards": 1 + } + } + }"""); + Response createResponse = client().performRequest(createIndex); + assertThat( + entityToMap(createResponse.getEntity(), XContentType.JSON), + matchesMap().entry("shards_acknowledged", true).entry("index", "test").entry("acknowledged", true) + ); + + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("refresh", ""); + bulk.setJsonEntity(String.format(""" + {"create":{"_index":"test"}} + {"test":"%s"} + """, value)); + Response bulkResponse = client().performRequest(bulk); + assertThat(entityToMap(bulkResponse.getEntity(), XContentType.JSON), matchesMap().entry("errors", false).extraOk()); + } + + private static final Pattern TO_NAME = Pattern.compile("\\[.+", Pattern.DOTALL); + + private static String checkOperatorProfile(Map o, String query) { + String name = (String) o.get("operator"); + name = TO_NAME.matcher(name).replaceAll(""); + if (name.equals("LuceneSourceOperator")) { + MapMatcher expectedOp = matchesMap().entry("operator", startsWith(name)) + .entry("status", matchesMap().entry("processed_queries", List.of(query)).extraOk()); + assertMap(o, expectedOp); + } + return name; + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index b16231b812233..56f34d4178768 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -313,7 +313,7 @@ public void testProfile() throws IOException { @SuppressWarnings("unchecked") List> operators = (List>) p.get("operators"); for (Map o : operators) { - sig.add(checkOperatorProfile(o, "*:*")); + sig.add(checkOperatorProfile(o)); } String description = p.get("description").toString(); switch (description) { @@ -411,195 +411,6 @@ public void testProfileParsing() throws IOException { } } - public void testPushEqualityOnDefaults() throws IOException { - indexTimestampData(1); - - String value = "v".repeat(between(0, 256)); - indexValue(value); - - RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | WHERE test == \"" + value + "\""); - builder.profile(true); - Map result = runEsql(builder); - assertResultMap( - result, - getResultMatcher(result).entry( - "profile", - matchesMap().entry("drivers", instanceOf(List.class)) - .entry("planning", matchesMap().extraOk()) - .entry("query", matchesMap().extraOk()) - ), - matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) - .item(matchesMap().entry("name", "test").entry("type", "text")) - .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) - .item(matchesMap().entry("name", "value").entry("type", "long")), - equalTo(List.of(Arrays.asList(null, value, value, null))) - ); - - @SuppressWarnings("unchecked") - List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); - for (Map p : profiles) { - fixTypesOnProfile(p); - assertThat(p, commonProfile()); - List sig = new ArrayList<>(); - @SuppressWarnings("unchecked") - List> operators = (List>) p.get("operators"); - for (Map o : operators) { - // The query here is the most important bit - we *do* push to lucene. - sig.add(checkOperatorProfile(o, "#test.keyword:" + value + " -_ignored:test.keyword")); - } - String description = p.get("description").toString(); - switch (description) { - case "data" -> assertMap( - sig, - matchesList().item("LuceneSourceOperator") - .item("ValuesSourceReaderOperator") - .item("ProjectOperator") - .item("ExchangeSinkOperator") - ); - case "node_reduce" -> assertThat( - sig, - either(matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator")).or( - matchesList().item("ExchangeSourceOperator").item("AggregationOperator").item("ExchangeSinkOperator") - ) - ); - case "final" -> assertMap(sig, matchesList().item("ExchangeSourceOperator").item("LimitOperator").item("OutputOperator")); - default -> throw new IllegalArgumentException("can't match " + description); - } - } - } - - public void testPushEqualityOnDefaultsTooBigToPush() throws IOException { - indexTimestampData(1); - - String value = "a".repeat(between(257, 1000)); - indexValue(value); - - RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | WHERE test == \"" + value + "\""); - builder.profile(true); - Map result = runEsql(builder); - assertResultMap( - result, - getResultMatcher(result).entry( - "profile", - matchesMap().entry("drivers", instanceOf(List.class)) - .entry("planning", matchesMap().extraOk()) - .entry("query", matchesMap().extraOk()) - ), - matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) - .item(matchesMap().entry("name", "test").entry("type", "text")) - .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) - .item(matchesMap().entry("name", "value").entry("type", "long")), - equalTo(List.of(Arrays.asList(null, value, null, null))) - ); - - @SuppressWarnings("unchecked") - List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); - for (Map p : profiles) { - fixTypesOnProfile(p); - assertThat(p, commonProfile()); - List sig = new ArrayList<>(); - @SuppressWarnings("unchecked") - List> operators = (List>) p.get("operators"); - for (Map o : operators) { - // The query here is the most important bit - we do *not* push to lucene. - sig.add(checkOperatorProfile(o, "*:*")); - } - String description = p.get("description").toString(); - switch (description) { - case "data" -> assertMap( - sig, - matchesList().item("LuceneSourceOperator") - .item("ValuesSourceReaderOperator") - .item("FilterOperator") - .item("LimitOperator") - .item("ValuesSourceReaderOperator") - .item("ProjectOperator") - .item("ExchangeSinkOperator") - ); - case "node_reduce" -> assertThat( - sig, - either(matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator")).or( - matchesList().item("ExchangeSourceOperator").item("AggregationOperator").item("ExchangeSinkOperator") - ) - ); - case "final" -> assertMap(sig, matchesList().item("ExchangeSourceOperator").item("LimitOperator").item("OutputOperator")); - default -> throw new IllegalArgumentException("can't match " + description); - } - } - } - - public void testPushCaseInsensitiveEqualityOnDefaults() throws IOException { - indexTimestampData(1); - - String value = "a".repeat(between(0, 256)); - indexValue(value); - - RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | WHERE TO_LOWER(test) == \"" + value + "\""); - builder.profile(true); - Map result = runEsql(builder); - assertResultMap( - result, - getResultMatcher(result).entry( - "profile", - matchesMap().entry("drivers", instanceOf(List.class)) - .entry("planning", matchesMap().extraOk()) - .entry("query", matchesMap().extraOk()) - ), - matchesList().item(matchesMap().entry("name", "@timestamp").entry("type", "date")) - .item(matchesMap().entry("name", "test").entry("type", "text")) - .item(matchesMap().entry("name", "test.keyword").entry("type", "keyword")) - .item(matchesMap().entry("name", "value").entry("type", "long")), - equalTo(List.of(Arrays.asList(null, value, null, null))) - ); - - @SuppressWarnings("unchecked") - List> profiles = (List>) ((Map) result.get("profile")).get("drivers"); - for (Map p : profiles) { - fixTypesOnProfile(p); - assertThat(p, commonProfile()); - List sig = new ArrayList<>(); - @SuppressWarnings("unchecked") - List> operators = (List>) p.get("operators"); - for (Map o : operators) { - // The query here is the most important bit - we do *not* push to lucene. - sig.add(checkOperatorProfile(o, "*:*")); - } - String description = p.get("description").toString(); - switch (description) { - case "data" -> assertMap( - sig, - matchesList().item("LuceneSourceOperator") - .item("ValuesSourceReaderOperator") - .item("FilterOperator") - .item("LimitOperator") - .item("ValuesSourceReaderOperator") - .item("ProjectOperator") - .item("ExchangeSinkOperator") - ); - case "node_reduce" -> assertThat( - sig, - either(matchesList().item("ExchangeSourceOperator").item("ExchangeSinkOperator")).or( - matchesList().item("ExchangeSourceOperator").item("AggregationOperator").item("ExchangeSinkOperator") - ) - ); - case "final" -> assertMap(sig, matchesList().item("ExchangeSourceOperator").item("LimitOperator").item("OutputOperator")); - default -> throw new IllegalArgumentException("can't match " + description); - } - } - } - - private void indexValue(String value) throws IOException { - Request bulk = new Request("POST", "/_bulk"); - bulk.addParameter("filter_path", "errors"); - bulk.addParameter("refresh", ""); - bulk.setJsonEntity(String.format(""" - {"create":{"_index":"%s"}} - {"test":"%s"} - """, testIndexName(), value)); - Response response = client().performRequest(bulk); - Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); - } - @SuppressWarnings("unchecked") public void assertProcessMetadataForNextNode(Map nodeMetadata, Set expectedNamesForNodes, int seenNodes) { assertEquals("M", nodeMetadata.get("ph")); @@ -710,7 +521,7 @@ public void testInlineStatsProfile() throws IOException { @SuppressWarnings("unchecked") List> operators = (List>) p.get("operators"); for (Map o : operators) { - sig.add(checkOperatorProfile(o, "*:*")); + sig.add(checkOperatorProfile(o)); } signatures.add(sig); } @@ -837,7 +648,7 @@ public void testForceSleepsProfile() throws IOException { } } - private MapMatcher commonProfile() { + static MapMatcher commonProfile() { return matchesMap() // .entry("description", any(String.class)) .entry("cluster_name", any(String.class)) @@ -858,15 +669,15 @@ private MapMatcher commonProfile() { * come back as integers and sometimes longs. This just promotes * them to long every time. */ - private void fixTypesOnProfile(Map profile) { + static void fixTypesOnProfile(Map profile) { profile.put("iterations", ((Number) profile.get("iterations")).longValue()); profile.put("cpu_nanos", ((Number) profile.get("cpu_nanos")).longValue()); profile.put("took_nanos", ((Number) profile.get("took_nanos")).longValue()); } - private String checkOperatorProfile(Map o, String query) { + private String checkOperatorProfile(Map o) { String name = (String) o.get("operator"); - name = name.replaceAll("\\[(.|\n)+", ""); + name = name.replaceAll("\\[.+", ""); MapMatcher status = switch (name) { case "LuceneSourceOperator" -> matchesMap().entry("processed_slices", greaterThan(0)) .entry("processed_shards", List.of(testIndexName() + ":0")) @@ -878,7 +689,7 @@ private String checkOperatorProfile(Map o, String query) { .entry("pages_emitted", greaterThan(0)) .entry("rows_emitted", greaterThan(0)) .entry("process_nanos", greaterThan(0)) - .entry("processed_queries", List.of(query)) + .entry("processed_queries", List.of("*:*")) .entry("partitioning_strategies", matchesMap().entry("rest-esql-test:0", "SHARD")); case "ValuesSourceReaderOperator" -> basicProfile().entry("values_loaded", greaterThanOrEqualTo(0)) .entry("readers_built", matchesMap().extraOk()); @@ -891,7 +702,7 @@ private String checkOperatorProfile(Map o, String query) { case "ExchangeSourceOperator" -> matchesMap().entry("pages_waiting", 0) .entry("pages_emitted", greaterThan(0)) .entry("rows_emitted", greaterThan(0)); - case "ProjectOperator", "EvalOperator", "FilterOperator" -> basicProfile(); + case "ProjectOperator", "EvalOperator" -> basicProfile(); case "LimitOperator" -> matchesMap().entry("pages_processed", greaterThan(0)) .entry("limit", 1000) .entry("limit_remaining", 999) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 0cb522f1fb84d..32dab37b02c0d 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -1235,7 +1235,8 @@ public static Map runEsqlAsync(RequestObjectBuilder requestObjec return runEsqlAsync(requestObject, randomBoolean(), new AssertWarnings.NoWarnings()); } - static Map runEsql(RequestObjectBuilder requestObject, AssertWarnings assertWarnings, Mode mode) throws IOException { + public static Map runEsql(RequestObjectBuilder requestObject, AssertWarnings assertWarnings, Mode mode) + throws IOException { if (mode == ASYNC) { return runEsqlAsync(requestObject, randomBoolean(), assertWarnings); } else { From 0f9b54fb2ecf57f9641fbbf695cdda45808fa234 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 21 Apr 2025 09:26:49 -0400 Subject: [PATCH 16/17] Revert things we don't need --- .../xpack/esql/optimizer/PhysicalPlanOptimizerTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index d04a36f39cd19..88fa6d2605138 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -235,7 +235,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase { private final Configuration config; - public record TestDataSource(Map mapping, EsIndex index, Analyzer analyzer, SearchStats stats) {} + private record TestDataSource(Map mapping, EsIndex index, Analyzer analyzer, SearchStats stats) {} @ParametersFactory(argumentFormatting = PARAM_FORMATTING) public static List params() { @@ -7996,7 +7996,7 @@ private PhysicalPlan physicalPlan(String query) { return physicalPlan(query, testData); } - public PhysicalPlan physicalPlan(String query, TestDataSource dataSource) { + private PhysicalPlan physicalPlan(String query, TestDataSource dataSource) { return physicalPlan(query, dataSource, true); } From ac558f3c1a0eaa9a190b48b20d047df5dee357e6 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 21 Apr 2025 12:53:34 -0400 Subject: [PATCH 17/17] Updates --- .../testFixtures/src/main/resources/data/mv_text.csv | 1 + .../testFixtures/src/main/resources/string.csv-spec | 12 ++++++++++++ .../query/EqualsSyntheticSourceDelegate.java | 1 + .../xpack/esql/querydsl/query/SingleValueQuery.java | 10 ++++++---- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv index 1e24cf85881f7..222cc42d27cde 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/mv_text.csv @@ -2,3 +2,4 @@ 2023-10-23T13:55:01.543Z,[Connected to 10.1.0.1, Banana] 2023-10-23T13:55:01.544Z,Connected to 10.1.0.1 2023-10-23T13:55:01.545Z,[Connected to 10.1.0.1, More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100] +2023-10-23T13:55:01.546Z,More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index e16a213532b58..53b4d8c606a81 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -2320,3 +2320,15 @@ warning:Line 2:9: java.lang.IllegalArgumentException: single-value function enco @timestamp:date | message:text 2023-10-23T13:55:01.544Z|Connected to 10.1.0.1 ; + +mvStringEqualsLongString +FROM mv_text +| WHERE message == "More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100" +| KEEP @timestamp, message +; +warning:Line 2:9: evaluation of [message == \"More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100\"] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + + @timestamp:date | message:text +2023-10-23T13:55:01.546Z|More than one hundred characters long so it isn't indexed by the sub keyword field with ignore_above:100 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java index def564e21310a..1db12c873a763 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java @@ -53,6 +53,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { + // This is just translated on the data node and not sent over the wire. throw new UnsupportedOperationException(); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index 39032e4fa61c1..3af78787808f0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -256,7 +256,9 @@ protected AbstractBuilder rewrite(QueryBuilder next) { /** * Builds a {@code bool} query combining the "next" query, a {@link SingleValueMatchQuery}, - * and a {@link TermQuery} making sure we didn't ignore any values. + * and a {@link TermQuery} making sure we didn't ignore any values. Three total queries. + * This is only used if the "next" query matches fields that would not be ignored. Read all + * the paragraphs below to understand it. It's tricky! *

* This is used in the case when you do {@code text_field == "foo"} and {@code text_field} * has a {@code keyword} sub-field. See, {@code text} typed fields can't do our equality - @@ -267,12 +269,12 @@ protected AbstractBuilder rewrite(QueryBuilder next) { * But there's a big wrinkle! If you index a field longer than {@code ignore_above} into * {@code text_field.raw} field then it'll drop its value on the floor. So the * {@link SingleValueMatchQuery} isn't enough to emulate {@code ==}. You have to remove - * any matches that ignored a field. Luckilly we have {@link IgnoredFieldMapper}! We can + * any matches that ignored a field. Luckily we have {@link IgnoredFieldMapper}! We can * do a {@link TermQuery} like {@code NOT(_ignored:text_field.raw)} to filter those out. *

*

- * You may be asking "how would the first {@code text_field.raw:foo} query work if the - * value we're searching for is very long? In that case we never use this query at all. + * You may be asking, "how would the first {@code text_field.raw:foo} query work if the + * value we're searching for is very long?" In that case we never use this query at all. * We have to delegate the filtering to the compute engine. No fancy lucene searches in * that case. *