From fcf243a88740fc33dbcec846ac7e7762a9c352b1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 2 Jun 2025 08:35:15 -0400 Subject: [PATCH 01/11] WIP ``` text eq {"took": 13,"documents_found": 1000} not text eq {"took":4482,"documents_found":10000000} ``` --- .../index/mapper/TextFieldMapper.java | 39 +++++++- .../esql/qa/single_node/PushQueriesIT.java | 90 ++++++++++++++----- .../xpack/esql/EsqlTestUtils.java | 5 ++ .../esql/capabilities/TranslationAware.java | 17 +++- .../predicate/operator/comparison/Equals.java | 41 ++++++--- .../comparison/EsqlBinaryComparison.java | 12 +-- .../local/LucenePushdownPredicates.java | 13 +++ .../xpack/esql/planner/TranslatorHandler.java | 6 +- .../xpack/esql/stats/SearchContextStats.java | 18 ++++ .../xpack/esql/stats/SearchStats.java | 7 ++ 10 files changed, 194 insertions(+), 54 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 0e49506cd92e7..7364741a6c411 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -405,7 +405,8 @@ private TextFieldType buildFieldType( SyntheticSourceHelper.syntheticSourceDelegate(fieldType, multiFields), meta.getValue(), eagerGlobalOrdinals.getValue(), - indexPhrases.getValue() + indexPhrases.getValue(), + matchQueryYieldsCandidateMatchesForEquality() ); if (fieldData.getValue()) { ft.setFielddata(true, freqFilter.getValue()); @@ -414,6 +415,25 @@ private TextFieldType buildFieldType( return ft; } + /** + * Does a `match` query generate all valid candidates for `==`? Meaning, + * if I do a match query for any string, say `foo bar baz`, then that + * query will find all documents that indexed the same string. + *

+ * This should be true for most sanely configured text fields. That's + * just how we use them for search. But it's quite possible to make + * the index analyzer not agree with the search analyzer, for example. + *

+ *

+ * So this implementation is ultra-paranoid. + *

+ */ + private boolean matchQueryYieldsCandidateMatchesForEquality() { + return index.getValue() == Boolean.TRUE + && analyzers.indexAnalyzer.isConfigured() == false + && analyzers.searchAnalyzer.isConfigured() == false; + } + private SubFieldInfo buildPrefixInfo(MapperBuilderContext context, FieldType fieldType, TextFieldType tft) { if (indexPrefixes.get() == null) { return null; @@ -686,6 +706,9 @@ public static class TextFieldType extends StringFieldType { */ private final KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate; + // NOCOMMIT docs + private final boolean matchQueryYieldsCandidateMatchesForEquality; + public TextFieldType( String name, boolean indexed, @@ -695,7 +718,8 @@ public TextFieldType( KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate, Map meta, boolean eagerGlobalOrdinals, - boolean indexPhrases + boolean indexPhrases, + boolean matchQueryYieldsCandidateMatchesForEquality ) { super(name, indexed, stored, false, tsi, meta); fielddata = false; @@ -704,6 +728,7 @@ public TextFieldType( this.syntheticSourceDelegate = syntheticSourceDelegate; this.eagerGlobalOrdinals = eagerGlobalOrdinals; this.indexPhrases = indexPhrases; + this.matchQueryYieldsCandidateMatchesForEquality = matchQueryYieldsCandidateMatchesForEquality; } public TextFieldType(String name, boolean indexed, boolean stored, Map meta) { @@ -720,6 +745,7 @@ public TextFieldType(String name, boolean indexed, boolean stored, Map meta) { - super(name, indexed, stored, tsi, false, null, meta, false, false); + super(name, indexed, stored, tsi, false, null, meta, false, false, /* unused */ false); } public ConstantScoreTextFieldType(String name) { diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index b7e94b8590fe2..a6ec16bc017e6 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.qa.single_node; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.Repeat; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.elasticsearch.client.Request; @@ -50,6 +51,7 @@ /** * Tests for pushing queries to lucene. */ +//@Repeat(iterations = 10) @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class PushQueriesIT extends ESRestTestCase { @ClassRule @@ -57,7 +59,7 @@ public class PushQueriesIT extends ESRestTestCase { @ParametersFactory(argumentFormatting = "%1s") public static List args() { - return Stream.of("auto", "text", "match_only_text", "semantic_text").map(s -> new Object[] { s }).toList(); + return Stream.of("auto", "text_alone", "text", "match_only_text", "semantic_text").map(s -> new Object[] { s }).toList(); } private final String type; @@ -74,13 +76,14 @@ public void testEquality() throws IOException { """; String luceneQuery = switch (type) { case "text", "auto" -> "#test.keyword:%value -_ignored:test.keyword"; + case "text_alone" -> "test:%value"; case "match_only_text" -> "*:*"; case "semantic_text" -> "FieldExistsQuery [field=_primary_term]"; default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); }; boolean filterInCompute = switch (type) { case "text", "auto" -> false; - case "match_only_text", "semantic_text" -> true; + case "text_alone", "match_only_text", "semantic_text" -> true; default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); }; testPushQuery(value, esqlQuery, List.of(luceneQuery), filterInCompute, true); @@ -92,12 +95,21 @@ public void testEqualityTooBigToPush() throws IOException { FROM test | WHERE test == "%value" """; - String luceneQuery = switch (type) { - case "text", "auto", "match_only_text" -> "*:*"; - case "semantic_text" -> "FieldExistsQuery [field=_primary_term]"; + List luceneQueryOptions = switch (type) { + case "text", "auto" -> { + // We split tokens at 256 characters by in the standard analyzer. + String first = "#test:" + "a".repeat(255); + if (value.length() % 255 == 0) { + yield List.of(first); + } + String rest = "#test:" + "a".repeat(value.length() % 255); + yield List.of(first + " " + rest, rest + " " + first); + } + case "match_only_text" -> List.of("*:*"); + case "semantic_text" -> List.of("FieldExistsQuery [field=_primary_term]"); default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); }; - testPushQuery(value, esqlQuery, List.of(luceneQuery), true, true); + testPushQuery(value, esqlQuery, luceneQueryOptions, true, true); } /** @@ -195,6 +207,29 @@ public void testInequalityTooBigToPush() throws IOException { testPushQuery(value, esqlQuery, List.of(luceneQuery), true, false); } + /** + * {@code NOT !=} should function just like {@code ==}. + */ + public void testNotInequality() throws IOException { + String value = "v".repeat(between(0, 256)); + String esqlQuery = """ + FROM test + | WHERE NOT test != "%value" + """; + String luceneQuery = switch (type) { + case "text", "auto" -> "#test.keyword:%value -_ignored:test.keyword"; + case "match_only_text" -> "*:*"; + case "semantic_text" -> "FieldExistsQuery [field=_primary_term]"; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + boolean filterInCompute = switch (type) { + case "text", "auto" -> false; + case "match_only_text", "semantic_text" -> true; + default -> throw new UnsupportedOperationException("unknown type [" + type + "]"); + }; + testPushQuery(value, esqlQuery, List.of(luceneQuery), filterInCompute, true); + } + public void testCaseInsensitiveEquality() throws IOException { String value = "a".repeat(between(0, 256)); String esqlQuery = """ @@ -217,6 +252,7 @@ private void testPushQuery(String value, String esqlQuery, List luceneQu String replacedQuery = esqlQuery.replaceAll("%value", value).replaceAll("%different_value", differentValue); RestEsqlTestCase.RequestObjectBuilder builder = requestObjectBuilder().query(replacedQuery + "\n| KEEP test"); builder.profile(true); + builder.allowPartialResults(false); Map result = runEsql(builder, new AssertWarnings.NoWarnings(), RestEsqlTestCase.Mode.SYNC); assertResultMap( result, @@ -310,22 +346,36 @@ private void indexValue(String value) throws IOException { } } }"""; - default -> """ - , - "mappings": { - "properties": { - "test": { - "type": "%type", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + default -> { + if (type.endsWith("_alone")) { + yield """ + , + "mappings": { + "properties": { + "test": { + "type": "%type" } } } - } - } - }""".replace("%type", type); + }""".replace("%type", type.replace("_alone", "")); + } + yield """ + , + "mappings": { + "properties": { + "test": { + "type": "%type", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + }""".replace("%type", type); + } }; json += "}"; createIndex.setJsonEntity(json); @@ -365,7 +415,7 @@ protected String getTestRestCluster() { @Override protected boolean preserveClusterUponCompletion() { - // Preserve the cluser to speed up the semantic_text tests + // Preserve the cluster to speed up the semantic_text tests return true; } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 7b5843704eac8..7fc0e3d144ff2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -302,6 +302,11 @@ public boolean isSingleValue(FieldName field) { public boolean canUseEqualityOnSyntheticSourceDelegate(FieldName name, String value) { return false; } + + @Override + public boolean matchQueryYieldsCandidateMatchesForEquality(String name) { + return false; + } } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java index 1a2fd81db4b6c..6c02d6926da18 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java @@ -11,6 +11,7 @@ import org.elasticsearch.compute.operator.FilterOperator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.planner.TranslatorHandler; @@ -117,11 +118,19 @@ public FinishedTranslatable finish() { return finish; } + /** + * Essentially the {@link TranslationAware#translatable} + * implementation for the {@link Not} expression. When you wrap an expression + * in {@link Not} the result is mostly pushable in the same + * way as the original expression. But there are some expressions that aren't + * need rechecks or can't be pushed at all. This handles that. + */ public Translatable negate() { - if (this == YES_BUT_RECHECK_NEGATED) { - return RECHECK; - } - return this; + return switch (this) { + case YES_BUT_RECHECK_NEGATED -> Translatable.RECHECK; + case RECHECK -> Translatable.NO; + default -> this; + }; } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index 01f57342da1f0..b05ad47d585f3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -24,6 +25,7 @@ import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import org.elasticsearch.xpack.esql.querydsl.query.EqualsSyntheticSourceDelegate; +import org.elasticsearch.xpack.esql.querydsl.query.MatchQuery; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import java.time.ZoneId; @@ -129,26 +131,37 @@ public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { @Override public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { - if (right() instanceof Literal lit) { - if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) { - if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, ((BytesRef) lit.value()).utf8ToString())) { - return Translatable.YES_BUT_RECHECK_NEGATED; - } - } + if (right() instanceof Literal rhs && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute lhs) { + return translatableText(pushdownPredicates, lhs, ((BytesRef) rhs.value()).utf8ToString()); + } + return super.translatable(pushdownPredicates); + } + + private Translatable translatableText(LucenePushdownPredicates pushdownPredicates, FieldAttribute lhs, String rhs) { + if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(lhs, rhs)) { + return Translatable.YES_BUT_RECHECK_NEGATED; + } + if (pushdownPredicates.matchQueryYieldsCandidateMatchesForEquality(lhs)) { + return Translatable.RECHECK; } return super.translatable(pushdownPredicates); } @Override public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { - if (right() instanceof Literal lit) { - if (left().dataType() == DataType.TEXT && left() instanceof FieldAttribute fa) { - String value = ((BytesRef) lit.value()).utf8ToString(); - if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(fa, value)) { - String name = handler.nameOf(fa); - return new SingleValueQuery(new EqualsSyntheticSourceDelegate(source(), name, value), name, true); - } - } + if (right() instanceof Literal rhs && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute lhs) { + return asQueryText(pushdownPredicates, handler, lhs, ((BytesRef) rhs.value()).utf8ToString()); + } + return super.asQuery(pushdownPredicates, handler); + } + + private Query asQueryText(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler, FieldAttribute lhs, String rhs) { + String name = handler.nameOf(lhs); + if (pushdownPredicates.canUseEqualityOnSyntheticSourceDelegate(lhs, rhs)) { + return new SingleValueQuery(new EqualsSyntheticSourceDelegate(source(), name, rhs), name, true); + } + if (pushdownPredicates.matchQueryYieldsCandidateMatchesForEquality(lhs)) { + return new MatchQuery(source(), name, rhs, Map.of(MatchQueryBuilder.OPERATOR_FIELD.getPreferredName(), "AND")); } return super.asQuery(pushdownPredicates, handler); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index 69ef99ba04d15..f2562a66ef125 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -66,10 +66,7 @@ import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString; -public abstract class EsqlBinaryComparison extends BinaryComparison - implements - EvaluatorMapper, - TranslationAware.SingleValueTranslationAware { +public abstract class EsqlBinaryComparison extends BinaryComparison implements EvaluatorMapper, TranslationAware { private static final Logger logger = LogManager.getLogger(EsqlBinaryComparison.class); @@ -367,12 +364,7 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand ); Query translated = translateOutOfRangeComparisons(); - return translated != null ? translated : translate(handler); - } - - @Override - public Expression singleValueField() { - return left(); + return translated != null ? translated : handler.forceToSingleValueQuery(left(), translate(handler)); } private Query translate(TranslatorHandler handler) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index a476086980534..411ff5b5f60e6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -49,6 +49,9 @@ public interface LucenePushdownPredicates { boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute attr, String value); + // NOCOMMIT javadoc this and above + boolean matchQueryYieldsCandidateMatchesForEquality(FieldAttribute attr); + /** * We see fields as pushable if either they are aggregatable or they are indexed. * This covers non-indexed cases like AbstractScriptFieldType which hard-coded isAggregatable to true, @@ -123,6 +126,11 @@ public boolean isIndexed(FieldAttribute attr) { public boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute attr, String value) { return false; } + + @Override + public boolean matchQueryYieldsCandidateMatchesForEquality(FieldAttribute attr) { + return false; + } }; /** @@ -157,6 +165,11 @@ public boolean isIndexed(FieldAttribute attr) { public boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute attr, String value) { return stats.canUseEqualityOnSyntheticSourceDelegate(attr.fieldName(), value); } + + @Override + public boolean matchQueryYieldsCandidateMatchesForEquality(FieldAttribute attr) { + return stats.matchQueryYieldsCandidateMatchesForEquality(attr.field().getName()); + } }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java index 4b7af5bf49de8..f88b3a748d05e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java @@ -33,13 +33,15 @@ private TranslatorHandler() {} public Query asQuery(LucenePushdownPredicates predicates, Expression e) { if (e instanceof TranslationAware ta) { Query query = ta.asQuery(predicates, this); - return ta instanceof TranslationAware.SingleValueTranslationAware sv ? wrapFunctionQuery(sv.singleValueField(), query) : query; + return ta instanceof TranslationAware.SingleValueTranslationAware sv + ? forceToSingleValueQuery(sv.singleValueField(), query) + : query; } throw new QlIllegalArgumentException("Don't know how to translate {} {}", e.nodeName(), e); } - private static Query wrapFunctionQuery(Expression field, Query query) { + public Query forceToSingleValueQuery(Expression field, Query query) { if (query instanceof SingleValueQuery) { // Already wrapped return query; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java index c9766bad1c602..9fe06a88c29c2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java @@ -322,6 +322,24 @@ public boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute.FieldName return true; } + @Override + public boolean matchQueryYieldsCandidateMatchesForEquality(String name) { + for (SearchExecutionContext ctx : contexts) { + MappedFieldType type = ctx.getFieldType(name); + if (type == null) { + return false; + } + if (type instanceof TextFieldMapper.TextFieldType t) { + if (false == t.matchQueryYieldsCandidateMatchesForEquality()) { + return false; + } + } else { + return false; + } + } + return true; + } + public String constantValue(String name) { String val = null; for (SearchExecutionContext ctx : contexts) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchStats.java index ff1701104eca9..dadf6e5fe1a6e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchStats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchStats.java @@ -41,6 +41,8 @@ public interface SearchStats { boolean canUseEqualityOnSyntheticSourceDelegate(FieldName name, String value); + boolean matchQueryYieldsCandidateMatchesForEquality(String name); + /** * Returns the value for a field if it's a constant (eg. a constant_keyword with only one value for the involved indices). * NULL if the field is not a constant. @@ -108,5 +110,10 @@ public boolean isSingleValue(FieldName field) { public boolean canUseEqualityOnSyntheticSourceDelegate(FieldName name, String value) { return false; } + + @Override + public boolean matchQueryYieldsCandidateMatchesForEquality(String name) { + return false; + } } } From 0c0f77efdae10719cae5a7d74f57fe79adc6ef20 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 16 Jun 2025 11:40:42 -0400 Subject: [PATCH 02/11] Name --- .../esql/qa/single_node/PushQueriesIT.java | 86 ++++++++++++------- 1 file changed, 57 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index 55b52c2bbf10a..fa05b4617e32e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -63,9 +63,13 @@ public static List args() { public enum Type { AUTO(false), + CONSTANT_KEYWORD(false), KEYWORD(false), - // NOCOMMIT all these without keyword + MATCH_ONLY_TEXT(false), + SEMANTIC_TEXT(true), + TEXT(false), + MATCH_ONLY_TEXT_WITH_KEYWORD(false), SEMANTIC_TEXT_WITH_KEYWORD(true), TEXT_WITH_KEYWORD(false); @@ -91,13 +95,14 @@ public void testEquality() throws IOException { """; String luceneQuery = switch (type) { case AUTO, TEXT_WITH_KEYWORD -> "#test.keyword:%value -_ignored:test.keyword"; - case KEYWORD -> "test:%value"; - case CONSTANT_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD -> "*:*"; - case SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; + case KEYWORD, TEXT -> "test:%value"; + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD -> "*:*"; + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; }; ComputeSignature dataNodeSignature = switch (type) { case AUTO, CONSTANT_KEYWORD, KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_QUERY; - case MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; + case MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, TEXT, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> + ComputeSignature.FILTER_IN_COMPUTE; }; testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true); } @@ -109,14 +114,15 @@ public void testEqualityTooBigToPush() throws IOException { | WHERE test == "%value" """; List luceneQuery = switch (type) { - case AUTO, TEXT_WITH_KEYWORD -> emulateLargeTextTokens(value); - case CONSTANT_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD -> List.of("*:*"); + case AUTO, TEXT, TEXT_WITH_KEYWORD -> emulateLargeTextTokens(value); + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD -> List.of("*:*"); case KEYWORD -> List.of("#test:%value #single_value_match(test)"); - case SEMANTIC_TEXT_WITH_KEYWORD -> List.of("FieldExistsQuery [field=_primary_term]"); + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> List.of("FieldExistsQuery [field=_primary_term]"); }; ComputeSignature dataNodeSignature = switch (type) { case CONSTANT_KEYWORD, KEYWORD -> ComputeSignature.FILTER_IN_QUERY; - case AUTO, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT_WITH_KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; + case AUTO, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD, TEXT, TEXT_WITH_KEYWORD -> + ComputeSignature.FILTER_IN_COMPUTE; }; testPushQuery(value, esqlQuery, luceneQuery, dataNodeSignature, type != Type.KEYWORD); } @@ -125,7 +131,7 @@ public void testEqualityTooBigToPush() throws IOException { * {@code NOT !=} should function just like {@code ==}. */ public void testNotInequality() throws IOException { - // NOCOMMIT + // NOCOMMIT copy from previous commit } @@ -140,13 +146,14 @@ public void testEqualityOrTooBig() throws IOException { | WHERE test == "%value" OR test == "%tooBig" """.replace("%tooBig", tooBig); String luceneQuery = switch (type) { - case AUTO, CONSTANT_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD, TEXT_WITH_KEYWORD -> "*:*"; + case AUTO, CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, TEXT, TEXT_WITH_KEYWORD -> "*:*"; case KEYWORD -> "test:(%tooBig %value)".replace("%tooBig", tooBig); - case SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; }; ComputeSignature dataNodeSignature = switch (type) { case CONSTANT_KEYWORD, KEYWORD -> ComputeSignature.FILTER_IN_QUERY; - case AUTO, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT_WITH_KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; + case AUTO, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD, TEXT, TEXT_WITH_KEYWORD -> + ComputeSignature.FILTER_IN_COMPUTE; }; testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true); } @@ -159,13 +166,15 @@ public void testEqualityOrOther() throws IOException { """; String luceneQuery = switch (type) { case AUTO, TEXT_WITH_KEYWORD -> "(#test.keyword:%value -_ignored:test.keyword) foo:[2 TO 2]"; + case TEXT -> "#test:%value foo:[2 TO 2]"; case KEYWORD -> "test:%value foo:[2 TO 2]"; - case CONSTANT_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD -> "*:*"; - case SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD -> "*:*"; + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; }; ComputeSignature dataNodeSignature = switch (type) { case AUTO, CONSTANT_KEYWORD, KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_QUERY; - case MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; + case MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD, TEXT -> + ComputeSignature.FILTER_IN_COMPUTE; }; testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true); } @@ -178,9 +187,10 @@ public void testEqualityAndOther() throws IOException { """; List luceneQueryOptions = switch (type) { case AUTO, TEXT_WITH_KEYWORD -> List.of("#test.keyword:%value -_ignored:test.keyword #foo:[1 TO 1]"); + case TEXT -> List.of("#foo:[1 TO 1] #test:%value"); case KEYWORD -> List.of("#test:%value #foo:[1 TO 1]"); - case CONSTANT_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD -> List.of("foo:[1 TO 1]"); - case SEMANTIC_TEXT_WITH_KEYWORD -> + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD -> List.of("foo:[1 TO 1]"); + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> /* * single_value_match is here because there are extra documents hiding in the index * that don't have the `foo` field. @@ -189,7 +199,8 @@ public void testEqualityAndOther() throws IOException { }; ComputeSignature dataNodeSignature = switch (type) { case AUTO, CONSTANT_KEYWORD, KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_QUERY; - case MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; + case MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD, TEXT -> + ComputeSignature.FILTER_IN_COMPUTE; }; testPushQuery(value, esqlQuery, luceneQueryOptions, dataNodeSignature, true); } @@ -202,13 +213,14 @@ public void testInequality() throws IOException { """; String luceneQuery = switch (type) { case AUTO, TEXT_WITH_KEYWORD -> "(-test.keyword:%different_value #*:*) _ignored:test.keyword"; - case CONSTANT_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD -> "*:*"; + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, TEXT -> "*:*"; case KEYWORD -> "-test:%different_value #*:*"; - case SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; }; ComputeSignature dataNodeSignature = switch (type) { case CONSTANT_KEYWORD, KEYWORD -> ComputeSignature.FILTER_IN_QUERY; - case AUTO, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT_WITH_KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; + case AUTO, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD, TEXT, TEXT_WITH_KEYWORD -> + ComputeSignature.FILTER_IN_COMPUTE; }; testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true); } @@ -220,12 +232,13 @@ public void testInequalityTooBigToPush() throws IOException { | WHERE test != "%value" """; String luceneQuery = switch (type) { - case AUTO, CONSTANT_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD, TEXT_WITH_KEYWORD -> "*:*"; + case AUTO, CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, TEXT, TEXT_WITH_KEYWORD -> "*:*"; case KEYWORD -> "-test:%value #single_value_match(test)"; - case SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; }; ComputeSignature dataNodeSignature = switch (type) { - case AUTO, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT_WITH_KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; + case AUTO, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD, TEXT, TEXT_WITH_KEYWORD -> + ComputeSignature.FILTER_IN_COMPUTE; case CONSTANT_KEYWORD -> ComputeSignature.FIND_NONE; case KEYWORD -> ComputeSignature.FILTER_IN_QUERY; }; @@ -239,13 +252,14 @@ public void testCaseInsensitiveEquality() throws IOException { | WHERE TO_LOWER(test) == "%value" """; String luceneQuery = switch (type) { - case AUTO, CONSTANT_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD, TEXT_WITH_KEYWORD -> "*:*"; + case AUTO, CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, TEXT, TEXT_WITH_KEYWORD -> "*:*"; case KEYWORD -> "CaseInsensitiveTermQuery{test:%value}"; - case SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; }; ComputeSignature dataNodeSignature = switch (type) { case CONSTANT_KEYWORD, KEYWORD -> ComputeSignature.FILTER_IN_QUERY; - case AUTO, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT_WITH_KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; + case AUTO, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD, TEXT, TEXT_WITH_KEYWORD -> + ComputeSignature.FILTER_IN_COMPUTE; }; testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true); } @@ -360,8 +374,9 @@ private void indexValue(String value) throws IOException { }"""; json += switch (type) { case AUTO -> ""; - case CONSTANT_KEYWORD -> justType(); + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, TEXT -> justType(); case KEYWORD -> keyword(); + case SEMANTIC_TEXT -> justSemanticText(); case SEMANTIC_TEXT_WITH_KEYWORD -> semanticTextWithKeyword(); case TEXT_WITH_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD -> typeWithKeyword(); }; @@ -426,6 +441,19 @@ private String typeWithKeyword() { }""".replace("%type", type.name().replace("_WITH_KEYWORD", "").toLowerCase(Locale.ROOT)); } + private String justSemanticText() { + return """ + , + "mappings": { + "properties": { + "test": { + "type": "semantic_text", + "inference_id": "test" + } + } + }"""; + } + private String semanticTextWithKeyword() { return """ , From 3e3e5e642ca2a4c65b566143dee15b782fac0b93 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 16 Jun 2025 17:06:26 -0400 Subject: [PATCH 03/11] More --- .../esql/qa/single_node/PushQueriesIT.java | 80 +++++++++++++------ 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index fa05b4617e32e..d6e6de9b13f28 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.qa.single_node; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.Repeat; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.elasticsearch.client.Request; @@ -15,6 +16,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.core.Nullable; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.test.MapMatcher; import org.elasticsearch.test.TestClustersThreadFilter; @@ -34,6 +36,7 @@ import java.util.Locale; import java.util.Map; import java.util.regex.Pattern; +import java.util.stream.Stream; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; @@ -51,6 +54,7 @@ /** * Tests for pushing queries to lucene. */ +// @Repeat(iterations = 50) @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class PushQueriesIT extends ESRestTestCase { @ClassRule @@ -93,22 +97,23 @@ public void testEquality() throws IOException { FROM test | WHERE test == "%value" """; - String luceneQuery = switch (type) { - case AUTO, TEXT_WITH_KEYWORD -> "#test.keyword:%value -_ignored:test.keyword"; - case KEYWORD, TEXT -> "test:%value"; - case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD -> "*:*"; - case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; + List luceneQuery = switch (type) { + case AUTO, TEXT_WITH_KEYWORD -> List.of("#test.keyword:%value -_ignored:test.keyword"); + case KEYWORD -> List.of("test:%value"); + case TEXT -> emulateLargeTextTokens(value); + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD -> List.of("*:*"); + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> List.of("FieldExistsQuery [field=_primary_term]"); }; ComputeSignature dataNodeSignature = switch (type) { case AUTO, CONSTANT_KEYWORD, KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_QUERY; case MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, TEXT, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; }; - testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true); + testPushQuery(value, esqlQuery, luceneQuery, dataNodeSignature, true); } public void testEqualityTooBigToPush() throws IOException { - String value = "a".repeat(between(257, 1000)); + String value = "v".repeat(between(257, 1000)); String esqlQuery = """ FROM test | WHERE test == "%value" @@ -131,8 +136,24 @@ public void testEqualityTooBigToPush() throws IOException { * {@code NOT !=} should function just like {@code ==}. */ public void testNotInequality() throws IOException { - // NOCOMMIT copy from previous commit - + String value = "v".repeat(between(0, 256)); + String esqlQuery = """ + FROM test + | WHERE NOT test != "%value" + """; + List luceneQuery = switch (type) { + case AUTO, TEXT_WITH_KEYWORD -> List.of("#test.keyword:%value -_ignored:test.keyword"); + case KEYWORD -> List.of("test:%value"); + case TEXT -> emulateLargeTextTokens(value); + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD -> List.of("*:*"); + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> List.of("FieldExistsQuery [field=_primary_term]"); + }; + ComputeSignature dataNodeSignature = switch (type) { + case AUTO, CONSTANT_KEYWORD, KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_QUERY; + case MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, TEXT, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> + ComputeSignature.FILTER_IN_COMPUTE; + }; + testPushQuery(value, esqlQuery, luceneQuery, dataNodeSignature, true); } /** @@ -159,36 +180,45 @@ public void testEqualityOrTooBig() throws IOException { } public void testEqualityOrOther() throws IOException { - String value = "v".repeat(between(0, 256)); + String value = "v".repeat(256); String esqlQuery = """ FROM test | WHERE test == "%value" OR foo == 2 """; - String luceneQuery = switch (type) { - case AUTO, TEXT_WITH_KEYWORD -> "(#test.keyword:%value -_ignored:test.keyword) foo:[2 TO 2]"; - case TEXT -> "#test:%value foo:[2 TO 2]"; - case KEYWORD -> "test:%value foo:[2 TO 2]"; - case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD -> "*:*"; - case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; + List luceneQuery = switch (type) { + case AUTO, TEXT_WITH_KEYWORD -> List.of("(#test.keyword:%value -_ignored:test.keyword) foo:[2 TO 2]"); + case TEXT -> { + List q = emulateLargeTextTokens(value); + if (q.size() > 1) { + String s = "(" + String.join(" ", q) + ")"; + yield List.of(s + " foo:[2 TO 2]", "foo:[2 TO 2] " + s); + } + yield List.of(q.get(0) + " foo:[2 TO 2]", "foo:[2 TO 2] " + q.get(0)); + } + case KEYWORD -> List.of("test:%value foo:[2 TO 2]"); + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD -> List.of("*:*"); + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> List.of("FieldExistsQuery [field=_primary_term]"); }; ComputeSignature dataNodeSignature = switch (type) { case AUTO, CONSTANT_KEYWORD, KEYWORD, TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_QUERY; case MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD, TEXT -> ComputeSignature.FILTER_IN_COMPUTE; }; - testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true); + testPushQuery(value, esqlQuery, luceneQuery, dataNodeSignature, true); } public void testEqualityAndOther() throws IOException { - String value = "v".repeat(between(0, 256)); + String value = "v".repeat(256); String esqlQuery = """ FROM test | WHERE test == "%value" AND foo == 1 """; List luceneQueryOptions = switch (type) { case AUTO, TEXT_WITH_KEYWORD -> List.of("#test.keyword:%value -_ignored:test.keyword #foo:[1 TO 1]"); - case TEXT -> List.of("#foo:[1 TO 1] #test:%value"); case KEYWORD -> List.of("#test:%value #foo:[1 TO 1]"); + case TEXT -> emulateLargeTextTokens(value).stream() + .flatMap(o -> Stream.of("#" + o + " #foo:[1 TO 1]", "#foo:[1 TO 1] #" + o)) + .toList(); case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD -> List.of("foo:[1 TO 1]"); case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> /* @@ -226,7 +256,7 @@ public void testInequality() throws IOException { } public void testInequalityTooBigToPush() throws IOException { - String value = "a".repeat(between(257, 1000)); + String value = "v".repeat(between(257, 1000)); String esqlQuery = """ FROM test | WHERE test != "%value" @@ -246,7 +276,7 @@ public void testInequalityTooBigToPush() throws IOException { } public void testCaseInsensitiveEquality() throws IOException { - String value = "a".repeat(between(0, 256)); + String value = "v".repeat(between(0, 256)); String esqlQuery = """ FROM test | WHERE TO_LOWER(test) == "%value" @@ -523,11 +553,11 @@ public void setUpTextEmbeddingInferenceEndpoint() throws IOException { private List emulateLargeTextTokens(String value) { // The default tokenizer splits at 255 characters - String first = "#test:" + "a".repeat(StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH); - if (value.length() % 255 == 0) { - return List.of(first); + if (value.length() < StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH) { + return List.of("test:" + "v".repeat(value.length())); } - String rest = "#test:" + "a".repeat(value.length() % 255); + String first = "#test:" + "v".repeat(StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH); + String rest = "#test:" + "v".repeat(value.length() % StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH); // We don't know what order they'll show up, so either is fine. return List.of(first + " " + rest, rest + " " + first); } From e769c32e969becc90ec547196a1a88679e64f838 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 17 Jun 2025 09:10:25 -0400 Subject: [PATCH 04/11] Compile --- .../index/mapper/annotatedtext/AnnotatedTextFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 8ee639ffc8431..b96d544dc610c 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -492,7 +492,7 @@ private AnnotatedTextFieldType( KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate, Map meta ) { - super(name, true, store, tsi, isSyntheticSource, syntheticSourceDelegate, meta, false, false); + super(name, true, store, tsi, isSyntheticSource, syntheticSourceDelegate, meta, false, false, false); } public AnnotatedTextFieldType(String name, Map meta) { From 07f92d76d7d5afeccce8e4bad3c8e7c264bfe1ff Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 26 Jun 2025 12:06:58 -0400 Subject: [PATCH 05/11] Explain --- .../org/elasticsearch/index/mapper/TextFieldMapper.java | 6 +++++- .../compute/lucene/ValuesSourceReaderOperatorTests.java | 2 ++ .../xpack/esql/qa/single_node/PushQueriesIT.java | 6 ++---- .../rules/physical/local/LucenePushdownPredicates.java | 9 ++++++++- .../esql/optimizer/LocalPhysicalPlanOptimizerTests.java | 2 +- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 8018c74a7eb3e..4b3ed767c1bca 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -67,6 +67,7 @@ import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.StoredFieldSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; +import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.script.field.DelegateDocValuesField; @@ -714,7 +715,10 @@ public static class TextFieldType extends StringFieldType { */ private final KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate; - // NOCOMMIT docs + /** + * Does a {@link MatchQueryBuilder} produce all documents + * that might have equal text to the query's value. + */ private final boolean matchQueryYieldsCandidateMatchesForEquality; public TextFieldType( diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java index 1550b6dc013ab..4f4d5600b2866 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java @@ -1421,6 +1421,7 @@ private TextFieldMapper.TextFieldType storedTextField(String name) { null, Map.of(), false, + false, false ); } @@ -1435,6 +1436,7 @@ private TextFieldMapper.TextFieldType textFieldWithDelegate(String name, Keyword delegate, Map.of(), false, + false, false ); } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index 80c949d72b7d6..d6676b52195a8 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -8,11 +8,10 @@ package org.elasticsearch.xpack.esql.qa.single_node; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import com.carrotsearch.randomizedtesting.annotations.Repeat; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import org.elasticsearch.client.Request; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.collect.Iterators; @@ -35,7 +34,6 @@ import java.util.Locale; import java.util.Map; import java.util.regex.Pattern; -import java.util.stream.Stream; import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; @@ -53,7 +51,7 @@ /** * Tests for pushing queries to lucene. */ -//@Repeat(iterations = 50) +// @Repeat(iterations = 50) @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class PushQueriesIT extends ESRestTestCase { @ClassRule diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index 411ff5b5f60e6..c965762865362 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; +import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; @@ -47,9 +48,15 @@ public interface LucenePushdownPredicates { */ boolean isIndexed(FieldAttribute attr); + /** + * Can the synthetic _source delegate perform {@code ==} on the provided string? + */ boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute attr, String value); - // NOCOMMIT javadoc this and above + /** + * Does a {@link MatchQueryBuilder} produce a complete list of all possible documents + * that might be {@code ==} to the value passed to the query. + */ boolean matchQueryYieldsCandidateMatchesForEquality(FieldAttribute attr); /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index b7d1243759493..0b4bbce9f1c13 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -795,7 +795,7 @@ public void testOutOfRangeFilterPushdown() { var query = "from test | where " + comparison; Source expectedSource = new Source(1, 18, comparison); - logger.info("Query: " + query); + logger.info("Query: {}", query); EsQueryExec actualQueryExec = doTestOutOfRangeFilterPushdown(query, allTypeMappingAnalyzer); assertThat(actualQueryExec.query(), is(instanceOf(SingleValueQuery.Builder.class))); From 89590807d9f36b86c5594657abc49ec430b3a642 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 26 Jun 2025 13:47:35 -0400 Subject: [PATCH 06/11] Fixup --- .../predicate/operator/comparison/Equals.java | 2 +- .../predicate/operator/comparison/GreaterThan.java | 11 ++++++++++- .../operator/comparison/GreaterThanOrEqual.java | 9 ++++++++- .../predicate/operator/comparison/LessThan.java | 9 ++++++++- .../operator/comparison/LessThanOrEqual.java | 9 ++++++++- .../predicate/operator/comparison/NotEquals.java | 9 ++++++++- 6 files changed, 43 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index b05ad47d585f3..b0b3679b84faa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -152,7 +152,7 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand if (right() instanceof Literal rhs && left().dataType() == DataType.TEXT && left() instanceof FieldAttribute lhs) { return asQueryText(pushdownPredicates, handler, lhs, ((BytesRef) rhs.value()).utf8ToString()); } - return super.asQuery(pushdownPredicates, handler); + return handler.forceToSingleValueQuery(left(), super.asQuery(pushdownPredicates, handler)); } private Query asQueryText(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler, FieldAttribute lhs, String rhs) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java index 7205f66749b0b..33d1d7346608c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -22,7 +23,10 @@ import java.time.ZoneId; import java.util.Map; -public class GreaterThan extends EsqlBinaryComparison implements Negatable { +public class GreaterThan extends EsqlBinaryComparison + implements + Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "GreaterThan", @@ -117,6 +121,11 @@ public EsqlBinaryComparison reverse() { return new LessThan(source(), left(), right(), zoneId()); } + @Override + public Expression singleValueField() { + return left(); + } + @Evaluator(extraName = "Ints") static boolean processInts(int lhs, int rhs) { return lhs > rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java index 7fadea9a3cf7a..e1716fda042d1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -22,7 +23,8 @@ import java.time.ZoneId; import java.util.Map; -public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatable { +public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "GreaterThanOrEqual", @@ -112,6 +114,11 @@ public LessThan negate() { return new LessThan(source(), left(), right(), zoneId()); } + @Override + public Expression singleValueField() { + return left(); + } + @Override public EsqlBinaryComparison reverse() { return new LessThanOrEqual(source(), left(), right(), zoneId()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java index 728a01b81ead3..47d2fe5e06027 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -22,7 +23,8 @@ import java.time.ZoneId; import java.util.Map; -public class LessThan extends EsqlBinaryComparison implements Negatable { +public class LessThan extends EsqlBinaryComparison implements Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "LessThan", @@ -109,6 +111,11 @@ public EsqlBinaryComparison reverse() { return new GreaterThan(source(), left(), right(), zoneId()); } + @Override + public Expression singleValueField() { + return left(); + } + @Evaluator(extraName = "Ints") static boolean processInts(int lhs, int rhs) { return lhs < rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java index 64cd835f62722..0438df9765d15 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -22,7 +23,8 @@ import java.time.ZoneId; import java.util.Map; -public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable { +public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "LessThanOrEqual", @@ -109,6 +111,11 @@ public EsqlBinaryComparison reverse() { return new GreaterThanOrEqual(source(), left(), right(), zoneId()); } + @Override + public Expression singleValueField() { + return left(); + } + @Evaluator(extraName = "Ints") static boolean processInts(int lhs, int rhs) { return lhs <= rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java index d020f983ec180..4bd74b43e1c87 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.ann.Evaluator; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -22,7 +23,8 @@ import java.time.ZoneId; import java.util.Map; -public class NotEquals extends EsqlBinaryComparison implements Negatable { +public class NotEquals extends EsqlBinaryComparison implements Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "NotEquals", @@ -189,4 +191,9 @@ public NotEquals swapLeftAndRight() { public EsqlBinaryComparison negate() { return new Equals(source(), left(), right(), zoneId()); } + + @Override + public Expression singleValueField() { + return left(); + } } From 6e519cdc706159d4aca85ea430559070e6eab2d6 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 1 Jul 2025 08:31:36 -0400 Subject: [PATCH 07/11] Update docs/changelog/130387.yaml --- docs/changelog/130387.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/130387.yaml diff --git a/docs/changelog/130387.yaml b/docs/changelog/130387.yaml new file mode 100644 index 0000000000000..ea2c95364e932 --- /dev/null +++ b/docs/changelog/130387.yaml @@ -0,0 +1,5 @@ +pr: 130387 +summary: Push `==` to `text` fields to lucene +area: ES|QL +type: enhancement +issues: [] From 0a2491266ef8a4403e163f2e99a9e2c8131de065 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 1 Jul 2025 12:40:13 +0000 Subject: [PATCH 08/11] [CI] Auto commit changes from spotless --- .../predicate/operator/comparison/GreaterThanOrEqual.java | 6 ++++-- .../expression/predicate/operator/comparison/LessThan.java | 6 ++++-- .../predicate/operator/comparison/LessThanOrEqual.java | 6 ++++-- .../expression/predicate/operator/comparison/NotEquals.java | 6 ++++-- .../rules/physical/local/LucenePushdownPredicates.java | 2 +- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java index e1716fda042d1..c28253ce1e374 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java @@ -23,8 +23,10 @@ import java.time.ZoneId; import java.util.Map; -public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatable, - TranslationAware.SingleValueTranslationAware { +public class GreaterThanOrEqual extends EsqlBinaryComparison + implements + Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "GreaterThanOrEqual", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java index 47d2fe5e06027..149d52e190a06 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java @@ -23,8 +23,10 @@ import java.time.ZoneId; import java.util.Map; -public class LessThan extends EsqlBinaryComparison implements Negatable, - TranslationAware.SingleValueTranslationAware { +public class LessThan extends EsqlBinaryComparison + implements + Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "LessThan", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java index 0438df9765d15..9cc2efee56ed1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java @@ -23,8 +23,10 @@ import java.time.ZoneId; import java.util.Map; -public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable, - TranslationAware.SingleValueTranslationAware { +public class LessThanOrEqual extends EsqlBinaryComparison + implements + Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "LessThanOrEqual", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java index 4bd74b43e1c87..a534967a9e5cc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java @@ -23,8 +23,10 @@ import java.time.ZoneId; import java.util.Map; -public class NotEquals extends EsqlBinaryComparison implements Negatable, - TranslationAware.SingleValueTranslationAware { +public class NotEquals extends EsqlBinaryComparison + implements + Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "NotEquals", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index 01a7a74cac0f6..f58643fc56c65 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -7,9 +7,9 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; -import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.TransportVersion; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; From b63a62669a31a2285534528435c4a7ad7eff69bb Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 15 Jul 2025 15:02:12 -0400 Subject: [PATCH 09/11] Fixup --- .../predicate/operator/comparison/GreaterThanOrEqual.java | 6 ++++-- .../expression/predicate/operator/comparison/LessThan.java | 6 ++++-- .../predicate/operator/comparison/LessThanOrEqual.java | 6 ++++-- .../expression/predicate/operator/comparison/NotEquals.java | 6 ++++-- .../rules/physical/local/LucenePushdownPredicates.java | 2 +- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java index e1716fda042d1..c28253ce1e374 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java @@ -23,8 +23,10 @@ import java.time.ZoneId; import java.util.Map; -public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatable, - TranslationAware.SingleValueTranslationAware { +public class GreaterThanOrEqual extends EsqlBinaryComparison + implements + Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "GreaterThanOrEqual", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java index 47d2fe5e06027..149d52e190a06 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java @@ -23,8 +23,10 @@ import java.time.ZoneId; import java.util.Map; -public class LessThan extends EsqlBinaryComparison implements Negatable, - TranslationAware.SingleValueTranslationAware { +public class LessThan extends EsqlBinaryComparison + implements + Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "LessThan", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java index 0438df9765d15..9cc2efee56ed1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java @@ -23,8 +23,10 @@ import java.time.ZoneId; import java.util.Map; -public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable, - TranslationAware.SingleValueTranslationAware { +public class LessThanOrEqual extends EsqlBinaryComparison + implements + Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "LessThanOrEqual", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java index 4bd74b43e1c87..a534967a9e5cc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java @@ -23,8 +23,10 @@ import java.time.ZoneId; import java.util.Map; -public class NotEquals extends EsqlBinaryComparison implements Negatable, - TranslationAware.SingleValueTranslationAware { +public class NotEquals extends EsqlBinaryComparison + implements + Negatable, + TranslationAware.SingleValueTranslationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "NotEquals", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index 3c1968dae67b9..1798e2c8c33b3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -7,9 +7,9 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; -import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.TransportVersion; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; From 0c1efa505890e41df29d868bbd67ed4a67318d46 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 17 Jul 2025 10:36:36 -0400 Subject: [PATCH 10/11] Fixup --- .../xpack/esql/qa/single_node/PushQueriesIT.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index 990d7439586c7..7310020f95109 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -348,12 +348,13 @@ public void testRLike() throws IOException { """; String luceneQuery = switch (type) { case KEYWORD -> "test:/%value.*/"; - case CONSTANT_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD, AUTO, TEXT_WITH_KEYWORD -> "*:*"; - case SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, AUTO, TEXT, TEXT_WITH_KEYWORD -> "*:*"; + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; }; ComputeSignature dataNodeSignature = switch (type) { case CONSTANT_KEYWORD, KEYWORD -> ComputeSignature.FILTER_IN_QUERY; - case AUTO, TEXT_WITH_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; + case AUTO, TEXT, TEXT_WITH_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> + ComputeSignature.FILTER_IN_COMPUTE; }; testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true); } @@ -365,13 +366,14 @@ public void testRLikeList() throws IOException { | WHERE test rlike ("%value.*", "abc.*") """; String luceneQuery = switch (type) { - case CONSTANT_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD, AUTO, TEXT_WITH_KEYWORD -> "*:*"; - case SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; + case CONSTANT_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, AUTO, TEXT, TEXT_WITH_KEYWORD -> "*:*"; + case SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD -> "FieldExistsQuery [field=_primary_term]"; case KEYWORD -> "test:RLIKE(\"%value.*\", \"abc.*\"), caseInsensitive=false"; }; ComputeSignature dataNodeSignature = switch (type) { case CONSTANT_KEYWORD, KEYWORD -> ComputeSignature.FILTER_IN_QUERY; - case AUTO, TEXT_WITH_KEYWORD, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT_WITH_KEYWORD -> ComputeSignature.FILTER_IN_COMPUTE; + case AUTO, TEXT_WITH_KEYWORD, MATCH_ONLY_TEXT, MATCH_ONLY_TEXT_WITH_KEYWORD, SEMANTIC_TEXT, SEMANTIC_TEXT_WITH_KEYWORD, TEXT -> + ComputeSignature.FILTER_IN_COMPUTE; }; testPushQuery(value, esqlQuery, List.of(luceneQuery), dataNodeSignature, true); } From 96060ecb8728d015863a3e166043f0b4f22c9de1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 17 Jul 2025 14:10:15 -0400 Subject: [PATCH 11/11] Off by one fun --- .../elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java index 7310020f95109..fd60d51f60c22 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/PushQueriesIT.java @@ -51,7 +51,6 @@ /** * Tests for pushing queries to lucene. */ -// @Repeat(iterations = 50) @ThreadLeakFilters(filters = TestClustersThreadFilter.class) public class PushQueriesIT extends ESRestTestCase { @ClassRule @@ -639,7 +638,7 @@ public void setUpTextEmbeddingInferenceEndpoint() throws IOException { private List emulateLargeTextTokens(String value) { // The default tokenizer splits at 255 characters - if (value.length() < StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH) { + if (value.length() <= StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH) { return List.of("test:" + "v".repeat(value.length())); } String first = "#test:" + "v".repeat(StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);