diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java index d2811962dd29d..3b4d445002073 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java @@ -70,11 +70,11 @@ public class QueryPlanningBenchmark { private EsqlParser defaultParser; private Analyzer manyFieldsAnalyzer; private LogicalPlanOptimizer defaultOptimizer; + private Configuration config; @Setup public void setup() { - - var config = new Configuration( + this.config = new Configuration( DateUtils.UTC, Locale.US, null, @@ -116,7 +116,7 @@ public void setup() { } private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) { - var parsed = parser.createStatement(query, new QueryParams(), telemetry); + var parsed = parser.createStatement(query, new QueryParams(), telemetry, config); var analyzed = analyzer.analyze(parsed); var optimized = optimizer.optimize(analyzed); return optimized; diff --git a/docs/changelog/130019.yaml b/docs/changelog/130019.yaml new file mode 100644 index 0000000000000..13b327015c4a1 --- /dev/null +++ b/docs/changelog/130019.yaml @@ -0,0 +1,6 @@ +pr: 130019 +summary: Fix behavior for `_index` LIKE for ESQL +area: ES|QL +type: bug +issues: + - 129511 diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 486804a1f57ee..87bf199711b61 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -329,6 +329,7 @@ static TransportVersion def(int id) { public static final TransportVersion PROJECT_STATE_REGISTRY_RECORDS_DELETIONS = def(9_113_0_00); public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_114_0_00); public static final TransportVersion ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED = def(9_115_0_00); + public static final TransportVersion ESQL_FIXED_INDEX_LIKE = def(9_116_0_00); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java index 5ecb75b09408c..0b246c2492fdb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java @@ -15,6 +15,8 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.Nullable; @@ -23,6 +25,7 @@ import java.util.Collection; import java.util.Map; +import java.util.function.Supplier; /** * A {@link MappedFieldType} that has the same value for all documents. @@ -135,9 +138,47 @@ public final Query wildcardQuery(String value, boolean caseInsensitive, QueryRew } } + /** + * Returns a query that matches all documents or no documents + * It usually calls {@link #wildcardQuery(String, boolean, QueryRewriteContext)} + * except for IndexFieldType which overrides this method to use its own matching logic. + */ + public Query wildcardLikeQuery(String value, boolean caseInsensitive, QueryRewriteContext context) { + return wildcardQuery(value, caseInsensitive, context); + } + @Override public final boolean fieldHasValue(FieldInfos fieldInfos) { // We consider constant field types to always have value. return true; } + + /** + * Returns the constant value of this field as a string. + * Based on the field type, we need to get it in a different way. + */ + public abstract String getConstantFieldValue(SearchExecutionContext context); + + /** + * Returns a query that matches all documents or no documents + * depending on whether the constant value of this field matches or not + */ + @Override + public Query automatonQuery( + Supplier automatonSupplier, + Supplier characterRunAutomatonSupplier, + @Nullable MultiTermQuery.RewriteMethod method, + SearchExecutionContext context, + String description + ) { + CharacterRunAutomaton compiled = characterRunAutomatonSupplier.get(); + boolean matches = compiled.run(getConstantFieldValue(context)); + if (matches) { + return new MatchAllDocsQuery(); + } else { + return new MatchNoDocsQuery( + "The \"" + context.getFullyQualifiedIndex().getName() + "\" query was rewritten to a \"match_none\" query." + ); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 33c6ff15cccfd..79754a4c63b30 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -10,9 +10,13 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -27,6 +31,7 @@ import java.util.Collections; import java.util.List; +import java.util.Locale; public class IndexFieldMapper extends MetadataFieldMapper { @@ -102,6 +107,38 @@ public StoredFieldsSpec storedFieldsSpec() { }; } + @Override + public Query wildcardLikeQuery( + String value, + @Nullable MultiTermQuery.RewriteMethod method, + boolean caseInsensitve, + SearchExecutionContext context + ) { + String indexName = context.getFullyQualifiedIndex().getName(); + return getWildcardLikeQuery(value, caseInsensitve, indexName); + } + + @Override + public Query wildcardLikeQuery(String value, boolean caseInsensitive, QueryRewriteContext context) { + String indexName = context.getFullyQualifiedIndex().getName(); + return getWildcardLikeQuery(value, caseInsensitive, indexName); + } + + private static Query getWildcardLikeQuery(String value, boolean caseInsensitve, String indexName) { + if (caseInsensitve) { + value = value.toLowerCase(Locale.ROOT); + indexName = indexName.toLowerCase(Locale.ROOT); + } + if (Regex.simpleMatch(value, indexName)) { + return new MatchAllDocsQuery(); + } + return new MatchNoDocsQuery("The \"" + indexName + "\" query was rewritten to a \"match_none\" query."); + } + + @Override + public String getConstantFieldValue(SearchExecutionContext context) { + return context.getFullyQualifiedIndex().getName(); + } } public IndexFieldMapper() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java index 9708753926e1d..82960224e13bf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java @@ -57,6 +57,11 @@ protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteC return Regex.simpleMatch(pattern, indexMode, caseInsensitive); } + @Override + public String getConstantFieldValue(SearchExecutionContext context) { + return context.getIndexSettings().getMode().getName(); + } + @Override public Query existsQuery(SearchExecutionContext context) { return new MatchAllDocsQuery(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index f8af7043b13dd..cf731cc5cbc65 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiTerms; +import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.MultiTermQuery; @@ -31,6 +32,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.CompiledAutomaton.AUTOMATON_TYPE; import org.apache.lucene.util.automaton.Operations; @@ -51,6 +53,7 @@ import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.StoredFieldSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.elasticsearch.index.query.AutomatonQueryWithDescription; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.script.Script; @@ -82,6 +85,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Supplier; import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; @@ -1042,6 +1046,17 @@ public IndexSortConfig getIndexSortConfig() { public boolean hasDocValuesSkipper() { return hasDocValuesSkipper; } + + @Override + public Query automatonQuery( + Supplier automatonSupplier, + Supplier characterRunAutomatonSupplier, + @Nullable MultiTermQuery.RewriteMethod method, + SearchExecutionContext context, + String description + ) { + return new AutomatonQueryWithDescription(new Term(name()), automatonSupplier.get(), description); + } } private final boolean indexed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index a9e67be4085dd..87877f926983e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -25,6 +25,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -54,6 +56,7 @@ import java.util.Objects; import java.util.Set; import java.util.function.Function; +import java.util.function.Supplier; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; @@ -329,6 +332,19 @@ public final Query wildcardQuery(String value, @Nullable MultiTermQuery.RewriteM return wildcardQuery(value, method, false, context); } + /** + * Similar to wildcardQuery, except that we change the behavior for ESQL + * to behave like a string LIKE query, where the value is matched as a string + */ + public Query wildcardLikeQuery( + String value, + @Nullable MultiTermQuery.RewriteMethod method, + boolean caseInsensitve, + SearchExecutionContext context + ) { + return wildcardQuery(value, method, caseInsensitve, context); + } + public Query wildcardQuery( String value, @Nullable MultiTermQuery.RewriteMethod method, @@ -370,6 +386,23 @@ public Query regexpQuery( ); } + /** + * Returns a Lucine pushable Query for the current field + * For now can only be AutomatonQuery or MatchAllDocsQuery() or MatchNoDocsQuery() + */ + public Query automatonQuery( + Supplier automatonSupplier, + Supplier characterRunAutomatonSupplier, + @Nullable MultiTermQuery.RewriteMethod method, + SearchExecutionContext context, + String description + ) { + throw new QueryShardException( + context, + "Can only use automaton queries on keyword fields - not on [" + name + "] which is of type [" + typeName() + "]" + ); + } + public Query existsQuery(SearchExecutionContext context) { if (hasDocValues() || getTextSearchInfo().hasNorms()) { return new FieldExistsQuery(name()); diff --git a/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryBuilder.java deleted file mode 100644 index 3fd957ee8acfb..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryBuilder.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.query; - -import org.apache.lucene.index.Term; -import org.apache.lucene.search.AutomatonQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.util.automaton.Automaton; -import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.Objects; - -/** - * Implements an Automaton query, which matches documents based on a Lucene Automaton. - * It does not support serialization or XContent representation. - */ -public class AutomatonQueryBuilder extends AbstractQueryBuilder implements MultiTermQueryBuilder { - private final String fieldName; - private final Automaton automaton; - private final String description; - - public AutomatonQueryBuilder(String fieldName, Automaton automaton, String description) { - if (Strings.isEmpty(fieldName)) { - throw new IllegalArgumentException("field name is null or empty"); - } - if (automaton == null) { - throw new IllegalArgumentException("automaton cannot be null"); - } - this.fieldName = fieldName; - this.automaton = automaton; - this.description = description; - } - - @Override - public String fieldName() { - return fieldName; - } - - public String description() { - return description; - } - - @Override - public String getWriteableName() { - throw new UnsupportedOperationException("AutomatonQueryBuilder does not support getWriteableName"); - } - - @Override - protected void doWriteTo(StreamOutput out) throws IOException { - throw new UnsupportedEncodingException("AutomatonQueryBuilder does not support doWriteTo"); - } - - @Override - protected void doXContent(XContentBuilder builder, Params params) throws IOException { - throw new UnsupportedEncodingException("AutomatonQueryBuilder does not support doXContent"); - } - - @Override - protected Query doToQuery(SearchExecutionContext context) throws IOException { - return new AutomatonQueryWithDescription(new Term(fieldName), automaton, description); - } - - @Override - protected int doHashCode() { - return Objects.hash(fieldName, automaton, description); - } - - @Override - protected boolean doEquals(AutomatonQueryBuilder other) { - return Objects.equals(fieldName, other.fieldName) - && Objects.equals(automaton, other.automaton) - && Objects.equals(description, other.description); - } - - @Override - public TransportVersion getMinimalSupportedVersion() { - throw new UnsupportedOperationException("AutomatonQueryBuilder does not support getMinimalSupportedVersion"); - } - - static class AutomatonQueryWithDescription extends AutomatonQuery { - private final String description; - - AutomatonQueryWithDescription(Term term, Automaton automaton, String description) { - super(term, automaton); - this.description = description; - } - - @Override - public String toString(String field) { - if (this.field.equals(field)) { - return description; - } - return this.field + ":" + description; - } - } -} diff --git a/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryWithDescription.java b/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryWithDescription.java new file mode 100644 index 0000000000000..78c285470e3b6 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/query/AutomatonQueryWithDescription.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.query; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.AutomatonQuery; +import org.apache.lucene.util.automaton.Automaton; + +/** + * A specialized {@link AutomatonQuery} that includes a description of the query. + * This can be useful for debugging or logging purposes, providing more context + * about the query being executed. + */ +public class AutomatonQueryWithDescription extends AutomatonQuery { + private final String description; + + public AutomatonQueryWithDescription(Term term, Automaton automaton, String description) { + super(term, automaton); + this.description = description; + } + + @Override + public String toString(String field) { + if (this.field.equals(field)) { + return description; + } + return this.field + ":" + description; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java index fd1f39a70d525..e0cfd86c10c6d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java @@ -72,6 +72,11 @@ protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteC return Regex.simpleMatch(pattern, tierPreference); } + @Override + public String getConstantFieldValue(SearchExecutionContext context) { + return context.getTierPreference(); + } + @Override public Query existsQuery(SearchExecutionContext context) { throw new UnsupportedOperationException("field exists query is not supported on the coordinator node"); diff --git a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java index fed6c3df15587..6b754c97d9d19 100644 --- a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java @@ -56,6 +56,13 @@ public class WildcardQueryBuilder extends AbstractQueryBuilder void assertResultMapForLike( requiredCapabilities.add("like_list_on_index_fields"); } // the feature is completely supported if both local and remote clusters support it - boolean isSupported = clusterHasCapability("POST", "/_query", List.of(), requiredCapabilities).orElse(false); - try (RestClient remoteClient = remoteClusterClient()) { - isSupported = isSupported - && clusterHasCapability(remoteClient, "POST", "/_query", List.of(), requiredCapabilities).orElse(false); - } + boolean isSupported = capabilitiesSupportedNewAndOld(requiredCapabilities); if (isSupported) { assertResultMap(includeCCSMetadata, result, columns, values, remoteOnly); @@ -433,6 +430,53 @@ public void testLikeIndex() throws Exception { assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false); } + public void testLikeIndexLegacySettingNoResults() throws Exception { + // the feature is completely supported if both local and remote clusters support it + assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("like_on_index_fields"))); + try ( + ClusterSettingToggle ignored = new ClusterSettingToggle(adminClient(), "esql.query.string_like_on_index", false, true); + RestClient remoteClient = remoteClusterClient(); + ClusterSettingToggle ignored2 = new ClusterSettingToggle(remoteClient, "esql.query.string_like_on_index", false, true) + ) { + // test code with the setting changed + boolean includeCCSMetadata = includeCCSMetadata(); + Map result = run(""" + FROM test-local-index,*:test-remote-index METADATA _index + | WHERE _index LIKE "*remote*" + | STATS c = COUNT(*) BY _index + | SORT _index ASC + """, includeCCSMetadata); + var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword")); + // we expect empty result, since the setting is false + var values = List.of(); + assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false); + } + } + + public void testLikeIndexLegacySettingResults() throws Exception { + // we require that the admin client supports the like_on_index_fields capability + // otherwise we will get an error when trying to toggle the setting + // the remote client does not have to support it + assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("like_on_index_fields"))); + try ( + ClusterSettingToggle ignored = new ClusterSettingToggle(adminClient(), "esql.query.string_like_on_index", false, true); + RestClient remoteClient = remoteClusterClient(); + ClusterSettingToggle ignored2 = new ClusterSettingToggle(remoteClient, "esql.query.string_like_on_index", false, true) + ) { + boolean includeCCSMetadata = includeCCSMetadata(); + Map result = run(""" + FROM test-local-index,*:test-remote-index METADATA _index + | WHERE _index LIKE "*remote*:*remote*" + | STATS c = COUNT(*) BY _index + | SORT _index ASC + """, includeCCSMetadata); + var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword")); + // we expect results, since the setting is false, but there is : in the LIKE query + var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex)); + assertResultMapForLike(includeCCSMetadata, result, columns, values, false, false); + } + } + public void testNotLikeIndex() throws Exception { boolean includeCCSMetadata = includeCCSMetadata(); Map result = run(""" @@ -447,12 +491,8 @@ public void testNotLikeIndex() throws Exception { } public void testLikeListIndex() throws Exception { - List requiredCapabilities = new ArrayList<>(List.of("like_list_on_index_fields")); // the feature is completely supported if both local and remote clusters support it - if (capabilitiesSupportedNewAndOld(requiredCapabilities) == false) { - logger.info("--> skipping testNotLikeListIndex, due to missing capability"); - return; - } + assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("like_list_on_index_fields"))); boolean includeCCSMetadata = includeCCSMetadata(); Map result = run(""" FROM test-local-index,*:test-remote-index METADATA _index @@ -466,12 +506,7 @@ public void testLikeListIndex() throws Exception { } public void testNotLikeListIndex() throws Exception { - List requiredCapabilities = new ArrayList<>(List.of("like_list_on_index_fields")); - // the feature is completely supported if both local and remote clusters support it - if (capabilitiesSupportedNewAndOld(requiredCapabilities) == false) { - logger.info("--> skipping testNotLikeListIndex, due to missing capability"); - return; - } + assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("like_list_on_index_fields"))); boolean includeCCSMetadata = includeCCSMetadata(); Map result = run(""" FROM test-local-index,*:test-remote-index METADATA _index @@ -484,13 +519,8 @@ public void testNotLikeListIndex() throws Exception { assertResultMapForLike(includeCCSMetadata, result, columns, values, false, true); } - public void testNotLikeListKeyWord() throws Exception { - List requiredCapabilities = new ArrayList<>(List.of("like_list_on_index_fields")); - // the feature is completely supported if both local and remote clusters support it - if (capabilitiesSupportedNewAndOld(requiredCapabilities) == false) { - logger.info("--> skipping testNotLikeListIndex, due to missing capability"); - return; - } + public void testNotLikeListKeyword() throws Exception { + assumeTrue("not supported", capabilitiesSupportedNewAndOld(List.of("like_with_list_of_patterns"))); boolean includeCCSMetadata = includeCCSMetadata(); Map result = run(""" FROM test-local-index,*:test-remote-index METADATA _index @@ -499,7 +529,11 @@ public void testNotLikeListKeyWord() throws Exception { | SORT _index ASC """, includeCCSMetadata); var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword")); - var values = List.of(List.of(localDocs.size(), localIndex)); + Predicate filter = d -> false == (d.color.contains("blue") || d.color.contains("red")); + var values = List.of( + List.of((int) remoteDocs.stream().filter(filter).count(), REMOTE_CLUSTER_NAME + ":" + remoteIndex), + List.of((int) localDocs.stream().filter(filter).count(), localIndex) + ); assertResultMapForLike(includeCCSMetadata, result, columns, values, false, true); } @@ -545,4 +579,28 @@ private static boolean capabilitiesEndpointAvailable() { private static boolean includeCCSMetadata() { return ccsMetadataAvailable() && randomBoolean(); } + + public static class ClusterSettingToggle implements AutoCloseable { + private final RestClient client; + private final String settingKey; + private final Object originalValue; + + public ClusterSettingToggle(RestClient client, String settingKey, Object newValue, Object restoreValue) throws IOException { + this.client = client; + this.settingKey = settingKey; + this.originalValue = restoreValue; + setValue(newValue); + } + + private void setValue(Object value) throws IOException { + Request set = new Request("PUT", "/_cluster/settings"); + set.setJsonEntity("{\"persistent\": {\"" + settingKey + "\": " + value + "}}"); + ESRestTestCase.assertOK(client.performRequest(set)); + } + + @Override + public void close() throws IOException { + setValue(originalValue == null ? "null" : originalValue); + } + } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 7ec9ee6344551..210a4be28f840 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -863,7 +863,7 @@ static Version randomVersion() { } public static WildcardLike wildcardLike(Expression left, String exp) { - return new WildcardLike(EMPTY, left, new WildcardPattern(exp)); + return new WildcardLike(EMPTY, left, new WildcardPattern(exp), false); } public static RLike rlike(Expression left, String exp) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 6ea22bff910ee..feb997bd1eb8b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1199,6 +1199,8 @@ public enum Cap { LIKE_WITH_LIST_OF_PATTERNS, + LIKE_LIST_ON_INDEX_FIELDS, + /** * Support parameters for SAMPLE command. */ @@ -1216,6 +1218,10 @@ public enum Cap { * (Re)Added EXPLAIN command */ EXPLAIN(Build.current().isSnapshot()), + /** + * Support improved behavior for LIKE operator when used with index fields. + */ + LIKE_ON_INDEX_FIELDS, /** * FUSE command diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java index 1a2fd81db4b6c..730cccb4dce45 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/TranslationAware.java @@ -7,8 +7,11 @@ package org.elasticsearch.xpack.esql.capabilities; +import org.apache.lucene.search.MultiTermQuery.RewriteMethod; import org.elasticsearch.compute.lucene.LuceneTopNSourceOperator; import org.elasticsearch.compute.operator.FilterOperator; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; @@ -47,6 +50,23 @@ static TranslationAware.Translatable translatable(Expression exp, LucenePushdown */ Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler); + /** + * Translates this expression into a Lucene {@link org.apache.lucene.search.Query}. + *

+ * Implementations should use the provided field type, rewrite method, and search execution context + * to construct an appropriate Lucene query for this expression. + * By default, this method throws {@link UnsupportedOperationException}; override it in subclasses + * that support Lucene query translation. + *

+ */ + default org.apache.lucene.search.Query asLuceneQuery( + MappedFieldType fieldType, + RewriteMethod constantScoreRewrite, + SearchExecutionContext context + ) { + throw new UnsupportedOperationException("asLuceneQuery is not implemented for " + getClass().getName()); + } + /** * Subinterface for expressions that can only process single values (and null out on MVs). */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java index 52dedcb670372..5a165d7c822a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWith.java @@ -151,7 +151,7 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand // TODO: Get the real FoldContext here var wildcardQuery = "*" + QueryParser.escape(BytesRefs.toString(suffix.fold(FoldContext.small()))); - return new WildcardQuery(source(), fieldName, wildcardQuery); + return new WildcardQuery(source(), fieldName, wildcardQuery, false, false); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java index f457bacf44268..3dc43e29f3fa9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWith.java @@ -148,7 +148,7 @@ public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHand // TODO: Get the real FoldContext here var wildcardQuery = QueryParser.escape(BytesRefs.toString(prefix.fold(FoldContext.small()))) + "*"; - return new WildcardQuery(source(), fieldName, wildcardQuery); + return new WildcardQuery(source(), fieldName, wildcardQuery, false, false); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java index 7f350243dc220..da7eb6d07c61e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java @@ -122,11 +122,14 @@ public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { var field = field(); LucenePushdownPredicates.checkIsPushableAttribute(field); - return translateField(handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field)); + return translateField( + handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field), + pushdownPredicates.flags().stringLikeOnIndex() + ); } // TODO: see whether escaping is needed - private Query translateField(String targetFieldName) { - return new WildcardQuery(source(), targetFieldName, pattern().asLuceneWildcard(), caseInsensitive()); + private Query translateField(String targetFieldName, boolean forceStringMatch) { + return new WildcardQuery(source(), targetFieldName, pattern().asLuceneWildcard(), caseInsensitive(), forceStringMatch); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLikeList.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLikeList.java index 182df940136ed..1155589dc8ab3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLikeList.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLikeList.java @@ -7,24 +7,32 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex; +import org.apache.lucene.search.MultiTermQuery.RewriteMethod; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPatternList; -import org.elasticsearch.xpack.esql.core.querydsl.query.AutomatonQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.io.stream.ExpressionQuery; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; import org.elasticsearch.xpack.esql.planner.TranslatorHandler; import java.io.IOException; +import java.util.function.Supplier; import java.util.stream.Collectors; public class WildcardLikeList extends RegexMatch { @@ -34,6 +42,30 @@ public class WildcardLikeList extends RegexMatch { WildcardLikeList::new ); + Supplier automatonSupplier = new Supplier<>() { + Automaton cached; + + @Override + public Automaton get() { + if (cached == null) { + cached = pattern().createAutomaton(caseInsensitive()); + } + return cached; + } + }; + + Supplier characterRunAutomatonSupplier = new Supplier<>() { + CharacterRunAutomaton cached; + + @Override + public CharacterRunAutomaton get() { + if (cached == null) { + cached = new CharacterRunAutomaton(automatonSupplier.get()); + } + return cached; + } + }; + /** * The documentation for this function is in WildcardLike, and shown to the users `LIKE` in the docs. */ @@ -92,10 +124,10 @@ protected WildcardLikeList replaceChild(Expression newLeft) { */ @Override public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { - if (pushdownPredicates.minTransportVersion() == null) { + if (supportsPushdown(pushdownPredicates.minTransportVersion())) { return pushdownPredicates.isPushableAttribute(field()) ? Translatable.YES : Translatable.NO; } else { - // The AutomatonQuery that we use right now isn't serializable. + // The ExpressionQuery we use isn't serializable to all nodes in the cluster. return Translatable.NO; } } @@ -108,20 +140,40 @@ public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { var field = field(); LucenePushdownPredicates.checkIsPushableAttribute(field); - return translateField(handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field)); + String targetFieldName = handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field); + return translateField(targetFieldName); } - /** - * Translates the field to a {@link WildcardQuery} using the first pattern in the list. - * Throws an {@link IllegalArgumentException} if the pattern list contains more than one pattern. - */ - private Query translateField(String targetFieldName) { - return new AutomatonQuery(source(), targetFieldName, pattern().createAutomaton(caseInsensitive()), getAutomatonDescription()); + private boolean supportsPushdown(TransportVersion version) { + return version == null || version.onOrAfter(TransportVersions.ESQL_FIXED_INDEX_LIKE); } - private String getAutomatonDescription() { + @Override + public org.apache.lucene.search.Query asLuceneQuery( + MappedFieldType fieldType, + RewriteMethod constantScoreRewrite, + SearchExecutionContext context + ) { + return fieldType.automatonQuery( + automatonSupplier, + characterRunAutomatonSupplier, + constantScoreRewrite, + context, + getLuceneQueryDescription() + ); + } + + private String getLuceneQueryDescription() { // we use the information used to create the automaton to describe the query here String patternDesc = pattern().patternList().stream().map(WildcardPattern::pattern).collect(Collectors.joining("\", \"")); return "LIKE(\"" + patternDesc + "\"), caseInsensitive=" + caseInsensitive(); } + + /** + * Translates the field to a {@link WildcardQuery} using the first pattern in the list. + * Throws an {@link IllegalArgumentException} if the pattern list contains more than one pattern. + */ + private Query translateField(String targetFieldName) { + return new ExpressionQuery(source(), targetFieldName, this); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQuery.java new file mode 100644 index 0000000000000..f3051b36adc06 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQuery.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.io.stream; + +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.util.Objects; + +/** + * Implements an Expression query, which matches documents based on a given expression. + */ +public class ExpressionQuery extends Query { + + private final String targetFieldName; + private final Expression expression; + + public ExpressionQuery(Source source, String targetFieldName, Expression expression) { + super(source); + this.targetFieldName = targetFieldName; + this.expression = expression; + } + + public String field() { + return targetFieldName; + } + + @Override + protected QueryBuilder asBuilder() { + return new ExpressionQueryBuilder(targetFieldName, expression); + } + + @Override + public int hashCode() { + return Objects.hash(targetFieldName, expression); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + ExpressionQuery other = (ExpressionQuery) obj; + return Objects.equals(targetFieldName, other.targetFieldName) && Objects.equals(expression, other.expression); + } + + @Override + protected String innerToString() { + return "ExpressionQuery{" + "field='" + targetFieldName + '\'' + '}'; + } + + @Override + public boolean containsPlan() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQueryBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQueryBuilder.java new file mode 100644 index 0000000000000..7c4d26f2dff86 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/ExpressionQueryBuilder.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.io.stream; + +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.MultiTermQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.esql.capabilities.TranslationAware; +import org.elasticsearch.xpack.esql.core.expression.Expression; + +import java.io.IOException; +import java.util.Objects; + +import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE; + +/** + * Implements an Expression query builder, which matches documents based on a given expression. + * The expression itself must provide the {@link TranslationAware#asLuceneQuery} interface to be translated into a Lucene query. + * It allows for serialization of the expression and generate an AutomatonQuery on the data node + * as Automaton does not support serialization. + */ +public class ExpressionQueryBuilder extends AbstractQueryBuilder implements MultiTermQueryBuilder { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + QueryBuilder.class, + "expressionQueryBuilder", + ExpressionQueryBuilder::new + ); + private final String fieldName; + private final Expression expression; + + public ExpressionQueryBuilder(String fieldName, Expression expression) { + if (Strings.isEmpty(fieldName)) { + throw new IllegalArgumentException("field name is null or empty"); + } + if (expression == null) { + throw new IllegalArgumentException("expression cannot be null"); + } + this.fieldName = fieldName; + this.expression = expression; + } + + /** + * Read from a stream. + */ + private ExpressionQueryBuilder(StreamInput in) throws IOException { + super(in); + fieldName = in.readString(); + assert in instanceof PlanStreamInput; + this.expression = in.readNamedWriteable(Expression.class); + } + + public Expression getExpression() { + return expression; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeString(this.fieldName); + assert out instanceof PlanStreamOutput; + out.writeNamedWriteable(expression); + } + + @Override + public String fieldName() { + return fieldName; + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(ENTRY.name); // Use the appropriate query name + builder.field("field", fieldName); + builder.field("expression", expression.toString()); + builder.endObject(); + } + + @Override + protected Query doToQuery(SearchExecutionContext context) { + if (expression instanceof TranslationAware translationAware) { + MappedFieldType fieldType = context.getFieldType(fieldName); + if (fieldType == null) { + return new MatchNoDocsQuery("Field [" + fieldName + "] does not exist"); + } + return translationAware.asLuceneQuery(fieldType, CONSTANT_SCORE_REWRITE, context); + } else { + throw new UnsupportedOperationException("ExpressionQueryBuilder does not support non-automaton expressions"); + } + } + + @Override + protected int doHashCode() { + return Objects.hash(fieldName, expression); + } + + @Override + protected boolean doEquals(ExpressionQueryBuilder other) { + return Objects.equals(fieldName, other.fieldName) && Objects.equals(expression, other.expression); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + throw new UnsupportedOperationException("AutomatonQueryBuilder does not support getMinimalSupportedVersion"); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamWrapperQueryBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamWrapperQueryBuilder.java new file mode 100644 index 0000000000000..bb9ca136f6d66 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamWrapperQueryBuilder.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.io.stream; + +import org.apache.lucene.search.Query; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.esql.session.Configuration; + +import java.io.IOException; + +/** + * A {@link QueryBuilder} that wraps another {@linkplain QueryBuilder} + * so it read with a {@link PlanStreamInput}. + */ +public class PlanStreamWrapperQueryBuilder implements QueryBuilder { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + QueryBuilder.class, + "planwrapper", + PlanStreamWrapperQueryBuilder::new + ); + + private final Configuration configuration; + private final QueryBuilder next; + + public PlanStreamWrapperQueryBuilder(Configuration configuration, QueryBuilder next) { + this.configuration = configuration; + this.next = next; + } + + public PlanStreamWrapperQueryBuilder(StreamInput in) throws IOException { + configuration = Configuration.readWithoutTables(in); + PlanStreamInput planStreamInput = new PlanStreamInput(in, in.namedWriteableRegistry(), configuration); + next = planStreamInput.readNamedWriteable(QueryBuilder.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + configuration.withoutTables().writeTo(out); + new PlanStreamOutput(out, configuration).writeNamedWriteable(next); + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.ESQL_FIXED_INDEX_LIKE; + } + + @Override + public Query toQuery(SearchExecutionContext context) throws IOException { + return next.toQuery(context); + } + + @Override + public QueryBuilder queryName(String queryName) { + next.queryName(queryName); + return this; + } + + @Override + public String queryName() { + return next.queryName(); + } + + @Override + public float boost() { + return next.boost(); + } + + @Override + public QueryBuilder boost(float boost) { + next.boost(boost); + return this; + } + + @Override + public String getName() { + return getWriteableName(); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return next.toXContent(builder, params); + } + + public QueryBuilder next() { + return next; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java index 22e07b45310fb..1be024c9af76a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java @@ -8,7 +8,8 @@ package org.elasticsearch.xpack.esql.optimizer; import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.stats.SearchStats; -public record LocalPhysicalOptimizerContext(Configuration configuration, FoldContext foldCtx, SearchStats searchStats) {} +public record LocalPhysicalOptimizerContext(EsqlFlags flags, Configuration configuration, FoldContext foldCtx, SearchStats searchStats) {} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java index 1e976ca2e6263..3e087bf64c1a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java @@ -77,14 +77,14 @@ public class EnableSpatialDistancePushdown extends PhysicalOptimizerRules.Parame protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext ctx) { PhysicalPlan plan = filterExec; if (filterExec.child() instanceof EsQueryExec esQueryExec) { - plan = rewrite(ctx.foldCtx(), filterExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats())); + plan = rewrite(ctx.foldCtx(), filterExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats(), ctx.flags())); } else if (filterExec.child() instanceof EvalExec evalExec && evalExec.child() instanceof EsQueryExec esQueryExec) { plan = rewriteBySplittingFilter( ctx.foldCtx(), filterExec, evalExec, esQueryExec, - LucenePushdownPredicates.from(ctx.searchStats()) + LucenePushdownPredicates.from(ctx.searchStats(), ctx.flags()) ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index fcae24cb09c19..aa9ea3b0e004b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Check; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.stats.SearchStats; /** @@ -51,6 +52,8 @@ public interface LucenePushdownPredicates { @Nullable TransportVersion minTransportVersion(); + EsqlFlags flags(); + /** * For TEXT fields, we need to check if the field has a subfield of type KEYWORD that can be used instead. */ @@ -122,18 +125,23 @@ static String pushableAttributeName(TypedAttribute attribute) { * In particular, it assumes TEXT fields have no exact subfields (underlying keyword field), * and that isAggregatable means indexed and has hasDocValues. */ - LucenePushdownPredicates DEFAULT = forCanMatch(null); + LucenePushdownPredicates DEFAULT = forCanMatch(null, new EsqlFlags(true)); /** * A {@link LucenePushdownPredicates} for use with the {@code can_match} phase. */ - static LucenePushdownPredicates forCanMatch(TransportVersion minTransportVersion) { + static LucenePushdownPredicates forCanMatch(TransportVersion minTransportVersion, EsqlFlags flags) { return new LucenePushdownPredicates() { @Override public TransportVersion minTransportVersion() { return minTransportVersion; } + @Override + public EsqlFlags flags() { + return flags; + } + @Override public boolean hasExactSubfield(FieldAttribute attr) { return false; @@ -162,7 +170,7 @@ public boolean canUseEqualityOnSyntheticSourceDelegate(FieldAttribute attr, Stri * If we have access to {@link SearchStats} over a collection of shards, we can make more fine-grained decisions about what can be * pushed down. This should open up more opportunities for lucene pushdown. */ - static LucenePushdownPredicates from(SearchStats stats) { + static LucenePushdownPredicates from(SearchStats stats, EsqlFlags flags) { // TODO: use FieldAttribute#fieldName, otherwise this doesn't apply to field attributes used for union types. // C.f. https://github.com/elastic/elasticsearch/issues/128905 return new LucenePushdownPredicates() { @@ -171,6 +179,11 @@ public TransportVersion minTransportVersion() { return null; } + @Override + public EsqlFlags flags() { + return flags; + } + @Override public boolean hasExactSubfield(FieldAttribute attr) { return stats.hasExactSubfield(new FieldAttribute.FieldName(attr.name())); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java index 1f8341c4768d2..ba382b9800ece 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushFiltersToSource.java @@ -53,7 +53,7 @@ protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext } private static PhysicalPlan planFilterExec(FilterExec filterExec, EsQueryExec queryExec, LocalPhysicalOptimizerContext ctx) { - LucenePushdownPredicates pushdownPredicates = LucenePushdownPredicates.from(ctx.searchStats()); + LucenePushdownPredicates pushdownPredicates = LucenePushdownPredicates.from(ctx.searchStats(), ctx.flags()); List pushable = new ArrayList<>(); List nonPushable = new ArrayList<>(); for (Expression exp : splitAnd(filterExec.condition())) { @@ -75,7 +75,7 @@ private static PhysicalPlan planFilterExec( EsQueryExec queryExec, LocalPhysicalOptimizerContext ctx ) { - LucenePushdownPredicates pushdownPredicates = LucenePushdownPredicates.from(ctx.searchStats()); + LucenePushdownPredicates pushdownPredicates = LucenePushdownPredicates.from(ctx.searchStats(), ctx.flags()); AttributeMap aliasReplacedBy = getAliasReplacedBy(evalExec); List pushable = new ArrayList<>(); List nonPushable = new ArrayList<>(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java index 02d2f49605ced..8ec6d6b4bee39 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java @@ -63,7 +63,7 @@ public class PushTopNToSource extends PhysicalOptimizerRules.ParameterizedOptimi @Override protected PhysicalPlan rule(TopNExec topNExec, LocalPhysicalOptimizerContext ctx) { - Pushable pushable = evaluatePushable(ctx.foldCtx(), topNExec, LucenePushdownPredicates.from(ctx.searchStats())); + Pushable pushable = evaluatePushable(ctx.foldCtx(), topNExec, LucenePushdownPredicates.from(ctx.searchStats(), ctx.flags())); return pushable.rewrite(topNExec); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java index a8ee18d8b2777..326aa183e1891 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlParser.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; import java.util.BitSet; @@ -98,20 +99,20 @@ public void setEsqlConfig(EsqlConfig config) { } // testing utility - public LogicalPlan createStatement(String query) { - return createStatement(query, new QueryParams()); + public LogicalPlan createStatement(String query, Configuration configuration) { + return createStatement(query, new QueryParams(), configuration); } // testing utility - public LogicalPlan createStatement(String query, QueryParams params) { - return createStatement(query, params, new PlanTelemetry(new EsqlFunctionRegistry())); + public LogicalPlan createStatement(String query, QueryParams params, Configuration configuration) { + return createStatement(query, params, new PlanTelemetry(new EsqlFunctionRegistry()), configuration); } - public LogicalPlan createStatement(String query, QueryParams params, PlanTelemetry metrics) { + public LogicalPlan createStatement(String query, QueryParams params, PlanTelemetry metrics, Configuration configuration) { if (log.isDebugEnabled()) { log.debug("Parsing as statement: {}", query); } - return invokeParser(query, params, metrics, EsqlBaseParser::singleStatement, AstBuilder::plan); + return invokeParser(query, params, metrics, EsqlBaseParser::singleStatement, AstBuilder::plan, configuration); } private T invokeParser( @@ -119,7 +120,8 @@ private T invokeParser( QueryParams params, PlanTelemetry metrics, Function parseFunction, - BiFunction result + BiFunction result, + Configuration configuration ) { if (query.length() > MAX_LENGTH) { throw new ParsingException("ESQL statement is too large [{} characters > {}]", query.length(), MAX_LENGTH); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 9c021b1ba193b..e96f65d372587 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -90,7 +90,6 @@ import java.util.function.Function; import static java.util.Collections.emptyList; -import static org.elasticsearch.xpack.esql.common.Failure.fail; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index 614d5901c519f..da0ea1c0adff6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -26,11 +26,13 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.core.util.Queries; import org.elasticsearch.xpack.esql.expression.predicate.Predicates; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder; import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.LocalPhysicalOptimizerContext; @@ -50,6 +52,7 @@ import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.mapper.LocalMapper; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.stats.SearchContextStats; import org.elasticsearch.xpack.esql.stats.SearchStats; @@ -163,17 +166,26 @@ private static void forEachRelation(PhysicalPlan plan, Consumer acti } public static PhysicalPlan localPlan( + EsqlFlags flags, List searchContexts, Configuration configuration, FoldContext foldCtx, PhysicalPlan plan ) { - return localPlan(configuration, foldCtx, plan, SearchContextStats.from(searchContexts)); + return localPlan(flags, configuration, foldCtx, plan, SearchContextStats.from(searchContexts)); } - public static PhysicalPlan localPlan(Configuration configuration, FoldContext foldCtx, PhysicalPlan plan, SearchStats searchStats) { + public static PhysicalPlan localPlan( + EsqlFlags flags, + Configuration configuration, + FoldContext foldCtx, + PhysicalPlan plan, + SearchStats searchStats + ) { final var logicalOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats)); - var physicalOptimizer = new LocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, foldCtx, searchStats)); + var physicalOptimizer = new LocalPhysicalPlanOptimizer( + new LocalPhysicalOptimizerContext(flags, configuration, foldCtx, searchStats) + ); return localPlan(plan, logicalOptimizer, physicalOptimizer); } @@ -213,8 +225,13 @@ public static PhysicalPlan localPlan( /** * Extracts a filter that can be used to skip unmatched shards on the coordinator. */ - public static QueryBuilder canMatchFilter(TransportVersion minTransportVersion, PhysicalPlan plan) { - return detectFilter(minTransportVersion, plan, CoordinatorRewriteContext.SUPPORTED_FIELDS::contains); + public static QueryBuilder canMatchFilter( + EsqlFlags flags, + Configuration configuration, + TransportVersion minTransportVersion, + PhysicalPlan plan + ) { + return detectFilter(flags, configuration, minTransportVersion, plan, CoordinatorRewriteContext.SUPPORTED_FIELDS::contains); } /** @@ -222,10 +239,16 @@ public static QueryBuilder canMatchFilter(TransportVersion minTransportVersion, * We currently only use this filter for the @timestamp field, which is always a date field. Any tests that wish to use this should * take care to not use it with TEXT fields. */ - static QueryBuilder detectFilter(TransportVersion minTransportVersion, PhysicalPlan plan, Predicate fieldName) { + static QueryBuilder detectFilter( + EsqlFlags flags, + Configuration configuration, + TransportVersion minTransportVersion, + PhysicalPlan plan, + Predicate fieldName + ) { // first position is the REST filter, the second the query filter final List requestFilters = new ArrayList<>(); - final LucenePushdownPredicates ctx = LucenePushdownPredicates.forCanMatch(minTransportVersion); + final LucenePushdownPredicates ctx = LucenePushdownPredicates.forCanMatch(minTransportVersion, flags); plan.forEachDown(FragmentExec.class, fe -> { if (fe.esFilter() != null && fe.esFilter().supportsVersion(minTransportVersion)) { requestFilters.add(fe.esFilter()); @@ -253,7 +276,12 @@ && translatable(exp, ctx).finish() == TranslationAware.FinishedTranslatable.YES) } } if (matches.isEmpty() == false) { - requestFilters.add(TRANSLATOR_HANDLER.asQuery(ctx, Predicates.combineAnd(matches)).toQueryBuilder()); + Query qlQuery = TRANSLATOR_HANDLER.asQuery(ctx, Predicates.combineAnd(matches)); + QueryBuilder builder = qlQuery.toQueryBuilder(); + if (qlQuery.containsPlan()) { + builder = new PlanStreamWrapperQueryBuilder(configuration, builder); + } + requestFilters.add(builder); } }); }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java index cd4ff13700515..5064b2bbd101a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ClusterComputeHandler.java @@ -250,6 +250,7 @@ void runComputeOnRemoteCluster( final String localSessionId = clusterAlias + ":" + globalSessionId; final PhysicalPlan coordinatorPlan = ComputeService.reductionPlan(plan, true); final AtomicReference finalResponse = new AtomicReference<>(); + final EsqlFlags flags = computeService.createFlags(); final long startTimeInNanos = System.nanoTime(); final Runnable cancelQueryOnFailure = computeService.cancelQueryOnFailure(parentTask); try (var computeListener = new ComputeListener(transportService.getThreadPool(), cancelQueryOnFailure, listener.map(profiles -> { @@ -269,6 +270,7 @@ void runComputeOnRemoteCluster( localSessionId, "remote_reduce", clusterAlias, + flags, List.of(), configuration, configuration.newFoldContext(), @@ -282,6 +284,7 @@ void runComputeOnRemoteCluster( localSessionId, clusterAlias, parentTask, + flags, configuration, plan, concreteIndices, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeContext.java index dd94c52263235..d771de83685eb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeContext.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeContext.java @@ -21,6 +21,7 @@ record ComputeContext( String sessionId, String description, String clusterAlias, + EsqlFlags flags, List searchContexts, Configuration configuration, FoldContext foldCtx, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index 4adc97d28fee0..7053b02eaafd5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -183,6 +183,7 @@ public ComputeService( public void execute( String sessionId, CancellableTask rootTask, + EsqlFlags flags, PhysicalPlan physicalPlan, Configuration configuration, FoldContext foldContext, @@ -195,7 +196,7 @@ public void execute( // we have no sub plans, so we can just execute the given plan if (subplans == null || subplans.isEmpty()) { - executePlan(sessionId, rootTask, physicalPlan, configuration, foldContext, execInfo, null, listener, null); + executePlan(sessionId, rootTask, flags, physicalPlan, configuration, foldContext, execInfo, null, listener, null); return; } @@ -222,6 +223,7 @@ public void execute( mainSessionId, "main.final", LOCAL_CLUSTER, + flags, List.of(), configuration, foldContext, @@ -254,6 +256,7 @@ public void execute( executePlan( childSessionId, rootTask, + flags, subplan, configuration, foldContext, @@ -278,6 +281,7 @@ public void execute( public void executePlan( String sessionId, CancellableTask rootTask, + EsqlFlags flags, PhysicalPlan physicalPlan, Configuration configuration, FoldContext foldContext, @@ -322,6 +326,7 @@ public void executePlan( newChildSession(sessionId), profileDescription(profileQualifier, "single"), LOCAL_CLUSTER, + flags, List.of(), configuration, foldContext, @@ -408,6 +413,7 @@ public void executePlan( sessionId, profileDescription(profileQualifier, "final"), LOCAL_CLUSTER, + flags, List.of(), configuration, foldContext, @@ -424,6 +430,7 @@ public void executePlan( sessionId, LOCAL_CLUSTER, rootTask, + flags, configuration, dataNodePlan, Set.of(localConcreteIndices.indices()), @@ -576,7 +583,13 @@ public SourceProvider createSourceProvider() { LOGGER.debug("Received physical plan:\n{}", plan); - var localPlan = PlannerUtils.localPlan(context.searchExecutionContexts(), context.configuration(), context.foldCtx(), plan); + var localPlan = PlannerUtils.localPlan( + context.flags(), + context.searchExecutionContexts(), + context.configuration(), + context.foldCtx(), + plan + ); // the planner will also set the driver parallelism in LocalExecutionPlanner.LocalExecutionPlan (used down below) // it's doing this in the planning of EsQueryExec (the source of the data) // see also EsPhysicalOperationProviders.sourcePhysicalOperation @@ -652,6 +665,10 @@ CancellableTask createGroupTask(Task parentTask, Supplier description) t } } + public EsqlFlags createFlags() { + return new EsqlFlags(clusterService.getClusterSettings()); + } + private static class ComputeGroupTaskRequest extends AbstractTransportRequest { private final Supplier parentDescription; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java index abf321f3564ea..aea1c7d6d2d67 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/DataNodeComputeHandler.java @@ -99,6 +99,7 @@ void startComputeOnDataNodes( String sessionId, String clusterAlias, CancellableTask parentTask, + EsqlFlags flags, Configuration configuration, PhysicalPlan dataNodePlan, Set concreteIndices, @@ -116,7 +117,7 @@ void startComputeOnDataNodes( esqlExecutor, parentTask, originalIndices, - PlannerUtils.canMatchFilter(clusterService.state().getMinTransportVersion(), dataNodePlan), + PlannerUtils.canMatchFilter(flags, configuration, clusterService.state().getMinTransportVersion(), dataNodePlan), clusterAlias, configuration.allowPartialResults(), maxConcurrentNodesPerCluster == null ? -1 : maxConcurrentNodesPerCluster, @@ -219,6 +220,7 @@ protected void sendRequest( } private class DataNodeRequestExecutor { + private final EsqlFlags flags; private final DataNodeRequest request; private final CancellableTask parentTask; private final ExchangeSinkHandler exchangeSink; @@ -229,6 +231,7 @@ private class DataNodeRequestExecutor { private final Map shardLevelFailures; DataNodeRequestExecutor( + EsqlFlags flags, DataNodeRequest request, CancellableTask parentTask, ExchangeSinkHandler exchangeSink, @@ -237,6 +240,7 @@ private class DataNodeRequestExecutor { Map shardLevelFailures, ComputeListener computeListener ) { + this.flags = flags; this.request = request; this.parentTask = parentTask; this.exchangeSink = exchangeSink; @@ -297,6 +301,7 @@ public void onFailure(Exception e) { sessionId, "data", clusterAlias, + flags, searchContexts, configuration, configuration.newFoldContext(), @@ -422,7 +427,9 @@ private void runComputeOnDataNode( exchangeService.finishSinkHandler(externalId, new TaskCancelledException(task.getReasonCancelled())); exchangeService.finishSinkHandler(request.sessionId(), new TaskCancelledException(task.getReasonCancelled())); }); + EsqlFlags flags = computeService.createFlags(); DataNodeRequestExecutor dataNodeRequestExecutor = new DataNodeRequestExecutor( + flags, request, task, internalSink, @@ -442,6 +449,7 @@ private void runComputeOnDataNode( request.sessionId(), "node_reduce", request.clusterAlias(), + flags, List.of(), request.configuration(), new FoldContext(request.pragmas().foldLimit().getBytes()), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFlags.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFlags.java new file mode 100644 index 0000000000000..09ef93d56e9ee --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFlags.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.plugin; + +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.Setting; + +public class EsqlFlags { + public static final Setting ESQL_STRING_LIKE_ON_INDEX = Setting.boolSetting( + "esql.query.string_like_on_index", + true, + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + + private final boolean stringLikeOnIndex; + + public EsqlFlags(boolean stringLikeOnIndex) { + this.stringLikeOnIndex = stringLikeOnIndex; + } + + public EsqlFlags(ClusterSettings settings) { + this.stringLikeOnIndex = settings.get(ESQL_STRING_LIKE_ON_INDEX); + } + + public boolean stringLikeOnIndex() { + return stringLikeOnIndex; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 293a7be6be041..9561f3abeb53e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -72,6 +72,8 @@ import org.elasticsearch.xpack.esql.enrich.LookupFromIndexOperator; import org.elasticsearch.xpack.esql.execution.PlanExecutor; import org.elasticsearch.xpack.esql.expression.ExpressionWritables; +import org.elasticsearch.xpack.esql.io.stream.ExpressionQueryBuilder; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder; import org.elasticsearch.xpack.esql.plan.PlanWritables; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import org.elasticsearch.xpack.esql.querylog.EsqlQueryLog; @@ -262,7 +264,8 @@ public List> getSettings() { ESQL_QUERYLOG_THRESHOLD_WARN_SETTING, ESQL_QUERYLOG_INCLUDE_USER_SETTING, DEFAULT_DATA_PARTITIONING, - STORED_FIELDS_SEQUENTIAL_PROPORTION + STORED_FIELDS_SEQUENTIAL_PROPORTION, + EsqlFlags.ESQL_STRING_LIKE_ON_INDEX ); } @@ -325,6 +328,8 @@ public List getNamedWriteables() { entries.add(AsyncOperator.Status.ENTRY); entries.add(EnrichLookupOperator.Status.ENTRY); entries.add(LookupFromIndexOperator.Status.ENTRY); + entries.add(ExpressionQueryBuilder.ENTRY); + entries.add(PlanStreamWrapperQueryBuilder.ENTRY); entries.addAll(ExpressionWritables.getNamedWriteables()); entries.addAll(PlanWritables.getNamedWriteables()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index 4b83301a16488..4be7b31bb96c0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -181,6 +181,7 @@ public TransportEsqlQueryAction( defaultAllowPartialResults = EsqlPlugin.QUERY_ALLOW_PARTIAL_RESULTS.get(clusterService.getSettings()); clusterService.getClusterSettings() .addSettingsUpdateConsumer(EsqlPlugin.QUERY_ALLOW_PARTIAL_RESULTS, v -> defaultAllowPartialResults = v); + } @Override @@ -220,6 +221,7 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener computeService.execute( sessionId, (CancellableTask) task, + flags, plan, configuration, foldCtx, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java index 1db12c873a763..812e741c64d29 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/EqualsSyntheticSourceDelegate.java @@ -35,6 +35,11 @@ protected String innerToString() { return fieldName + "(delegate):" + value; } + @Override + public boolean containsPlan() { + return false; + } + private class Builder extends BaseTermQueryBuilder { private Builder(String name, String value) { super(name, value); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java index aa0e896dfc013..2946af2ac5c23 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KnnQuery.java @@ -81,4 +81,9 @@ public int hashCode() { public boolean scorable() { return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java index 7ef272f84ec9f..56229ec325d73 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/KqlQuery.java @@ -87,4 +87,9 @@ protected String innerToString() { public boolean scorable() { return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchPhraseQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchPhraseQuery.java index be6f244ac4acf..6a10542687490 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchPhraseQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchPhraseQuery.java @@ -108,4 +108,9 @@ public Map options() { public boolean scorable() { return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java index 57489cc930bf2..d3d9d810d78fc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MatchQuery.java @@ -130,4 +130,9 @@ public Map options() { public boolean scorable() { return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java index 5526283dfe5a0..0caa22ef92669 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/MultiMatchQuery.java @@ -114,4 +114,9 @@ protected String innerToString() { public boolean scorable() { return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java index 753b36aa5e3d9..ca1816feba426 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueQuery.java @@ -125,6 +125,11 @@ public int hashCode() { return Objects.hash(super.hashCode(), next, field, useSyntheticSourceDelegate); } + @Override + public boolean containsPlan() { + return next.containsPlan(); + } + public abstract static class AbstractBuilder extends AbstractQueryBuilder { private final QueryBuilder next; private final String field; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java index bf0fa72d3af41..98939b5a495f6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SpatialRelatesQuery.java @@ -86,6 +86,11 @@ public ShapeRelation shapeRelation() { }; } + @Override + public boolean containsPlan() { + return false; + } + /** * This class is a minimal implementation of the QueryBuilder interface. * We only need the toQuery method, but ESQL makes extensive use of QueryBuilder and trimming that interface down for ESQL only would diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java index d61101c2f594c..8106f6c6661aa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/TranslationAwareExpressionQuery.java @@ -40,4 +40,9 @@ public boolean scorable() { // All Full Text Functions are translated to queries using this method return true; } + + @Override + public boolean containsPlan() { + return false; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java index 928c7734cf422..0f66a839bb429 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/Configuration.java @@ -211,6 +211,23 @@ public Map> tables() { return tables; } + public Configuration withoutTables() { + return new Configuration( + zoneId, + locale, + username, + clusterName, + pragmas, + resultTruncationMaxSize, + resultTruncationDefaultSize, + query, + profile, + Map.of(), + queryStartTimeNanos, + allowPartialResults + ); + } + /** * Enable profiling, sacrificing performance to return information about * what operations are taking the most time. @@ -309,4 +326,11 @@ public String toString() { + '}'; } + /** + * Reads a {@link Configuration} that doesn't contain any {@link Configuration#tables()}. + */ + public static Configuration readWithoutTables(StreamInput in) throws IOException { + BlockStreamInput blockStreamInput = new BlockStreamInput(in, null); + return new Configuration(blockStreamInput); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 4ff65f59bbd72..fdf884500a8bd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -326,7 +326,7 @@ private LocalRelation resultToPlan(LogicalPlan plan, Result result) { } private LogicalPlan parse(String query, QueryParams params) { - var parsed = new EsqlParser().createStatement(query, params, planTelemetry); + var parsed = new EsqlParser().createStatement(query, params, planTelemetry, configuration); LOGGER.debug("Parsed logical plan:\n{}", parsed); return parsed; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 9062bdef62d76..bdf2ba39edc66 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -88,6 +88,7 @@ import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.planner.TestPhysicalOperationProviders; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.session.EsqlSession; @@ -566,7 +567,7 @@ private static TestPhysicalOperationProviders testOperationProviders( } private ActualResults executePlan(BigArrays bigArrays) throws Exception { - LogicalPlan parsed = parser.createStatement(testCase.query); + LogicalPlan parsed = parser.createStatement(testCase.query, EsqlTestUtils.TEST_CFG); var testDatasets = testDatasets(parsed); LogicalPlan analyzed = analyzedPlan(parsed, testDatasets); @@ -715,7 +716,7 @@ void executeSubPlan( var searchStats = new DisabledSearchStats(); var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats)); var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer( - new LocalPhysicalOptimizerContext(configuration, foldCtx, searchStats) + new LocalPhysicalOptimizerContext(new EsqlFlags(true), configuration, foldCtx, searchStats) ); var csvDataNodePhysicalPlan = PlannerUtils.localPlan(dataNodePlan, logicalTestOptimizer, physicalTestOptimizer); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index 5e6c37545a396..c806fec61dc64 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -129,7 +129,7 @@ public static LogicalPlan analyze(String query, String index, String mapping) { } public static LogicalPlan analyze(String query, Analyzer analyzer) { - var plan = new EsqlParser().createStatement(query); + var plan = new EsqlParser().createStatement(query, configuration(query)); // System.out.println(plan); var analyzed = analyzer.analyze(plan); // System.out.println(analyzed); @@ -137,7 +137,7 @@ public static LogicalPlan analyze(String query, Analyzer analyzer) { } public static LogicalPlan analyze(String query, String mapping, QueryParams params) { - var plan = new EsqlParser().createStatement(query, params); + var plan = new EsqlParser().createStatement(query, params, configuration(query)); var analyzer = analyzer(loadMapping(mapping, "test"), TEST_VERIFIER, configuration(query)); return analyzer.analyze(plan); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java index 348c4d4266b8e..1cda4bd599d61 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java @@ -96,7 +96,7 @@ public void testInlineCast() throws IOException { if (EsqlDataTypeConverter.converterFunctionFactory(expectedType) == null) { continue; } - LogicalPlan plan = parser.createStatement("ROW a = 1::" + nameOrAlias); + LogicalPlan plan = parser.createStatement("ROW a = 1::" + nameOrAlias, TEST_CFG); Row row = as(plan, Row.class); assertThat(row.fields(), hasSize(1)); Function functionCall = (Function) row.fields().get(0).child(); @@ -182,7 +182,7 @@ public void testInvalidSample() { } private String error(String query) { - ParsingException e = expectThrows(ParsingException.class, () -> defaultAnalyzer.analyze(parser.createStatement(query))); + ParsingException e = expectThrows(ParsingException.class, () -> defaultAnalyzer.analyze(parser.createStatement(query, TEST_CFG))); String message = e.getMessage(); assertTrue(message.startsWith("line ")); return message.substring("line ".length()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 6555a303592f3..fe900fbc211eb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.Set; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.esql.EsqlTestUtils.paramAsConstant; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; @@ -2241,7 +2242,7 @@ private void query(String query) { } private void query(String query, Analyzer analyzer) { - analyzer.analyze(parser.createStatement(query)); + analyzer.analyze(parser.createStatement(query, TEST_CFG)); } private String error(String query) { @@ -2272,7 +2273,7 @@ private String error(String query, Analyzer analyzer, Class Throwable e = expectThrows( exception, "Expected error for query [" + query + "] but no error was raised", - () -> analyzer.analyze(parser.createStatement(query, new QueryParams(parameters))) + () -> analyzer.analyze(parser.createStatement(query, new QueryParams(parameters), TEST_CFG)) ); assertThat(e, instanceOf(exception)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java index 68a6f38cdd69a..039b34252c12e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java @@ -79,7 +79,7 @@ public EsqlFunctionRegistry snapshotRegistry() { } }; - var plan = parser.createStatement(esql); + var plan = parser.createStatement(esql, EsqlTestUtils.TEST_CFG); plan = plan.transformDown( Limit.class, l -> Objects.equals(l.limit().fold(FoldContext.small()), 10) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java index d6a6106f9372e..9c644d30f24c5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/EndsWithTests.java @@ -134,6 +134,6 @@ public void testLuceneQuery_NonFoldableSuffix_Translatable() { var query = function.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER); - assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "*a\\*b\\?c\\\\"))); + assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "*a\\*b\\?c\\\\", false, true))); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java index c716457dd8378..9118397a0b34a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/StartsWithTests.java @@ -94,6 +94,6 @@ public void testLuceneQuery_NonFoldablePrefix_Translatable() { var query = function.asQuery(LucenePushdownPredicates.DEFAULT, TranslatorHandler.TRANSLATOR_HANDLER); - assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "a\\*b\\?c\\\\*"))); + assertThat(query, equalTo(new WildcardQuery(Source.EMPTY, "field", "a\\*b\\?c\\\\*", false, true))); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeListTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeListTests.java index 0de1a098507fa..78e4c951829d5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeListTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeListTests.java @@ -11,7 +11,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.TransportVersion; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -26,12 +25,14 @@ import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLikeList; import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import java.util.ArrayList; import java.util.List; import java.util.function.Function; import java.util.function.Supplier; +import static org.elasticsearch.TransportVersions.V_8_17_0; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.startsWith; @@ -104,7 +105,7 @@ static Expression buildWildcardLikeList(Source source, List args) { public void testNotPushableOverCanMatch() { TranslationAware translatable = (TranslationAware) buildFieldExpression(testCase); assertThat( - translatable.translatable(LucenePushdownPredicates.forCanMatch(TransportVersion.current())).finish(), + translatable.translatable(LucenePushdownPredicates.forCanMatch(V_8_17_0, new EsqlFlags(true))).finish(), equalTo(TranslationAware.FinishedTranslatable.NO) ); } @@ -112,7 +113,7 @@ public void testNotPushableOverCanMatch() { public void testPushable() { TranslationAware translatable = (TranslationAware) buildFieldExpression(testCase); assertThat( - translatable.translatable(LucenePushdownPredicates.from(new EsqlTestUtils.TestSearchStats())).finish(), + translatable.translatable(LucenePushdownPredicates.from(new EsqlTestUtils.TestSearchStats(), new EsqlFlags(true))).finish(), equalTo(TranslationAware.FinishedTranslatable.YES) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java index a9e9e5f917785..858d7235b2c70 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java @@ -83,7 +83,7 @@ protected Expression build(Source source, List args) { return buildWildcardLike(source, args); } - static Expression buildWildcardLike(Source source, List args) { + Expression buildWildcardLike(Source source, List args) { Expression expression = args.get(0); Literal pattern = (Literal) args.get(1); Literal caseInsensitive = args.size() > 2 ? (Literal) args.get(2) : null; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java index bff9843a89e1f..4263b8d07c8f2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java @@ -180,7 +180,7 @@ protected LogicalPlan plan(String query) { } protected LogicalPlan plan(String query, LogicalPlanOptimizer optimizer) { - var analyzed = analyzer.analyze(parser.createStatement(query)); + var analyzed = analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); // System.out.println(analyzed); var optimized = optimizer.optimize(analyzed); // System.out.println(optimized); @@ -188,7 +188,7 @@ protected LogicalPlan plan(String query, LogicalPlanOptimizer optimizer) { } protected LogicalPlan planAirports(String query) { - var analyzed = analyzerAirports.analyze(parser.createStatement(query)); + var analyzed = analyzerAirports.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); // System.out.println(analyzed); var optimized = logicalOptimizer.optimize(analyzed); // System.out.println(optimized); @@ -196,7 +196,7 @@ protected LogicalPlan planAirports(String query) { } protected LogicalPlan planExtra(String query) { - var analyzed = analyzerExtra.analyze(parser.createStatement(query)); + var analyzed = analyzerExtra.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); // System.out.println(analyzed); var optimized = logicalOptimizer.optimize(analyzed); // System.out.println(optimized); @@ -204,11 +204,11 @@ protected LogicalPlan planExtra(String query) { } protected LogicalPlan planTypes(String query) { - return logicalOptimizer.optimize(analyzerTypes.analyze(parser.createStatement(query))); + return logicalOptimizer.optimize(analyzerTypes.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); } protected LogicalPlan planMultiIndex(String query) { - return logicalOptimizer.optimize(multiIndexAnalyzer.analyze(parser.createStatement(query))); + return logicalOptimizer.optimize(multiIndexAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index f2b70c99253b8..0009b6ac897dc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -508,7 +508,7 @@ public void testSparseDocument() throws Exception { TEST_VERIFIER ); - var analyzed = analyzer.analyze(parser.createStatement(query)); + var analyzed = analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); var optimized = logicalOptimizer.optimize(analyzed); var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small(), searchStats); var plan = new LocalLogicalPlanOptimizer(localContext).localOptimize(optimized); @@ -785,7 +785,7 @@ private LocalRelation asEmptyRelation(Object o) { } private LogicalPlan plan(String query, Analyzer analyzer) { - var analyzed = analyzer.analyze(parser.createStatement(query)); + var analyzed = analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); // System.out.println(analyzed); var optimized = logicalOptimizer.optimize(analyzed); // System.out.println(optimized); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 7795bf5c2d9ff..2cd28b817a184 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -5254,20 +5254,26 @@ public void testEmptyMappingIndex() { TEST_VERIFIER ); - var plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test"))); + var plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test", EsqlTestUtils.TEST_CFG))); as(plan, LocalRelation.class); assertThat(plan.output(), equalTo(NO_FIELDS)); - plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test metadata _id | eval x = 1"))); + plan = logicalOptimizer.optimize( + analyzer.analyze(parser.createStatement("from empty_test metadata _id | eval x = 1", EsqlTestUtils.TEST_CFG)) + ); as(plan, LocalRelation.class); assertThat(Expressions.names(plan.output()), contains("_id", "x")); - plan = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement("from empty_test metadata _id, _version | limit 5"))); + plan = logicalOptimizer.optimize( + analyzer.analyze(parser.createStatement("from empty_test metadata _id, _version | limit 5", EsqlTestUtils.TEST_CFG)) + ); as(plan, LocalRelation.class); assertThat(Expressions.names(plan.output()), contains("_id", "_version")); plan = logicalOptimizer.optimize( - analyzer.analyze(parser.createStatement("from empty_test | eval x = \"abc\" | enrich languages_idx on x")) + analyzer.analyze( + parser.createStatement("from empty_test | eval x = \"abc\" | enrich languages_idx on x", EsqlTestUtils.TEST_CFG) + ) ); LocalRelation local = as(plan, LocalRelation.class); assertThat(Expressions.names(local.output()), contains(NO_FIELDS.get(0).name(), "x", "language_code", "language_name")); @@ -5962,7 +5968,7 @@ private void doTestSimplifyComparisonArithmetics( private void assertSemanticMatching(String expected, String provided) { BinaryComparison bc = extractPlannedBinaryComparison(provided); - LogicalPlan exp = analyzerTypes.analyze(parser.createStatement("FROM types | WHERE " + expected)); + LogicalPlan exp = analyzerTypes.analyze(parser.createStatement("FROM types | WHERE " + expected, EsqlTestUtils.TEST_CFG)); assertSemanticMatching(bc, extractPlannedBinaryComparison(exp)); } @@ -5990,7 +5996,7 @@ private Expression getComparisonFromLogicalPlan(LogicalPlan plan) { private void assertNotSimplified(String comparison) { String query = "FROM types | WHERE " + comparison; Expression optimized = getComparisonFromLogicalPlan(planTypes(query)); - Expression raw = getComparisonFromLogicalPlan(analyzerTypes.analyze(parser.createStatement(query))); + Expression raw = getComparisonFromLogicalPlan(analyzerTypes.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); assertTrue(raw.semanticEquals(optimized)); } @@ -6683,7 +6689,7 @@ public void testMultipleLookupShadowing() { public void testTranslateMetricsWithoutGrouping() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "TS k8s | STATS max(rate(network.total_bytes_in))"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Limit limit = as(plan, Limit.class); Aggregate finalAggs = as(limit.child(), Aggregate.class); assertThat(finalAggs, not(instanceOf(TimeSeriesAggregate.class))); @@ -6704,7 +6710,7 @@ public void testTranslateMetricsWithoutGrouping() { public void testTranslateMixedAggsWithoutGrouping() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "TS k8s | STATS max(rate(network.total_bytes_in)), max(network.cost)"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Limit limit = as(plan, Limit.class); Aggregate finalAggs = as(limit.child(), Aggregate.class); assertThat(finalAggs, not(instanceOf(TimeSeriesAggregate.class))); @@ -6729,7 +6735,7 @@ public void testTranslateMixedAggsWithoutGrouping() { public void testTranslateMixedAggsWithMathWithoutGrouping() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "TS k8s | STATS max(rate(network.total_bytes_in)), max(network.cost + 0.2) * 1.1"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Project project = as(plan, Project.class); Eval mulEval = as(project.child(), Eval.class); assertThat(mulEval.fields(), hasSize(1)); @@ -6767,7 +6773,7 @@ public void testTranslateMixedAggsWithMathWithoutGrouping() { public void testTranslateMetricsGroupedByOneDimension() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "TS k8s | STATS sum(rate(network.total_bytes_in)) BY cluster | SORT cluster | LIMIT 10"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); TopN topN = as(plan, TopN.class); Aggregate aggsByCluster = as(topN.child(), Aggregate.class); assertThat(aggsByCluster, not(instanceOf(TimeSeriesAggregate.class))); @@ -6792,7 +6798,7 @@ public void testTranslateMetricsGroupedByOneDimension() { public void testTranslateMetricsGroupedByTwoDimension() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "TS k8s | STATS avg(rate(network.total_bytes_in)) BY cluster, pod"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Project project = as(plan, Project.class); Eval eval = as(project.child(), Eval.class); assertThat(eval.fields(), hasSize(1)); @@ -6832,7 +6838,7 @@ public void testTranslateMetricsGroupedByTwoDimension() { public void testTranslateMetricsGroupedByTimeBucket() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "TS k8s | STATS sum(rate(network.total_bytes_in)) BY bucket(@timestamp, 1h)"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Limit limit = as(plan, Limit.class); Aggregate finalAgg = as(limit.child(), Aggregate.class); assertThat(finalAgg, not(instanceOf(TimeSeriesAggregate.class))); @@ -6866,7 +6872,7 @@ public void testTranslateMetricsGroupedByTimeBucketAndDimensions() { | SORT cluster | LIMIT 10 """; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Project project = as(plan, Project.class); TopN topN = as(project.child(), TopN.class); Eval eval = as(topN.child(), Eval.class); @@ -6908,7 +6914,7 @@ public void testTranslateSumOfTwoRates() { | SORT cluster | LIMIT 10 """; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); TopN topN = as(plan, TopN.class); Aggregate finalAgg = as(topN.child(), Aggregate.class); Eval eval = as(finalAgg.child(), Eval.class); @@ -6929,7 +6935,7 @@ public void testTranslateMixedAggsGroupedByTimeBucketAndDimensions() { | SORT cluster | LIMIT 10 """; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Project project = as(plan, Project.class); TopN topN = as(project.child(), TopN.class); Eval eval = as(topN.child(), Eval.class); @@ -6981,7 +6987,7 @@ public void testAdjustMetricsRateBeforeFinalAgg() { | SORT cluster | LIMIT 10 """; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Project project = as(plan, Project.class); TopN topN = as(project.child(), TopN.class); Eval evalDiv = as(topN.child(), Eval.class); @@ -7034,7 +7040,7 @@ public void testAdjustMetricsRateBeforeFinalAgg() { public void testTranslateMaxOverTime() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "TS k8s | STATS sum(max_over_time(network.bytes_in)) BY bucket(@timestamp, 1h)"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Limit limit = as(plan, Limit.class); Aggregate finalAgg = as(limit.child(), Aggregate.class); assertThat(finalAgg, not(instanceOf(TimeSeriesAggregate.class))); @@ -7063,7 +7069,7 @@ public void testTranslateMaxOverTime() { public void testTranslateAvgOverTime() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "TS k8s | STATS sum(avg_over_time(network.bytes_in)) BY bucket(@timestamp, 1h)"; - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); Limit limit = as(plan, Limit.class); Aggregate finalAgg = as(limit.child(), Aggregate.class); assertThat(finalAgg, not(instanceOf(TimeSeriesAggregate.class))); @@ -7103,7 +7109,7 @@ public void testMetricsWithoutRate() { """); List plans = new ArrayList<>(); for (String query : queries) { - var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query))); + var plan = logicalOptimizer.optimize(metricsAnalyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); plans.add(plan); } for (LogicalPlan plan : plans) { @@ -7694,7 +7700,7 @@ public void testPruneRedundantOrderBy() { | mv_expand x | sort y """; - LogicalPlan analyzed = analyzer.analyze(parser.createStatement(query)); + LogicalPlan analyzed = analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG)); LogicalPlan optimized = rule.apply(analyzed); // check that all the redundant SORTs are removed in a single run diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java index b36cb3f6c6a42..fe4fd96120a01 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; @@ -128,7 +129,7 @@ protected Expression rule(Expression e, LogicalOptimizerContext ctx) { }; rule.apply( - new EsqlParser().createStatement("FROM index | EVAL x=f1+1 | KEEP x, f2 | LIMIT 1"), + new EsqlParser().createStatement("FROM index | EVAL x=f1+1 | KEEP x, f2 | LIMIT 1", EsqlTestUtils.TEST_CFG), new LogicalOptimizerContext(null, FoldContext.small()) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index 99eded20b1687..b6ef858bd00f3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -136,6 +136,7 @@ import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.querydsl.query.EqualsSyntheticSourceDelegate; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; @@ -7842,7 +7843,7 @@ private LocalExecutionPlanner.LocalExecutionPlan physicalOperationsFromPhysicalP // The TopN needs an estimated row size for the planner to work var plans = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(EstimatesRowSize.estimateRowSize(0, plan), config); plan = useDataNodePlan ? plans.v2() : plans.v1(); - plan = PlannerUtils.localPlan(config, FoldContext.small(), plan, TEST_SEARCH_STATS); + plan = PlannerUtils.localPlan(new EsqlFlags(true), config, FoldContext.small(), plan, TEST_SEARCH_STATS); ExchangeSinkHandler exchangeSinkHandler = new ExchangeSinkHandler(null, 10, () -> 10); LocalExecutionPlanner planner = new LocalExecutionPlanner( "test", @@ -8209,7 +8210,7 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) { // individually hence why here the plan is kept as is var l = p.transformUp(FragmentExec.class, fragment -> { - var localPlan = PlannerUtils.localPlan(config, FoldContext.small(), fragment, searchStats); + var localPlan = PlannerUtils.localPlan(new EsqlFlags(true), config, FoldContext.small(), fragment, searchStats); return EstimatesRowSize.estimateRowSize(fragment.estimatedRowSize(), localPlan); }); @@ -8247,7 +8248,7 @@ private PhysicalPlan physicalPlan(String query, TestDataSource dataSource) { } private PhysicalPlan physicalPlan(String query, TestDataSource dataSource, boolean assertSerialization) { - var logical = logicalOptimizer.optimize(dataSource.analyzer.analyze(parser.createStatement(query))); + var logical = logicalOptimizer.optimize(dataSource.analyzer.analyze(parser.createStatement(query, config))); // System.out.println("Logical\n" + logical); var physical = mapper.map(logical); // System.out.println("Physical\n" + physical); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java index e6a7d110f8c09..761902bebe19e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.stats.SearchStats; @@ -66,7 +67,7 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) { new LocalLogicalOptimizerContext(config, FoldContext.small(), searchStats) ); var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer( - new LocalPhysicalOptimizerContext(config, FoldContext.small(), searchStats), + new LocalPhysicalOptimizerContext(new EsqlFlags(true), config, FoldContext.small(), searchStats), true ); var l = PlannerUtils.localPlan(physicalPlan, logicalTestOptimizer, physicalTestOptimizer); @@ -79,7 +80,7 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) { } private PhysicalPlan physicalPlan(String query, Analyzer analyzer) { - var logical = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement(query))); + var logical = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); // System.out.println("Logical\n" + logical); var physical = mapper.map(logical); return physical; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java index 088b5c1c9205e..a7451ea2f5109 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java @@ -169,7 +169,7 @@ public void testGroupingAliasingMoved_To_LeftSideOfJoin_WithExpression() { } private LogicalPlan plan(String query, LogicalPlanOptimizer optimizer) { - return optimizer.optimize(analyzer.analyze(parser.createStatement(query))); + return optimizer.optimize(analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java index cacf6e422882b..00f8dfb9aaacc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.esql.plan.physical.EvalExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.plan.physical.TopNExec; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.stats.SearchStats; import java.io.IOException; @@ -416,7 +417,7 @@ private static void assertNoPushdownSort(TestPhysicalPlanBuilder builder, String private static PhysicalPlan pushTopNToSource(TopNExec topNExec) { var configuration = EsqlTestUtils.configuration("from test"); - var ctx = new LocalPhysicalOptimizerContext(configuration, FoldContext.small(), SearchStats.EMPTY); + var ctx = new LocalPhysicalOptimizerContext(new EsqlFlags(true), configuration, FoldContext.small(), SearchStats.EMPTY); var pushTopNToSource = new PushTopNToSource(); return pushTopNToSource.rule(topNExec, ctx); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java index 462b9f7373ecf..07ab624e4a2a9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -57,11 +58,11 @@ LogicalPlan statement(String e) { } LogicalPlan statement(String e, QueryParams params) { - return parser.createStatement(e, params); + return parser.createStatement(e, params, EsqlTestUtils.TEST_CFG); } LogicalPlan processingCommand(String e) { - return parser.createStatement("row a = 1 | " + e); + return parser.createStatement("row a = 1 | " + e, EsqlTestUtils.TEST_CFG); } static UnresolvedAttribute attribute(String name) { @@ -170,7 +171,7 @@ void expectVerificationError(String query, String errorMessage) { "Query [" + query + "] is expected to throw " + VerificationException.class + " with message [" + errorMessage + "]", VerificationException.class, containsString(errorMessage), - () -> parser.createStatement(query) + () -> parser.createStatement(query, EsqlTestUtils.TEST_CFG) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java index c39dc70f3ad39..0696f2c99e03e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; @@ -659,7 +660,7 @@ private Project projectExpression(String e) { } private LogicalPlan parse(String s) { - return parser.createStatement(s); + return parser.createStatement(s, EsqlTestUtils.TEST_CFG); } private Literal l(Object value, DataType type) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java index f31d9ee58c268..14ded33e297ee 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/GrammarInDevelopmentParsingTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.parser; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; @@ -35,7 +36,7 @@ public void testDevelopmentRerank() { } void parse(String query, String errorMessage) { - ParsingException pe = expectThrows(ParsingException.class, () -> parser().createStatement(query)); + ParsingException pe = expectThrows(ParsingException.class, () -> parser().createStatement(query, EsqlTestUtils.TEST_CFG)); assertThat(pe.getMessage(), containsString("mismatched input '" + errorMessage + "'")); // check the parser eliminated the DEV_ tokens from the message assertThat(pe.getMessage(), not(containsString("DEV_"))); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index dc7256e3c4521..a2f2bb0f7e176 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException; import org.elasticsearch.xpack.esql.core.expression.Alias; @@ -3845,7 +3846,7 @@ public void testRerankWithPositionalParameters() { List.of(paramAsConstant(null, "query text"), paramAsConstant(null, "reranker"), paramAsConstant(null, "rerank_score")) ); var rerank = as( - parser.createStatement("row a = 1 | RERANK ? ON title WITH inferenceId=?, scoreColumn=? ", queryParams), + parser.createStatement("row a = 1 | RERANK ? ON title WITH inferenceId=?, scoreColumn=? ", queryParams, EsqlTestUtils.TEST_CFG), Rerank.class ); @@ -3868,7 +3869,8 @@ public void testRerankWithNamedParameters() { var rerank = as( parser.createStatement( "row a = 1 | RERANK ?queryText ON title WITH inferenceId=?inferenceId, scoreColumn=?scoreColumnName", - queryParams + queryParams, + EsqlTestUtils.TEST_CFG ), Rerank.class ); @@ -3916,7 +3918,10 @@ public void testCompletionDefaultFieldName() { public void testCompletionWithPositionalParameters() { var queryParams = new QueryParams(List.of(paramAsConstant(null, "inferenceId"))); - var plan = as(parser.createStatement("row a = 1 | COMPLETION prompt_field WITH ?", queryParams), Completion.class); + var plan = as( + parser.createStatement("row a = 1 | COMPLETION prompt_field WITH ?", queryParams, EsqlTestUtils.TEST_CFG), + Completion.class + ); assertThat(plan.prompt(), equalTo(attribute("prompt_field"))); assertThat(plan.inferenceId(), equalTo(literalString("inferenceId"))); @@ -3925,7 +3930,10 @@ public void testCompletionWithPositionalParameters() { public void testCompletionWithNamedParameters() { var queryParams = new QueryParams(List.of(paramAsConstant("inferenceId", "myInference"))); - var plan = as(parser.createStatement("row a = 1 | COMPLETION prompt_field WITH ?inferenceId", queryParams), Completion.class); + var plan = as( + parser.createStatement("row a = 1 | COMPLETION prompt_field WITH ?inferenceId", queryParams, EsqlTestUtils.TEST_CFG), + Completion.class + ); assertThat(plan.prompt(), equalTo(attribute("prompt_field"))); assertThat(plan.inferenceId(), equalTo(literalString("myInference"))); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java index 69060f9cee0f2..cbf76f0cfca0e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.AutomatonQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.test.ESTestCase; @@ -30,8 +29,10 @@ import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; +import org.elasticsearch.xpack.esql.io.stream.ExpressionQueryBuilder; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamWrapperQueryBuilder; import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizer; @@ -39,6 +40,7 @@ import org.elasticsearch.xpack.esql.plan.physical.FragmentExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.mapper.Mapper; +import org.elasticsearch.xpack.esql.plugin.EsqlFlags; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; import org.elasticsearch.xpack.esql.session.Configuration; import org.junit.BeforeClass; @@ -50,6 +52,7 @@ import java.util.Set; import static java.util.Arrays.asList; +import static org.elasticsearch.TransportVersions.V_8_17_0; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; @@ -324,8 +327,8 @@ public void testLikeList() { |WHERE {} LIKE ("a+", "b+") """, LAST_NAME); var plan = plan(query, null); - - var filter = filterQueryForTransportNodes(TransportVersion.current(), plan); + // test with an older version, so like list is not supported + var filter = filterQueryForTransportNodes(V_8_17_0, plan); assertNull(filter); } @@ -342,12 +345,13 @@ public void testLikeListNullTransportVersion() { """, LAST_NAME); var plan = plan(query, null); - SingleValueQuery.Builder filter = (SingleValueQuery.Builder) filterQueryForTransportNodes(null, plan); + PlanStreamWrapperQueryBuilder filterWrapper = (PlanStreamWrapperQueryBuilder) filterQueryForTransportNodes(null, plan); + SingleValueQuery.Builder filter = (SingleValueQuery.Builder) filterWrapper.next(); assertEquals(LAST_NAME, filter.fieldName()); - AutomatonQueryBuilder innerFilter = (AutomatonQueryBuilder) filter.next(); + ExpressionQueryBuilder innerFilter = (ExpressionQueryBuilder) filter.next(); assertEquals(LAST_NAME, innerFilter.fieldName()); assertEquals(""" - LIKE("a+", "b+"), caseInsensitive=false""", innerFilter.description()); + last_name LIKE ("a+", "b+")""", innerFilter.getExpression().toString()); } /** @@ -378,7 +382,7 @@ public static QueryBuilder singleValueQuery(String query, QueryBuilder inner, St } private PhysicalPlan plan(String query, QueryBuilder restFilter) { - var logical = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement(query))); + var logical = logicalOptimizer.optimize(analyzer.analyze(parser.createStatement(query, EsqlTestUtils.TEST_CFG))); // System.out.println("Logical\n" + logical); var physical = mapper.map(logical); // System.out.println("physical\n" + physical); @@ -397,7 +401,13 @@ private QueryBuilder restFilterQuery(String field) { } private QueryBuilder filterQueryForTransportNodes(TransportVersion minTransportVersion, PhysicalPlan plan) { - return PlannerUtils.detectFilter(minTransportVersion, plan, Set.of(EMP_NO, LAST_NAME)::contains); + return PlannerUtils.detectFilter( + new EsqlFlags(true), + EsqlTestUtils.TEST_CFG, + minTransportVersion, + plan, + Set.of(EMP_NO, LAST_NAME)::contains + ); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java index ababc8ed37657..65587cf4d6876 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java @@ -201,7 +201,7 @@ static LogicalPlan parse(String query) { ), TEST_VERIFIER ); - return logicalOptimizer.optimize(analyzer.analyze(new EsqlParser().createStatement(query))); + return logicalOptimizer.optimize(analyzer.analyze(new EsqlParser().createStatement(query, EsqlTestUtils.TEST_CFG))); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java index 1fc481711df97..fe0e028db4c9c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java @@ -302,7 +302,7 @@ static LogicalPlan parse(String query) { ), TEST_VERIFIER ); - return logicalOptimizer.optimize(analyzer.analyze(new EsqlParser().createStatement(query))); + return logicalOptimizer.optimize(analyzer.analyze(new EsqlParser().createStatement(query, EsqlTestUtils.TEST_CFG))); } static PhysicalPlan mapAndMaybeOptimize(LogicalPlan logicalPlan) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 4264fdacc5d7b..c0bcfac8065ea 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.Build; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.parser.ParsingException; @@ -1600,7 +1601,7 @@ public void testEnrichOnDefaultField() { public void testMetrics() { var query = "TS k8s | STATS bytes=sum(rate(network.total_bytes_in)), sum(rate(network.total_cost)) BY cluster"; if (Build.current().isSnapshot() == false) { - var e = expectThrows(ParsingException.class, () -> parser.createStatement(query)); + var e = expectThrows(ParsingException.class, () -> parser.createStatement(query, EsqlTestUtils.TEST_CFG)); assertThat(e.getMessage(), containsString("line 1:1: mismatched input 'TS' expecting {")); return; } @@ -2076,7 +2077,8 @@ public void testForkWithStatsAndWhere() { private Set fieldNames(String query, Set enrichPolicyMatchFields) { var preAnalysisResult = new EsqlSession.PreAnalysisResult(null); - return EsqlSession.fieldNames(parser.createStatement(query), enrichPolicyMatchFields, preAnalysisResult).fieldNames(); + return EsqlSession.fieldNames(parser.createStatement(query, EsqlTestUtils.TEST_CFG), enrichPolicyMatchFields, preAnalysisResult) + .fieldNames(); } private void assertFieldNames(String query, Set expected) { @@ -2085,7 +2087,11 @@ private void assertFieldNames(String query, Set expected) { } private void assertFieldNames(String query, Set expected, Set wildCardIndices) { - var preAnalysisResult = EsqlSession.fieldNames(parser.createStatement(query), Set.of(), new EsqlSession.PreAnalysisResult(null)); + var preAnalysisResult = EsqlSession.fieldNames( + parser.createStatement(query, EsqlTestUtils.TEST_CFG), + Set.of(), + new EsqlSession.PreAnalysisResult(null) + ); assertThat("Query-wide field names", preAnalysisResult.fieldNames(), equalTo(expected)); assertThat("Lookup Indices that expect wildcard lookups", preAnalysisResult.wildcardJoinIndices(), equalTo(wildCardIndices)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java index 0fee82f4f6ee7..1f3047262c94d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.watcher.common.stats.Counters; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition; @@ -554,7 +555,7 @@ private Counters esql(String esql, Verifier v) { metrics = new Metrics(new EsqlFunctionRegistry()); verifier = new Verifier(metrics, new XPackLicenseState(() -> 0L)); } - analyzer(verifier).analyze(parser.createStatement(esql)); + analyzer(verifier).analyze(parser.createStatement(esql, EsqlTestUtils.TEST_CFG)); return metrics == null ? null : metrics.stats(); } diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 5a667df9ffbbf..2102aaaa72cb6 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -212,6 +212,11 @@ protected boolean matches(String pattern, boolean caseInsensitive, QueryRewriteC return Regex.simpleMatch(pattern, value, caseInsensitive); } + @Override + public String getConstantFieldValue(SearchExecutionContext context) { + return value; + } + @Override public Query existsQuery(SearchExecutionContext context) { return value != null ? new MatchAllDocsQuery() : new MatchNoDocsQuery();