From 5072e1432832ddd1ef76170103bea9c0f9ac2d69 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 18 Jun 2025 09:30:03 -0400 Subject: [PATCH 1/3] ESQL: Fix wildcard for `_index` field This is a fairly hacky fix for #129511, a bug where we don't run `LIKE` against the `_index` field in a consistent way. I feel like there should be a better way to fix this, but should work. --- .../index/mapper/ConstantFieldType.java | 7 +++ .../index/mapper/IndexFieldMapper.java | 2 +- .../index/query/SearchIndexNameMatcher.java | 4 ++ .../core/querydsl/query/WildcardQuery.java | 44 +++++++++++++- .../xpack/esql/ccq/MultiClustersIT.java | 58 +++++++++++++++++++ .../scalar/string/regex/RegexMatch.java | 2 + .../scalar/string/regex/WildcardLike.java | 9 +++ 7 files changed, 122 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java index 5ecb75b09408c..8042764285df9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java @@ -20,6 +20,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.logging.LogManager; import java.util.Collection; import java.util.Map; @@ -128,6 +129,12 @@ public final Query wildcardQuery( } public final Query wildcardQuery(String value, boolean caseInsensitive, QueryRewriteContext context) { + LogManager.getLogger(ConstantFieldType.class) + .error( + "ADSFA const eval {} {}", + value, + matches(value, caseInsensitive, context) + ); if (matches(value, caseInsensitive, context)) { return Queries.newMatchAllQuery(); } else { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 33c6ff15cccfd..bb6613ffa631f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -38,7 +38,7 @@ public class IndexFieldMapper extends MetadataFieldMapper { public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); - static final class IndexFieldType extends ConstantFieldType { + public static final class IndexFieldType extends ConstantFieldType { static final IndexFieldType INSTANCE = new IndexFieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchIndexNameMatcher.java b/server/src/main/java/org/elasticsearch/index/query/SearchIndexNameMatcher.java index 6799895d8e278..7b0e83e7d805f 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchIndexNameMatcher.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchIndexNameMatcher.java @@ -13,6 +13,8 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.index.mapper.ConstantFieldType; +import org.elasticsearch.logging.LogManager; import org.elasticsearch.transport.RemoteClusterAware; import java.util.function.Predicate; @@ -53,7 +55,9 @@ public SearchIndexNameMatcher( * the separator ':', and must match on both the cluster alias and index name. */ public boolean test(String pattern) { + String[] splitIndex = RemoteClusterAware.splitIndexName(pattern); + LogManager.getLogger(ConstantFieldType.class).error("ADSFA {}", (Object) splitIndex); if (splitIndex[0] == null) { return clusterAlias == null && matchesIndex(pattern); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java index 03d819cf7aa9b..3815a994d1af1 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java @@ -6,10 +6,23 @@ */ package org.elasticsearch.xpack.esql.core.querydsl.query; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.index.mapper.IndexFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.index.query.support.QueryParsers; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.esql.core.tree.Source; +import java.io.IOException; +import java.util.Locale; import java.util.Objects; import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; @@ -44,9 +57,34 @@ public Boolean caseInsensitive() { @Override protected QueryBuilder asBuilder() { - WildcardQueryBuilder wb = wildcardQuery(field, query); - // ES does not allow case_insensitive to be set to "false", it should be either "true" or not specified - return caseInsensitive == false ? wb : wb.caseInsensitive(caseInsensitive); + /* + * Builds WildcardQueryBuilder with simple text matching semantics for + * all fields, including the `_index` field which insists on implementing + * some fairly unexpected matching rules. + * + * Note that + */ + return new WildcardQueryBuilder(field, query) { + @Override + protected org.apache.lucene.search.Query doToQuery(SearchExecutionContext context) throws IOException { + MappedFieldType fieldType = context.getFieldType(fieldName()); + MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite(), null, LoggingDeprecationHandler.INSTANCE); + LogManager.getLogger(WildcardQuery.class).error("ADSFA special query {}", fieldType); + if (fieldType instanceof IndexFieldMapper.IndexFieldType) { + String value = value(); + String indexName = context.getFullyQualifiedIndex().getName(); + if (WildcardQuery.this.caseInsensitive) { + value = value.toLowerCase(Locale.ROOT); + indexName = indexName.toLowerCase(Locale.ROOT); + } + if (Regex.simpleMatch(value, indexName)) { + return new MatchAllDocsQuery(); + } + return new MatchNoDocsQuery(); + } + return fieldType.wildcardQuery(value(), method, WildcardQuery.this.caseInsensitive, context); + } + }; } @Override diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java index 4ea413e4fcd3b..dac503fd5e07f 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java @@ -371,6 +371,64 @@ public void testStats() throws IOException { assertThat(clusterData, hasKey("took")); } + public void testLikeIndex() throws Exception { + { + boolean includeCCSMetadata = includeCCSMetadata(); + Map result = run(""" + FROM test-local-index,*:test-remote-index METADATA _index + | WHERE _index LIKE "*remote*" + | STATS c = COUNT(*) BY _index + | SORT _index ASC + """, includeCCSMetadata); + var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword")); + var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex)); + + assertResultMap(includeCCSMetadata, result, columns, values, false); + } + { + boolean includeCCSMetadata = includeCCSMetadata(); + Map result = run(""" + FROM test-local-index,*:test-remote-index METADATA _index + | WHERE _index NOT LIKE "*remote*" + | STATS c = COUNT(*) BY _index + | SORT _index ASC + """, includeCCSMetadata); + var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword")); + var values = List.of(List.of(localDocs.size(), localIndex)); + + assertResultMap(includeCCSMetadata, result, columns, values, false); + } + } + + public void testRLikeIndex() throws Exception { + { + boolean includeCCSMetadata = includeCCSMetadata(); + Map result = run(""" + FROM test-local-index,*:test-remote-index METADATA _index + | WHERE _index RLIKE ".*remote.*" + | STATS c = COUNT(*) BY _index + | SORT _index ASC + """, includeCCSMetadata); + var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword")); + var values = List.of(List.of(remoteDocs.size(), REMOTE_CLUSTER_NAME + ":" + remoteIndex)); + + assertResultMap(includeCCSMetadata, result, columns, values, false); + } + { + boolean includeCCSMetadata = includeCCSMetadata(); + Map result = run(""" + FROM test-local-index,*:test-remote-index METADATA _index + | WHERE _index NOT RLIKE ".*remote.*" + | STATS c = COUNT(*) BY _index + | SORT _index ASC + """, includeCCSMetadata); + var columns = List.of(Map.of("name", "c", "type", "long"), Map.of("name", "_index", "type", "keyword")); + var values = List.of(List.of(localDocs.size(), localIndex)); + + assertResultMap(includeCCSMetadata, result, columns, values, false); + } + } + private RestClient remoteClusterClient() throws IOException { var clusterHosts = parseClusterHosts(remoteCluster.getHttpAddresses()); return buildClient(restClientSettings(), clusterHosts.toArray(new HttpHost[0])); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RegexMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RegexMatch.java index 140bbd697ce9e..ada7f5b843c83 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RegexMatch.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RegexMatch.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.logging.LogManager; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.capabilities.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -48,6 +49,7 @@ public Boolean fold(FoldContext ctx) { @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + LogManager.getLogger(WildcardLike.class).error("ADSFA toEvaluator"); return AutomataMatch.toEvaluator( source(), toEvaluator.apply(field()), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java index d7d1973fceda1..1c7e14172a059 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.logging.LogManager; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; @@ -108,6 +109,7 @@ protected WildcardLike replaceChild(Expression newLeft) { @Override public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { + LogManager.getLogger(WildcardLike.class).error("ADSFA translatable", new Exception()); return pushdownPredicates.isPushableAttribute(field()) ? Translatable.YES : Translatable.NO; } @@ -115,6 +117,13 @@ public Translatable translatable(LucenePushdownPredicates pushdownPredicates) { public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { var field = field(); LucenePushdownPredicates.checkIsPushableAttribute(field); + LogManager.getLogger(WildcardLike.class) + .error( + "ADSFA asQuery {} {}", + field, + translateField(handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field)), + new Exception() + ); return translateField(handler.nameOf(field instanceof FieldAttribute fa ? fa.exactAttribute() : field)); } From 49f0786c0845db183c05afcc6ab05c1108927af1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 18 Jun 2025 12:35:38 -0400 Subject: [PATCH 2/3] Update docs/changelog/129650.yaml --- docs/changelog/129650.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/129650.yaml diff --git a/docs/changelog/129650.yaml b/docs/changelog/129650.yaml new file mode 100644 index 0000000000000..ca514119bb067 --- /dev/null +++ b/docs/changelog/129650.yaml @@ -0,0 +1,5 @@ +pr: 129650 +summary: Fix wildcard for `_index` field +area: ES|QL +type: bug +issues: [] From 71e587c3ae547552aae58ceca616fd2fa782caad Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 18 Jun 2025 16:43:36 +0000 Subject: [PATCH 3/3] [CI] Auto commit changes from spotless --- .../org/elasticsearch/index/mapper/ConstantFieldType.java | 7 +------ .../xpack/esql/core/querydsl/query/WildcardQuery.java | 3 --- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java index 8042764285df9..78971859fdb60 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ConstantFieldType.java @@ -129,12 +129,7 @@ public final Query wildcardQuery( } public final Query wildcardQuery(String value, boolean caseInsensitive, QueryRewriteContext context) { - LogManager.getLogger(ConstantFieldType.class) - .error( - "ADSFA const eval {} {}", - value, - matches(value, caseInsensitive, context) - ); + LogManager.getLogger(ConstantFieldType.class).error("ADSFA const eval {} {}", value, matches(value, caseInsensitive, context)); if (matches(value, caseInsensitive, context)) { return Queries.newMatchAllQuery(); } else { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java index 3815a994d1af1..0381b0da3e532 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/WildcardQuery.java @@ -18,15 +18,12 @@ import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.index.query.support.QueryParsers; import org.elasticsearch.logging.LogManager; -import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.esql.core.tree.Source; import java.io.IOException; import java.util.Locale; import java.util.Objects; -import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; - public class WildcardQuery extends Query { private final String field, query;