Skip to content

Commit 81ef7dd

Browse files
jbaierajbaiera
authored
Support index pattern selector syntax in ES|QL (elastic#120660)
This PR updates the ES|QL grammar to include the selector portion of an index pattern. Patterns are recombined before being sent to field caps. Field caps already supports this functionality, so this is primarily wiring it up where needed.
1 parent 740d983 commit 81ef7dd

File tree

18 files changed

+2728
-2097
lines changed

18 files changed

+2728
-2097
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import java.util.function.Function;
6262
import java.util.function.LongSupplier;
6363
import java.util.function.Predicate;
64+
import java.util.function.Supplier;
6465

6566
/**
6667
* This class main focus is to resolve multi-syntax target expressions to resources or concrete indices. This resolution is influenced
@@ -2364,13 +2365,7 @@ private static <V> V splitSelectorExpression(String expression, BiFunction<Strin
23642365
int lastDoubleColon = expression.lastIndexOf(SELECTOR_SEPARATOR);
23652366
if (lastDoubleColon >= 0) {
23662367
String suffix = expression.substring(lastDoubleColon + SELECTOR_SEPARATOR.length());
2367-
IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix);
2368-
if (selector == null) {
2369-
throw new InvalidIndexNameException(
2370-
expression,
2371-
"invalid usage of :: separator, [" + suffix + "] is not a recognized selector"
2372-
);
2373-
}
2368+
doValidateSelectorString(() -> expression, suffix);
23742369
String expressionBase = expression.substring(0, lastDoubleColon);
23752370
ensureNoMoreSelectorSeparators(expressionBase, expression);
23762371
return bindFunction.apply(expressionBase, suffix);
@@ -2379,6 +2374,20 @@ private static <V> V splitSelectorExpression(String expression, BiFunction<Strin
23792374
return bindFunction.apply(expression, null);
23802375
}
23812376

2377+
public static void validateIndexSelectorString(String indexName, String suffix) {
2378+
doValidateSelectorString(() -> indexName + SELECTOR_SEPARATOR + suffix, suffix);
2379+
}
2380+
2381+
private static void doValidateSelectorString(Supplier<String> expression, String suffix) {
2382+
IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix);
2383+
if (selector == null) {
2384+
throw new InvalidIndexNameException(
2385+
expression.get(),
2386+
"invalid usage of :: separator, [" + suffix + "] is not a recognized selector"
2387+
);
2388+
}
2389+
}
2390+
23822391
/**
23832392
* Checks the selectors that have been returned from splitting an expression and throws an exception if any were present.
23842393
* @param expression Original expression

‎x-pack/plugin/esql/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ dependencies {
5959
testImplementation project(path: ':modules:analysis-common')
6060
testImplementation project(path: ':modules:ingest-common')
6161
testImplementation project(path: ':modules:legacy-geo')
62+
testImplementation project(path: ':modules:data-streams')
63+
testImplementation project(path: ':modules:mapper-extras')
6264
testImplementation project(xpackModule('esql:compute:test'))
6365
testImplementation('net.nextencia:rrdiagram:0.9.4')
6466
testImplementation('org.webjars.npm:fontsource__roboto-mono:4.5.7')

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,43 @@
88
package org.elasticsearch.xpack.esql.action;
99

1010
import org.elasticsearch.Build;
11+
import org.elasticsearch.action.DocWriteRequest;
1112
import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
1213
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
1314
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
15+
import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction;
16+
import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
1417
import org.elasticsearch.action.bulk.BulkRequestBuilder;
18+
import org.elasticsearch.action.bulk.BulkResponse;
19+
import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
1520
import org.elasticsearch.action.index.IndexRequest;
1621
import org.elasticsearch.action.index.IndexRequestBuilder;
1722
import org.elasticsearch.action.support.WriteRequest;
1823
import org.elasticsearch.client.internal.ClusterAdminClient;
24+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
25+
import org.elasticsearch.cluster.metadata.DataStream;
26+
import org.elasticsearch.cluster.metadata.DataStreamFailureStore;
27+
import org.elasticsearch.cluster.metadata.DataStreamOptions;
1928
import org.elasticsearch.cluster.metadata.IndexMetadata;
29+
import org.elasticsearch.cluster.metadata.ResettableValue;
30+
import org.elasticsearch.cluster.metadata.Template;
2031
import org.elasticsearch.cluster.node.DiscoveryNode;
2132
import org.elasticsearch.common.collect.Iterators;
33+
import org.elasticsearch.common.compress.CompressedXContent;
2234
import org.elasticsearch.common.settings.Setting;
2335
import org.elasticsearch.common.settings.Settings;
36+
import org.elasticsearch.core.TimeValue;
37+
import org.elasticsearch.datastreams.DataStreamsPlugin;
2438
import org.elasticsearch.index.Index;
2539
import org.elasticsearch.index.IndexService;
2640
import org.elasticsearch.index.IndexSettings;
41+
import org.elasticsearch.index.mapper.DateFieldMapper;
42+
import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
2743
import org.elasticsearch.index.query.QueryBuilder;
2844
import org.elasticsearch.index.query.RangeQueryBuilder;
2945
import org.elasticsearch.index.shard.IndexShard;
3046
import org.elasticsearch.indices.IndicesService;
47+
import org.elasticsearch.plugins.Plugin;
3148
import org.elasticsearch.test.ESTestCase;
3249
import org.elasticsearch.test.ListMatcher;
3350
import org.elasticsearch.xcontent.XContentBuilder;
@@ -38,11 +55,13 @@
3855
import org.elasticsearch.xpack.esql.parser.ParsingException;
3956
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
4057
import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
58+
import org.junit.Assume;
4159
import org.junit.Before;
4260

4361
import java.io.IOException;
4462
import java.util.ArrayList;
4563
import java.util.Arrays;
64+
import java.util.Collection;
4665
import java.util.Collections;
4766
import java.util.Comparator;
4867
import java.util.HashMap;
@@ -58,8 +77,10 @@
5877
import java.util.concurrent.TimeUnit;
5978
import java.util.concurrent.atomic.AtomicBoolean;
6079
import java.util.concurrent.atomic.AtomicLong;
80+
import java.util.function.BiConsumer;
6181
import java.util.stream.IntStream;
6282
import java.util.stream.LongStream;
83+
import java.util.stream.Stream;
6384

6485
import static java.util.Comparator.comparing;
6586
import static java.util.Comparator.naturalOrder;
@@ -100,6 +121,11 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
100121
.build();
101122
}
102123

124+
@Override
125+
protected Collection<Class<? extends Plugin>> nodePlugins() {
126+
return Stream.concat(super.nodePlugins().stream(), Stream.of(DataStreamsPlugin.class, MapperExtrasPlugin.class)).toList();
127+
}
128+
103129
public void testProjectConstant() {
104130
try (EsqlQueryResponse results = run("from test | eval x = 1 | keep x")) {
105131
assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("x", "integer"))));
@@ -992,6 +1018,176 @@ public void testIndexPatterns() throws Exception {
9921018
}
9931019
}
9941020

1021+
public void testDataStreamPatterns() throws Exception {
1022+
Assume.assumeTrue(DataStream.isFailureStoreFeatureFlagEnabled());
1023+
1024+
Map<String, Long> testCases = new HashMap<>();
1025+
// Concrete data stream with each selector
1026+
testCases.put("test_ds_patterns_1", 5L);
1027+
testCases.put("test_ds_patterns_1::data", 5L);
1028+
testCases.put("test_ds_patterns_1::failures", 3L);
1029+
testCases.put("test_ds_patterns_2", 5L);
1030+
testCases.put("test_ds_patterns_2::data", 5L);
1031+
testCases.put("test_ds_patterns_2::failures", 3L);
1032+
1033+
// Wildcard pattern with each selector
1034+
testCases.put("test_ds_patterns*", 15L);
1035+
testCases.put("test_ds_patterns*::data", 15L);
1036+
testCases.put("test_ds_patterns*::failures", 9L);
1037+
1038+
// Match all pattern with each selector
1039+
testCases.put("*", 15L);
1040+
testCases.put("*::data", 15L);
1041+
testCases.put("*::failures", 9L);
1042+
1043+
// Concrete multi-pattern
1044+
testCases.put("test_ds_patterns_1,test_ds_patterns_2", 10L);
1045+
testCases.put("test_ds_patterns_1::data,test_ds_patterns_2::data", 10L);
1046+
testCases.put("test_ds_patterns_1::failures,test_ds_patterns_2::failures", 6L);
1047+
1048+
// Wildcard multi-pattern
1049+
testCases.put("test_ds_patterns_1*,test_ds_patterns_2*", 10L);
1050+
testCases.put("test_ds_patterns_1*::data,test_ds_patterns_2*::data", 10L);
1051+
testCases.put("test_ds_patterns_1*::failures,test_ds_patterns_2*::failures", 6L);
1052+
1053+
// Wildcard pattern with data stream exclusions for each selector combination (data stream exclusions need * on the end to negate)
1054+
// None (default)
1055+
testCases.put("test_ds_patterns*,-test_ds_patterns_2*", 10L);
1056+
testCases.put("test_ds_patterns*,-test_ds_patterns_2*::data", 10L);
1057+
testCases.put("test_ds_patterns*,-test_ds_patterns_2*::failures", 15L);
1058+
// Subtracting from ::data
1059+
testCases.put("test_ds_patterns*::data,-test_ds_patterns_2*", 10L);
1060+
testCases.put("test_ds_patterns*::data,-test_ds_patterns_2*::data", 10L);
1061+
testCases.put("test_ds_patterns*::data,-test_ds_patterns_2*::failures", 15L);
1062+
// Subtracting from ::failures
1063+
testCases.put("test_ds_patterns*::failures,-test_ds_patterns_2*", 9L);
1064+
testCases.put("test_ds_patterns*::failures,-test_ds_patterns_2*::data", 9L);
1065+
testCases.put("test_ds_patterns*::failures,-test_ds_patterns_2*::failures", 6L);
1066+
// Subtracting from ::*
1067+
testCases.put("test_ds_patterns*::data,test_ds_patterns*::failures,-test_ds_patterns_2*", 19L);
1068+
testCases.put("test_ds_patterns*::data,test_ds_patterns*::failures,-test_ds_patterns_2*::data", 19L);
1069+
testCases.put("test_ds_patterns*::data,test_ds_patterns*::failures,-test_ds_patterns_2*::failures", 21L);
1070+
1071+
testCases.put("\"test_ds_patterns_1,test_ds_patterns_2\"::failures", 8L);
1072+
1073+
runDataStreamTest(testCases, new String[] { "test_ds_patterns_1", "test_ds_patterns_2", "test_ds_patterns_3" }, (key, value) -> {
1074+
try (var results = run("from " + key + " | stats count(@timestamp)")) {
1075+
assertEquals(key, 1, getValuesList(results).size());
1076+
assertEquals(key, value, getValuesList(results).get(0).get(0));
1077+
}
1078+
});
1079+
}
1080+
1081+
public void testDataStreamInvalidPatterns() throws Exception {
1082+
Assume.assumeTrue(DataStream.isFailureStoreFeatureFlagEnabled());
1083+
1084+
Map<String, String> testCases = new HashMap<>();
1085+
// === Errors
1086+
// Only recognized components can be selected
1087+
testCases.put("testXXX::custom", "invalid usage of :: separator, [custom] is not a recognized selector");
1088+
// Spelling is important
1089+
testCases.put("testXXX::failres", "invalid usage of :: separator, [failres] is not a recognized selector");
1090+
// Only the match all wildcard is supported
1091+
testCases.put("testXXX::d*ta", "invalid usage of :: separator, [d*ta] is not a recognized selector");
1092+
// The first instance of :: is split upon so that you cannot chain the selector
1093+
testCases.put("test::XXX::data", "mismatched input '::' expecting {<EOF>, '|', ',', 'metadata'}");
1094+
// Selectors must be outside of date math expressions or else they trip up the selector parsing
1095+
testCases.put("<test-{now/d}::failures>", "Invalid index name [<test-{now/d}], must not contain the following characters [");
1096+
// Only one selector separator is allowed per expression
1097+
testCases.put("::::data", "mismatched input '::' expecting {QUOTED_STRING, UNQUOTED_SOURCE}");
1098+
// Suffix case is not supported because there is no component named with the empty string
1099+
testCases.put("index::", "missing {QUOTED_STRING, UNQUOTED_SOURCE} at '|'");
1100+
1101+
runDataStreamTest(testCases, new String[] { "test_ds_patterns_1" }, (key, value) -> {
1102+
logger.info(key);
1103+
var exception = expectThrows(ParsingException.class, () -> { run("from " + key + " | stats count(@timestamp)").close(); });
1104+
assertThat(exception.getMessage(), containsString(value));
1105+
});
1106+
}
1107+
1108+
private <V> void runDataStreamTest(Map<String, V> testCases, String[] dsNames, BiConsumer<String, V> testMethod) throws IOException {
1109+
boolean deleteTemplate = false;
1110+
List<String> deleteDataStreams = new ArrayList<>();
1111+
try {
1112+
assertAcked(
1113+
client().execute(
1114+
TransportPutComposableIndexTemplateAction.TYPE,
1115+
new TransportPutComposableIndexTemplateAction.Request("test_ds_template").indexTemplate(
1116+
ComposableIndexTemplate.builder()
1117+
.indexPatterns(List.of("test_ds_patterns_*"))
1118+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
1119+
.template(
1120+
Template.builder()
1121+
.mappings(new CompressedXContent("""
1122+
{
1123+
"dynamic": false,
1124+
"properties": {
1125+
"@timestamp": {
1126+
"type": "date"
1127+
},
1128+
"count": {
1129+
"type": "long"
1130+
}
1131+
}
1132+
}"""))
1133+
.dataStreamOptions(
1134+
ResettableValue.create(
1135+
new DataStreamOptions.Template(
1136+
ResettableValue.create(new DataStreamFailureStore.Template(ResettableValue.create(true)))
1137+
)
1138+
)
1139+
)
1140+
.build()
1141+
)
1142+
.build()
1143+
)
1144+
)
1145+
);
1146+
deleteTemplate = true;
1147+
1148+
String time = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis());
1149+
int i = 0;
1150+
for (String dsName : dsNames) {
1151+
BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
1152+
for (String id : Arrays.asList("1", "2", "3", "4", "5")) {
1153+
bulk.add(createDoc(dsName, id, time, ++i * 1000));
1154+
}
1155+
for (String id : Arrays.asList("6", "7", "8")) {
1156+
bulk.add(createDoc(dsName, id, time, "garbage"));
1157+
}
1158+
BulkResponse bulkItemResponses = bulk.get();
1159+
assertThat(bulkItemResponses.hasFailures(), is(false));
1160+
deleteDataStreams.add(dsName);
1161+
ensureYellow(dsName);
1162+
}
1163+
1164+
for (Map.Entry<String, V> testCase : testCases.entrySet()) {
1165+
testMethod.accept(testCase.getKey(), testCase.getValue());
1166+
}
1167+
} finally {
1168+
if (deleteDataStreams.isEmpty() == false) {
1169+
assertAcked(
1170+
client().execute(
1171+
DeleteDataStreamAction.INSTANCE,
1172+
new DeleteDataStreamAction.Request(new TimeValue(30, TimeUnit.SECONDS), deleteDataStreams.toArray(String[]::new))
1173+
)
1174+
);
1175+
}
1176+
if (deleteTemplate) {
1177+
assertAcked(
1178+
client().execute(
1179+
TransportDeleteComposableIndexTemplateAction.TYPE,
1180+
new TransportDeleteComposableIndexTemplateAction.Request("test_ds_template")
1181+
)
1182+
);
1183+
}
1184+
}
1185+
}
1186+
1187+
private static IndexRequest createDoc(String dsName, String id, String ts, Object count) {
1188+
return new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE).id(id).source("@timestamp", ts, "count", count);
1189+
}
1190+
9951191
public void testOverlappingIndexPatterns() throws Exception {
9961192
String[] indexNames = { "test_overlapping_index_patterns_1", "test_overlapping_index_patterns_2" };
9971193

‎x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,19 @@ indexPatternAndMetadataFields:
9898

9999
indexPattern
100100
: (clusterString COLON)? indexString
101+
| {this.isDevVersion()}? indexString (CAST_OP selectorString)?
101102
;
102103

103104
clusterString
104105
: UNQUOTED_SOURCE
105106
| QUOTED_STRING
106107
;
107108

109+
selectorString
110+
: UNQUOTED_SOURCE
111+
| QUOTED_STRING
112+
;
113+
108114
indexString
109115
: UNQUOTED_SOURCE
110116
| QUOTED_STRING

‎x-pack/plugin/esql/src/main/antlr/lexer/From.g4‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ FROM_PIPE : PIPE -> type(PIPE), popMode;
1818
FROM_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET);
1919
FROM_CLOSING_BRACKET : CLOSING_BRACKET -> type(CLOSING_BRACKET);
2020
FROM_COLON : COLON -> type(COLON);
21+
FROM_SELECTOR : {this.isDevVersion()}? CAST_OP -> type(CAST_OP);
2122
FROM_COMMA : COMMA -> type(COMMA);
2223
FROM_ASSIGN : ASSIGN -> type(ASSIGN);
2324
METADATA : 'metadata';

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
package org.elasticsearch.xpack.esql.action;
99

1010
import org.elasticsearch.Build;
11+
import org.elasticsearch.cluster.metadata.DataStream;
1112
import org.elasticsearch.common.util.FeatureFlag;
1213
import org.elasticsearch.features.NodeFeature;
1314
import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction;
@@ -912,7 +913,12 @@ public enum Cap {
912913
/**
913914
* The metrics command
914915
*/
915-
METRICS_COMMAND(Build.current().isSnapshot());
916+
METRICS_COMMAND(Build.current().isSnapshot()),
917+
918+
/**
919+
* Index component selector syntax (my-data-stream-name::failures)
920+
*/
921+
INDEX_COMPONENT_SELECTORS(DataStream.isFailureStoreFeatureFlagEnabled());
916922

917923
private final boolean enabled;
918924

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp‎

Lines changed: 2 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)