Skip to content

Commit 3fdc90c

Browse files
jbaierajbaiera
authored
[8.x] Support index pattern selector syntax in ES|QL (elastic#120660) (elastic#125557)
* Support index pattern selector syntax in ES|QL (elastic#120660) This PR updates the ES|QL grammar to include the selector portion of an index pattern. Patterns are recombined before being sent to field caps. Field caps already supports this functionality, so this is primarily wiring it up where needed. * Add validation step to quoted join query construction (elastic#125731) * Add validation step to quoted join query construction * Fix double validation * Add lexer entry for METRICS commands * Simplifying test case * regen
1 parent aaf0b97 commit 3fdc90c

File tree

19 files changed

+2573
-1899
lines changed

19 files changed

+2573
-1899
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
import java.util.function.Function;
6161
import java.util.function.LongSupplier;
6262
import java.util.function.Predicate;
63+
import java.util.function.Supplier;
6364

6465
/**
6566
* This class main focus is to resolve multi-syntax target expressions to resources or concrete indices. This resolution is influenced
@@ -2126,13 +2127,7 @@ private static <V> V splitSelectorExpression(String expression, BiFunction<Strin
21262127
int lastDoubleColon = expression.lastIndexOf(SELECTOR_SEPARATOR);
21272128
if (lastDoubleColon >= 0) {
21282129
String suffix = expression.substring(lastDoubleColon + SELECTOR_SEPARATOR.length());
2129-
IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix);
2130-
if (selector == null) {
2131-
throw new InvalidIndexNameException(
2132-
expression,
2133-
"invalid usage of :: separator, [" + suffix + "] is not a recognized selector"
2134-
);
2135-
}
2130+
doValidateSelectorString(() -> expression, suffix);
21362131
String expressionBase = expression.substring(0, lastDoubleColon);
21372132
ensureNoMoreSelectorSeparators(expressionBase, expression);
21382133
return bindFunction.apply(expressionBase, suffix);
@@ -2141,6 +2136,20 @@ private static <V> V splitSelectorExpression(String expression, BiFunction<Strin
21412136
return bindFunction.apply(expression, null);
21422137
}
21432138

2139+
public static void validateIndexSelectorString(String indexName, String suffix) {
2140+
doValidateSelectorString(() -> indexName + SELECTOR_SEPARATOR + suffix, suffix);
2141+
}
2142+
2143+
private static void doValidateSelectorString(Supplier<String> expression, String suffix) {
2144+
IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix);
2145+
if (selector == null) {
2146+
throw new InvalidIndexNameException(
2147+
expression.get(),
2148+
"invalid usage of :: separator, [" + suffix + "] is not a recognized selector"
2149+
);
2150+
}
2151+
}
2152+
21442153
/**
21452154
* Checks the selectors that have been returned from splitting an expression and throws an exception if any were present.
21462155
* @param expression Original expression

‎x-pack/plugin/esql/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ dependencies {
5252
testImplementation project(path: ':modules:analysis-common')
5353
testImplementation project(path: ':modules:ingest-common')
5454
testImplementation project(path: ':modules:legacy-geo')
55+
testImplementation project(path: ':modules:data-streams')
56+
testImplementation project(path: ':modules:mapper-extras')
5557
testImplementation project(xpackModule('esql:compute:test'))
5658
testImplementation('net.nextencia:rrdiagram:0.9.4')
5759
testImplementation('org.webjars.npm:fontsource__roboto-mono:4.5.7')

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,43 @@
88
package org.elasticsearch.xpack.esql.action;
99

1010
import org.elasticsearch.Build;
11+
import org.elasticsearch.action.DocWriteRequest;
1112
import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
1213
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
1314
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
15+
import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction;
16+
import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
1417
import org.elasticsearch.action.bulk.BulkRequestBuilder;
18+
import org.elasticsearch.action.bulk.BulkResponse;
19+
import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
1520
import org.elasticsearch.action.index.IndexRequest;
1621
import org.elasticsearch.action.index.IndexRequestBuilder;
1722
import org.elasticsearch.action.support.WriteRequest;
1823
import org.elasticsearch.client.internal.ClusterAdminClient;
24+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
25+
import org.elasticsearch.cluster.metadata.DataStream;
26+
import org.elasticsearch.cluster.metadata.DataStreamFailureStore;
27+
import org.elasticsearch.cluster.metadata.DataStreamOptions;
1928
import org.elasticsearch.cluster.metadata.IndexMetadata;
29+
import org.elasticsearch.cluster.metadata.ResettableValue;
30+
import org.elasticsearch.cluster.metadata.Template;
2031
import org.elasticsearch.cluster.node.DiscoveryNode;
2132
import org.elasticsearch.common.collect.Iterators;
33+
import org.elasticsearch.common.compress.CompressedXContent;
2234
import org.elasticsearch.common.settings.Setting;
2335
import org.elasticsearch.common.settings.Settings;
36+
import org.elasticsearch.core.TimeValue;
37+
import org.elasticsearch.datastreams.DataStreamsPlugin;
2438
import org.elasticsearch.index.Index;
2539
import org.elasticsearch.index.IndexService;
2640
import org.elasticsearch.index.IndexSettings;
41+
import org.elasticsearch.index.mapper.DateFieldMapper;
42+
import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
2743
import org.elasticsearch.index.query.QueryBuilder;
2844
import org.elasticsearch.index.query.RangeQueryBuilder;
2945
import org.elasticsearch.index.shard.IndexShard;
3046
import org.elasticsearch.indices.IndicesService;
47+
import org.elasticsearch.plugins.Plugin;
3148
import org.elasticsearch.test.ESTestCase;
3249
import org.elasticsearch.test.ListMatcher;
3350
import org.elasticsearch.xcontent.XContentBuilder;
@@ -38,11 +55,13 @@
3855
import org.elasticsearch.xpack.esql.parser.ParsingException;
3956
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
4057
import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
58+
import org.junit.Assume;
4159
import org.junit.Before;
4260

4361
import java.io.IOException;
4462
import java.util.ArrayList;
4563
import java.util.Arrays;
64+
import java.util.Collection;
4665
import java.util.Collections;
4766
import java.util.Comparator;
4867
import java.util.HashMap;
@@ -58,8 +77,10 @@
5877
import java.util.concurrent.TimeUnit;
5978
import java.util.concurrent.atomic.AtomicBoolean;
6079
import java.util.concurrent.atomic.AtomicLong;
80+
import java.util.function.BiConsumer;
6181
import java.util.stream.IntStream;
6282
import java.util.stream.LongStream;
83+
import java.util.stream.Stream;
6384

6485
import static java.util.Comparator.comparing;
6586
import static java.util.Comparator.naturalOrder;
@@ -100,6 +121,11 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
100121
.build();
101122
}
102123

124+
@Override
125+
protected Collection<Class<? extends Plugin>> nodePlugins() {
126+
return Stream.concat(super.nodePlugins().stream(), Stream.of(DataStreamsPlugin.class, MapperExtrasPlugin.class)).toList();
127+
}
128+
103129
public void testProjectConstant() {
104130
try (EsqlQueryResponse results = run("from test | eval x = 1 | keep x")) {
105131
assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("x", "integer"))));
@@ -970,6 +996,176 @@ public void testIndexPatterns() throws Exception {
970996
}
971997
}
972998

999+
public void testDataStreamPatterns() throws Exception {
1000+
Assume.assumeTrue(DataStream.isFailureStoreFeatureFlagEnabled());
1001+
1002+
Map<String, Long> testCases = new HashMap<>();
1003+
// Concrete data stream with each selector
1004+
testCases.put("test_ds_patterns_1", 5L);
1005+
testCases.put("test_ds_patterns_1::data", 5L);
1006+
testCases.put("test_ds_patterns_1::failures", 3L);
1007+
testCases.put("test_ds_patterns_2", 5L);
1008+
testCases.put("test_ds_patterns_2::data", 5L);
1009+
testCases.put("test_ds_patterns_2::failures", 3L);
1010+
1011+
// Wildcard pattern with each selector
1012+
testCases.put("test_ds_patterns*", 15L);
1013+
testCases.put("test_ds_patterns*::data", 15L);
1014+
testCases.put("test_ds_patterns*::failures", 9L);
1015+
1016+
// Match all pattern with each selector
1017+
testCases.put("*", 15L);
1018+
testCases.put("*::data", 15L);
1019+
testCases.put("*::failures", 9L);
1020+
1021+
// Concrete multi-pattern
1022+
testCases.put("test_ds_patterns_1,test_ds_patterns_2", 10L);
1023+
testCases.put("test_ds_patterns_1::data,test_ds_patterns_2::data", 10L);
1024+
testCases.put("test_ds_patterns_1::failures,test_ds_patterns_2::failures", 6L);
1025+
1026+
// Wildcard multi-pattern
1027+
testCases.put("test_ds_patterns_1*,test_ds_patterns_2*", 10L);
1028+
testCases.put("test_ds_patterns_1*::data,test_ds_patterns_2*::data", 10L);
1029+
testCases.put("test_ds_patterns_1*::failures,test_ds_patterns_2*::failures", 6L);
1030+
1031+
// Wildcard pattern with data stream exclusions for each selector combination (data stream exclusions need * on the end to negate)
1032+
// None (default)
1033+
testCases.put("test_ds_patterns*,-test_ds_patterns_2*", 10L);
1034+
testCases.put("test_ds_patterns*,-test_ds_patterns_2*::data", 10L);
1035+
testCases.put("test_ds_patterns*,-test_ds_patterns_2*::failures", 15L);
1036+
// Subtracting from ::data
1037+
testCases.put("test_ds_patterns*::data,-test_ds_patterns_2*", 10L);
1038+
testCases.put("test_ds_patterns*::data,-test_ds_patterns_2*::data", 10L);
1039+
testCases.put("test_ds_patterns*::data,-test_ds_patterns_2*::failures", 15L);
1040+
// Subtracting from ::failures
1041+
testCases.put("test_ds_patterns*::failures,-test_ds_patterns_2*", 9L);
1042+
testCases.put("test_ds_patterns*::failures,-test_ds_patterns_2*::data", 9L);
1043+
testCases.put("test_ds_patterns*::failures,-test_ds_patterns_2*::failures", 6L);
1044+
// Subtracting from ::*
1045+
testCases.put("test_ds_patterns*::data,test_ds_patterns*::failures,-test_ds_patterns_2*", 19L);
1046+
testCases.put("test_ds_patterns*::data,test_ds_patterns*::failures,-test_ds_patterns_2*::data", 19L);
1047+
testCases.put("test_ds_patterns*::data,test_ds_patterns*::failures,-test_ds_patterns_2*::failures", 21L);
1048+
1049+
testCases.put("\"test_ds_patterns_1,test_ds_patterns_2\"::failures", 8L);
1050+
1051+
runDataStreamTest(testCases, new String[] { "test_ds_patterns_1", "test_ds_patterns_2", "test_ds_patterns_3" }, (key, value) -> {
1052+
try (var results = run("from " + key + " | stats count(@timestamp)")) {
1053+
assertEquals(key, 1, getValuesList(results).size());
1054+
assertEquals(key, value, getValuesList(results).get(0).get(0));
1055+
}
1056+
});
1057+
}
1058+
1059+
public void testDataStreamInvalidPatterns() throws Exception {
1060+
Assume.assumeTrue(DataStream.isFailureStoreFeatureFlagEnabled());
1061+
1062+
Map<String, String> testCases = new HashMap<>();
1063+
// === Errors
1064+
// Only recognized components can be selected
1065+
testCases.put("testXXX::custom", "invalid usage of :: separator, [custom] is not a recognized selector");
1066+
// Spelling is important
1067+
testCases.put("testXXX::failres", "invalid usage of :: separator, [failres] is not a recognized selector");
1068+
// Only the match all wildcard is supported
1069+
testCases.put("testXXX::d*ta", "invalid usage of :: separator, [d*ta] is not a recognized selector");
1070+
// The first instance of :: is split upon so that you cannot chain the selector
1071+
testCases.put("test::XXX::data", "mismatched input '::' expecting {");
1072+
// Selectors must be outside of date math expressions or else they trip up the selector parsing
1073+
testCases.put("<test-{now/d}::failures>", "Invalid index name [<test-{now/d}], must not contain the following characters [");
1074+
// Only one selector separator is allowed per expression
1075+
testCases.put("::::data", "mismatched input '::' expecting {");
1076+
// Suffix case is not supported because there is no component named with the empty string
1077+
testCases.put("index::", "missing {QUOTED_STRING, UNQUOTED_SOURCE} at '|'");
1078+
1079+
runDataStreamTest(testCases, new String[] { "test_ds_patterns_1" }, (key, value) -> {
1080+
logger.info(key);
1081+
var exception = expectThrows(ParsingException.class, () -> { run("from " + key + " | stats count(@timestamp)").close(); });
1082+
assertThat(exception.getMessage(), containsString(value));
1083+
});
1084+
}
1085+
1086+
private <V> void runDataStreamTest(Map<String, V> testCases, String[] dsNames, BiConsumer<String, V> testMethod) throws IOException {
1087+
boolean deleteTemplate = false;
1088+
List<String> deleteDataStreams = new ArrayList<>();
1089+
try {
1090+
assertAcked(
1091+
client().execute(
1092+
TransportPutComposableIndexTemplateAction.TYPE,
1093+
new TransportPutComposableIndexTemplateAction.Request("test_ds_template").indexTemplate(
1094+
ComposableIndexTemplate.builder()
1095+
.indexPatterns(List.of("test_ds_patterns_*"))
1096+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
1097+
.template(
1098+
Template.builder()
1099+
.mappings(new CompressedXContent("""
1100+
{
1101+
"dynamic": false,
1102+
"properties": {
1103+
"@timestamp": {
1104+
"type": "date"
1105+
},
1106+
"count": {
1107+
"type": "long"
1108+
}
1109+
}
1110+
}"""))
1111+
.dataStreamOptions(
1112+
ResettableValue.create(
1113+
new DataStreamOptions.Template(
1114+
ResettableValue.create(new DataStreamFailureStore.Template(ResettableValue.create(true)))
1115+
)
1116+
)
1117+
)
1118+
.build()
1119+
)
1120+
.build()
1121+
)
1122+
)
1123+
);
1124+
deleteTemplate = true;
1125+
1126+
String time = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis());
1127+
int i = 0;
1128+
for (String dsName : dsNames) {
1129+
BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
1130+
for (String id : Arrays.asList("1", "2", "3", "4", "5")) {
1131+
bulk.add(createDoc(dsName, id, time, ++i * 1000));
1132+
}
1133+
for (String id : Arrays.asList("6", "7", "8")) {
1134+
bulk.add(createDoc(dsName, id, time, "garbage"));
1135+
}
1136+
BulkResponse bulkItemResponses = bulk.get();
1137+
assertThat(bulkItemResponses.hasFailures(), is(false));
1138+
deleteDataStreams.add(dsName);
1139+
ensureYellow(dsName);
1140+
}
1141+
1142+
for (Map.Entry<String, V> testCase : testCases.entrySet()) {
1143+
testMethod.accept(testCase.getKey(), testCase.getValue());
1144+
}
1145+
} finally {
1146+
if (deleteDataStreams.isEmpty() == false) {
1147+
assertAcked(
1148+
client().execute(
1149+
DeleteDataStreamAction.INSTANCE,
1150+
new DeleteDataStreamAction.Request(new TimeValue(30, TimeUnit.SECONDS), deleteDataStreams.toArray(String[]::new))
1151+
)
1152+
);
1153+
}
1154+
if (deleteTemplate) {
1155+
assertAcked(
1156+
client().execute(
1157+
TransportDeleteComposableIndexTemplateAction.TYPE,
1158+
new TransportDeleteComposableIndexTemplateAction.Request("test_ds_template")
1159+
)
1160+
);
1161+
}
1162+
}
1163+
}
1164+
1165+
private static IndexRequest createDoc(String dsName, String id, String ts, Object count) {
1166+
return new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE).id(id).source("@timestamp", ts, "count", count);
1167+
}
1168+
9731169
public void testOverlappingIndexPatterns() throws Exception {
9741170
String[] indexNames = { "test_overlapping_index_patterns_1", "test_overlapping_index_patterns_2" };
9751171

‎x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,7 @@ FROM_PIPE : PIPE -> type(PIPE), popMode;
285285
FROM_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET);
286286
FROM_CLOSING_BRACKET : CLOSING_BRACKET -> type(CLOSING_BRACKET);
287287
FROM_COLON : COLON -> type(COLON);
288+
FROM_SELECTOR : {this.isDevVersion()}? CAST_OP -> type(CAST_OP);
288289
FROM_COMMA : COMMA -> type(COMMA);
289290
FROM_ASSIGN : ASSIGN -> type(ASSIGN);
290291
METADATA : 'metadata';
@@ -622,6 +623,10 @@ CLOSING_METRICS_COLON
622623
: COLON -> type(COLON), popMode, pushMode(METRICS_MODE)
623624
;
624625

626+
CLOSING_METRICS_SELECTOR
627+
: CAST_OP -> type(CAST_OP), popMode, pushMode(METRICS_MODE)
628+
;
629+
625630
CLOSING_METRICS_COMMA
626631
: COMMA -> type(COMMA), popMode, pushMode(METRICS_MODE)
627632
;

‎x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,19 @@ fromCommand
139139

140140
indexPattern
141141
: (clusterString COLON)? indexString
142+
| {this.isDevVersion()}? indexString (CAST_OP selectorString)?
142143
;
143144

144145
clusterString
145146
: UNQUOTED_SOURCE
146147
| QUOTED_STRING
147148
;
148149

150+
selectorString
151+
: UNQUOTED_SOURCE
152+
| QUOTED_STRING
153+
;
154+
149155
indexString
150156
: UNQUOTED_SOURCE
151157
| QUOTED_STRING

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
package org.elasticsearch.xpack.esql.action;
99

1010
import org.elasticsearch.Build;
11+
import org.elasticsearch.cluster.metadata.DataStream;
1112
import org.elasticsearch.common.util.FeatureFlag;
1213
import org.elasticsearch.features.NodeFeature;
1314
import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction;
@@ -761,7 +762,12 @@ public enum Cap {
761762
/**
762763
* Make numberOfChannels consistent with layout in DefaultLayout by removing duplicated ChannelSet.
763764
*/
764-
MAKE_NUMBER_OF_CHANNELS_CONSISTENT_WITH_LAYOUT;
765+
MAKE_NUMBER_OF_CHANNELS_CONSISTENT_WITH_LAYOUT,
766+
767+
/**
768+
* Index component selector syntax (my-data-stream-name::failures)
769+
*/
770+
INDEX_COMPONENT_SELECTORS(DataStream.isFailureStoreFeatureFlagEnabled());
765771

766772
private final boolean enabled;
767773

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp‎

Lines changed: 3 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)