Skip to content

Commit a91f2cc

Browse files
not-napoleonnot-napoleon
authored
Merge branch 'main' into esql-date-nanos-range-query-bug
2 parents 3807994 + 81ef7dd commit a91f2cc

File tree

21 files changed

+2759
-2098
lines changed

21 files changed

+2759
-2098
lines changed

‎build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.nio.file.Path;
2525
import java.nio.file.Paths;
2626
import java.util.ArrayList;
27+
import java.util.Arrays;
2728
import java.util.HashSet;
2829
import java.util.List;
2930
import java.util.Map;
@@ -45,6 +46,8 @@ public abstract class RunTask extends DefaultTestClustersTask {
4546

4647
private Boolean apmServerEnabled = false;
4748

49+
private List<String> plugins = List.of();
50+
4851
private Boolean preserveData = false;
4952

5053
private Path dataDir = null;
@@ -101,6 +104,22 @@ public void setApmServerEnabled(Boolean apmServerEnabled) {
101104
this.apmServerEnabled = apmServerEnabled;
102105
}
103106

107+
@Option(option = "with-plugins", description = "Run distribution with plugins installed")
108+
public void setPlugins(String plugins) {
109+
this.plugins = Arrays.asList(plugins.split(","));
110+
for (var cluster : getClusters()) {
111+
for (String plugin : this.plugins) {
112+
cluster.plugin(":plugins:" + plugin);
113+
}
114+
dependsOn(cluster.getPluginAndModuleConfigurations());
115+
}
116+
}
117+
118+
@Input
119+
public List<String> getPlugins() {
120+
return plugins;
121+
}
122+
104123
@Option(option = "data-dir", description = "Override the base data directory used by the testcluster")
105124
public void setDataDir(String dataDirStr) {
106125
dataDir = Paths.get(dataDirStr).toAbsolutePath();

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
import java.util.function.Function;
6262
import java.util.function.LongSupplier;
6363
import java.util.function.Predicate;
64+
import java.util.function.Supplier;
6465

6566
/**
6667
* This class main focus is to resolve multi-syntax target expressions to resources or concrete indices. This resolution is influenced
@@ -2364,13 +2365,7 @@ private static <V> V splitSelectorExpression(String expression, BiFunction<Strin
23642365
int lastDoubleColon = expression.lastIndexOf(SELECTOR_SEPARATOR);
23652366
if (lastDoubleColon >= 0) {
23662367
String suffix = expression.substring(lastDoubleColon + SELECTOR_SEPARATOR.length());
2367-
IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix);
2368-
if (selector == null) {
2369-
throw new InvalidIndexNameException(
2370-
expression,
2371-
"invalid usage of :: separator, [" + suffix + "] is not a recognized selector"
2372-
);
2373-
}
2368+
doValidateSelectorString(() -> expression, suffix);
23742369
String expressionBase = expression.substring(0, lastDoubleColon);
23752370
ensureNoMoreSelectorSeparators(expressionBase, expression);
23762371
return bindFunction.apply(expressionBase, suffix);
@@ -2379,6 +2374,20 @@ private static <V> V splitSelectorExpression(String expression, BiFunction<Strin
23792374
return bindFunction.apply(expression, null);
23802375
}
23812376

2377+
public static void validateIndexSelectorString(String indexName, String suffix) {
2378+
doValidateSelectorString(() -> indexName + SELECTOR_SEPARATOR + suffix, suffix);
2379+
}
2380+
2381+
private static void doValidateSelectorString(Supplier<String> expression, String suffix) {
2382+
IndexComponentSelector selector = IndexComponentSelector.getByKey(suffix);
2383+
if (selector == null) {
2384+
throw new InvalidIndexNameException(
2385+
expression.get(),
2386+
"invalid usage of :: separator, [" + suffix + "] is not a recognized selector"
2387+
);
2388+
}
2389+
}
2390+
23822391
/**
23832392
* Checks the selectors that have been returned from splitting an expression and throws an exception if any were present.
23842393
* @param expression Original expression

‎x-pack/plugin/esql/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ dependencies {
5959
testImplementation project(path: ':modules:analysis-common')
6060
testImplementation project(path: ':modules:ingest-common')
6161
testImplementation project(path: ':modules:legacy-geo')
62+
testImplementation project(path: ':modules:data-streams')
63+
testImplementation project(path: ':modules:mapper-extras')
6264
testImplementation project(xpackModule('esql:compute:test'))
6365
testImplementation('net.nextencia:rrdiagram:0.9.4')
6466
testImplementation('org.webjars.npm:fontsource__roboto-mono:4.5.7')

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,43 @@
88
package org.elasticsearch.xpack.esql.action;
99

1010
import org.elasticsearch.Build;
11+
import org.elasticsearch.action.DocWriteRequest;
1112
import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
1213
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
1314
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
15+
import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction;
16+
import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
1417
import org.elasticsearch.action.bulk.BulkRequestBuilder;
18+
import org.elasticsearch.action.bulk.BulkResponse;
19+
import org.elasticsearch.action.datastreams.DeleteDataStreamAction;
1520
import org.elasticsearch.action.index.IndexRequest;
1621
import org.elasticsearch.action.index.IndexRequestBuilder;
1722
import org.elasticsearch.action.support.WriteRequest;
1823
import org.elasticsearch.client.internal.ClusterAdminClient;
24+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
25+
import org.elasticsearch.cluster.metadata.DataStream;
26+
import org.elasticsearch.cluster.metadata.DataStreamFailureStore;
27+
import org.elasticsearch.cluster.metadata.DataStreamOptions;
1928
import org.elasticsearch.cluster.metadata.IndexMetadata;
29+
import org.elasticsearch.cluster.metadata.ResettableValue;
30+
import org.elasticsearch.cluster.metadata.Template;
2031
import org.elasticsearch.cluster.node.DiscoveryNode;
2132
import org.elasticsearch.common.collect.Iterators;
33+
import org.elasticsearch.common.compress.CompressedXContent;
2234
import org.elasticsearch.common.settings.Setting;
2335
import org.elasticsearch.common.settings.Settings;
36+
import org.elasticsearch.core.TimeValue;
37+
import org.elasticsearch.datastreams.DataStreamsPlugin;
2438
import org.elasticsearch.index.Index;
2539
import org.elasticsearch.index.IndexService;
2640
import org.elasticsearch.index.IndexSettings;
41+
import org.elasticsearch.index.mapper.DateFieldMapper;
42+
import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
2743
import org.elasticsearch.index.query.QueryBuilder;
2844
import org.elasticsearch.index.query.RangeQueryBuilder;
2945
import org.elasticsearch.index.shard.IndexShard;
3046
import org.elasticsearch.indices.IndicesService;
47+
import org.elasticsearch.plugins.Plugin;
3148
import org.elasticsearch.test.ESTestCase;
3249
import org.elasticsearch.test.ListMatcher;
3350
import org.elasticsearch.xcontent.XContentBuilder;
@@ -38,11 +55,13 @@
3855
import org.elasticsearch.xpack.esql.parser.ParsingException;
3956
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
4057
import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
58+
import org.junit.Assume;
4159
import org.junit.Before;
4260

4361
import java.io.IOException;
4462
import java.util.ArrayList;
4563
import java.util.Arrays;
64+
import java.util.Collection;
4665
import java.util.Collections;
4766
import java.util.Comparator;
4867
import java.util.HashMap;
@@ -58,8 +77,10 @@
5877
import java.util.concurrent.TimeUnit;
5978
import java.util.concurrent.atomic.AtomicBoolean;
6079
import java.util.concurrent.atomic.AtomicLong;
80+
import java.util.function.BiConsumer;
6181
import java.util.stream.IntStream;
6282
import java.util.stream.LongStream;
83+
import java.util.stream.Stream;
6384

6485
import static java.util.Comparator.comparing;
6586
import static java.util.Comparator.naturalOrder;
@@ -100,6 +121,11 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
100121
.build();
101122
}
102123

124+
@Override
125+
protected Collection<Class<? extends Plugin>> nodePlugins() {
126+
return Stream.concat(super.nodePlugins().stream(), Stream.of(DataStreamsPlugin.class, MapperExtrasPlugin.class)).toList();
127+
}
128+
103129
public void testProjectConstant() {
104130
try (EsqlQueryResponse results = run("from test | eval x = 1 | keep x")) {
105131
assertThat(results.columns(), equalTo(List.of(new ColumnInfoImpl("x", "integer"))));
@@ -992,6 +1018,176 @@ public void testIndexPatterns() throws Exception {
9921018
}
9931019
}
9941020

1021+
public void testDataStreamPatterns() throws Exception {
1022+
Assume.assumeTrue(DataStream.isFailureStoreFeatureFlagEnabled());
1023+
1024+
Map<String, Long> testCases = new HashMap<>();
1025+
// Concrete data stream with each selector
1026+
testCases.put("test_ds_patterns_1", 5L);
1027+
testCases.put("test_ds_patterns_1::data", 5L);
1028+
testCases.put("test_ds_patterns_1::failures", 3L);
1029+
testCases.put("test_ds_patterns_2", 5L);
1030+
testCases.put("test_ds_patterns_2::data", 5L);
1031+
testCases.put("test_ds_patterns_2::failures", 3L);
1032+
1033+
// Wildcard pattern with each selector
1034+
testCases.put("test_ds_patterns*", 15L);
1035+
testCases.put("test_ds_patterns*::data", 15L);
1036+
testCases.put("test_ds_patterns*::failures", 9L);
1037+
1038+
// Match all pattern with each selector
1039+
testCases.put("*", 15L);
1040+
testCases.put("*::data", 15L);
1041+
testCases.put("*::failures", 9L);
1042+
1043+
// Concrete multi-pattern
1044+
testCases.put("test_ds_patterns_1,test_ds_patterns_2", 10L);
1045+
testCases.put("test_ds_patterns_1::data,test_ds_patterns_2::data", 10L);
1046+
testCases.put("test_ds_patterns_1::failures,test_ds_patterns_2::failures", 6L);
1047+
1048+
// Wildcard multi-pattern
1049+
testCases.put("test_ds_patterns_1*,test_ds_patterns_2*", 10L);
1050+
testCases.put("test_ds_patterns_1*::data,test_ds_patterns_2*::data", 10L);
1051+
testCases.put("test_ds_patterns_1*::failures,test_ds_patterns_2*::failures", 6L);
1052+
1053+
// Wildcard pattern with data stream exclusions for each selector combination (data stream exclusions need * on the end to negate)
1054+
// None (default)
1055+
testCases.put("test_ds_patterns*,-test_ds_patterns_2*", 10L);
1056+
testCases.put("test_ds_patterns*,-test_ds_patterns_2*::data", 10L);
1057+
testCases.put("test_ds_patterns*,-test_ds_patterns_2*::failures", 15L);
1058+
// Subtracting from ::data
1059+
testCases.put("test_ds_patterns*::data,-test_ds_patterns_2*", 10L);
1060+
testCases.put("test_ds_patterns*::data,-test_ds_patterns_2*::data", 10L);
1061+
testCases.put("test_ds_patterns*::data,-test_ds_patterns_2*::failures", 15L);
1062+
// Subtracting from ::failures
1063+
testCases.put("test_ds_patterns*::failures,-test_ds_patterns_2*", 9L);
1064+
testCases.put("test_ds_patterns*::failures,-test_ds_patterns_2*::data", 9L);
1065+
testCases.put("test_ds_patterns*::failures,-test_ds_patterns_2*::failures", 6L);
1066+
// Subtracting from ::*
1067+
testCases.put("test_ds_patterns*::data,test_ds_patterns*::failures,-test_ds_patterns_2*", 19L);
1068+
testCases.put("test_ds_patterns*::data,test_ds_patterns*::failures,-test_ds_patterns_2*::data", 19L);
1069+
testCases.put("test_ds_patterns*::data,test_ds_patterns*::failures,-test_ds_patterns_2*::failures", 21L);
1070+
1071+
testCases.put("\"test_ds_patterns_1,test_ds_patterns_2\"::failures", 8L);
1072+
1073+
runDataStreamTest(testCases, new String[] { "test_ds_patterns_1", "test_ds_patterns_2", "test_ds_patterns_3" }, (key, value) -> {
1074+
try (var results = run("from " + key + " | stats count(@timestamp)")) {
1075+
assertEquals(key, 1, getValuesList(results).size());
1076+
assertEquals(key, value, getValuesList(results).get(0).get(0));
1077+
}
1078+
});
1079+
}
1080+
1081+
public void testDataStreamInvalidPatterns() throws Exception {
1082+
Assume.assumeTrue(DataStream.isFailureStoreFeatureFlagEnabled());
1083+
1084+
Map<String, String> testCases = new HashMap<>();
1085+
// === Errors
1086+
// Only recognized components can be selected
1087+
testCases.put("testXXX::custom", "invalid usage of :: separator, [custom] is not a recognized selector");
1088+
// Spelling is important
1089+
testCases.put("testXXX::failres", "invalid usage of :: separator, [failres] is not a recognized selector");
1090+
// Only the match all wildcard is supported
1091+
testCases.put("testXXX::d*ta", "invalid usage of :: separator, [d*ta] is not a recognized selector");
1092+
// The first instance of :: is split upon so that you cannot chain the selector
1093+
testCases.put("test::XXX::data", "mismatched input '::' expecting {<EOF>, '|', ',', 'metadata'}");
1094+
// Selectors must be outside of date math expressions or else they trip up the selector parsing
1095+
testCases.put("<test-{now/d}::failures>", "Invalid index name [<test-{now/d}], must not contain the following characters [");
1096+
// Only one selector separator is allowed per expression
1097+
testCases.put("::::data", "mismatched input '::' expecting {QUOTED_STRING, UNQUOTED_SOURCE}");
1098+
// Suffix case is not supported because there is no component named with the empty string
1099+
testCases.put("index::", "missing {QUOTED_STRING, UNQUOTED_SOURCE} at '|'");
1100+
1101+
runDataStreamTest(testCases, new String[] { "test_ds_patterns_1" }, (key, value) -> {
1102+
logger.info(key);
1103+
var exception = expectThrows(ParsingException.class, () -> { run("from " + key + " | stats count(@timestamp)").close(); });
1104+
assertThat(exception.getMessage(), containsString(value));
1105+
});
1106+
}
1107+
1108+
private <V> void runDataStreamTest(Map<String, V> testCases, String[] dsNames, BiConsumer<String, V> testMethod) throws IOException {
1109+
boolean deleteTemplate = false;
1110+
List<String> deleteDataStreams = new ArrayList<>();
1111+
try {
1112+
assertAcked(
1113+
client().execute(
1114+
TransportPutComposableIndexTemplateAction.TYPE,
1115+
new TransportPutComposableIndexTemplateAction.Request("test_ds_template").indexTemplate(
1116+
ComposableIndexTemplate.builder()
1117+
.indexPatterns(List.of("test_ds_patterns_*"))
1118+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
1119+
.template(
1120+
Template.builder()
1121+
.mappings(new CompressedXContent("""
1122+
{
1123+
"dynamic": false,
1124+
"properties": {
1125+
"@timestamp": {
1126+
"type": "date"
1127+
},
1128+
"count": {
1129+
"type": "long"
1130+
}
1131+
}
1132+
}"""))
1133+
.dataStreamOptions(
1134+
ResettableValue.create(
1135+
new DataStreamOptions.Template(
1136+
ResettableValue.create(new DataStreamFailureStore.Template(ResettableValue.create(true)))
1137+
)
1138+
)
1139+
)
1140+
.build()
1141+
)
1142+
.build()
1143+
)
1144+
)
1145+
);
1146+
deleteTemplate = true;
1147+
1148+
String time = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis());
1149+
int i = 0;
1150+
for (String dsName : dsNames) {
1151+
BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
1152+
for (String id : Arrays.asList("1", "2", "3", "4", "5")) {
1153+
bulk.add(createDoc(dsName, id, time, ++i * 1000));
1154+
}
1155+
for (String id : Arrays.asList("6", "7", "8")) {
1156+
bulk.add(createDoc(dsName, id, time, "garbage"));
1157+
}
1158+
BulkResponse bulkItemResponses = bulk.get();
1159+
assertThat(bulkItemResponses.hasFailures(), is(false));
1160+
deleteDataStreams.add(dsName);
1161+
ensureYellow(dsName);
1162+
}
1163+
1164+
for (Map.Entry<String, V> testCase : testCases.entrySet()) {
1165+
testMethod.accept(testCase.getKey(), testCase.getValue());
1166+
}
1167+
} finally {
1168+
if (deleteDataStreams.isEmpty() == false) {
1169+
assertAcked(
1170+
client().execute(
1171+
DeleteDataStreamAction.INSTANCE,
1172+
new DeleteDataStreamAction.Request(new TimeValue(30, TimeUnit.SECONDS), deleteDataStreams.toArray(String[]::new))
1173+
)
1174+
);
1175+
}
1176+
if (deleteTemplate) {
1177+
assertAcked(
1178+
client().execute(
1179+
TransportDeleteComposableIndexTemplateAction.TYPE,
1180+
new TransportDeleteComposableIndexTemplateAction.Request("test_ds_template")
1181+
)
1182+
);
1183+
}
1184+
}
1185+
}
1186+
1187+
private static IndexRequest createDoc(String dsName, String id, String ts, Object count) {
1188+
return new IndexRequest(dsName).opType(DocWriteRequest.OpType.CREATE).id(id).source("@timestamp", ts, "count", count);
1189+
}
1190+
9951191
public void testOverlappingIndexPatterns() throws Exception {
9961192
String[] indexNames = { "test_overlapping_index_patterns_1", "test_overlapping_index_patterns_2" };
9971193

‎x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,19 @@ indexPatternAndMetadataFields:
9898

9999
indexPattern
100100
: (clusterString COLON)? indexString
101+
| {this.isDevVersion()}? indexString (CAST_OP selectorString)?
101102
;
102103

103104
clusterString
104105
: UNQUOTED_SOURCE
105106
| QUOTED_STRING
106107
;
107108

109+
selectorString
110+
: UNQUOTED_SOURCE
111+
| QUOTED_STRING
112+
;
113+
108114
indexString
109115
: UNQUOTED_SOURCE
110116
| QUOTED_STRING

‎x-pack/plugin/esql/src/main/antlr/lexer/From.g4‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ FROM_PIPE : PIPE -> type(PIPE), popMode;
1818
FROM_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET);
1919
FROM_CLOSING_BRACKET : CLOSING_BRACKET -> type(CLOSING_BRACKET);
2020
FROM_COLON : COLON -> type(COLON);
21+
FROM_SELECTOR : {this.isDevVersion()}? CAST_OP -> type(CAST_OP);
2122
FROM_COMMA : COMMA -> type(COMMA);
2223
FROM_ASSIGN : ASSIGN -> type(ASSIGN);
2324
METADATA : 'metadata';

0 commit comments

Comments
 (0)