Skip to content

Commit 93354e9

Browse files
luigidellaquilaluigidellaquila
committed
Merge branch 'main' into esql/explain
2 parents 080e82e + 950a0e0 commit 93354e9

File tree

62 files changed

+1926
-1656
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+1926
-1656
lines changed

‎.buildkite/scripts/release-tests.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,13 @@ export BEATS_DIR=$(pwd)/distribution/docker/build/artifacts/beats
99
mkdir -p ${BEATS_DIR}
1010
curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
1111
curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
12-
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
12+
curl --fail -o "${BEATS_DIR}/metricbeat-fips-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-fips-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
13+
curl --fail -o "${BEATS_DIR}/metricbeat-fips-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-fips-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
14+
15+
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
1316
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
17+
curl --fail -o "${BEATS_DIR}/filebeat-fips-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-fips-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
18+
curl --fail -o "${BEATS_DIR}/filebeat-fips-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-fips-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
1419

1520
# Fetch ML artifacts
1621
export ML_IVY_REPO=$(mktemp -d)

‎benchmarks/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ dependencies {
4141
}
4242
api(project(':libs:h3'))
4343
api(project(':modules:aggregations'))
44+
implementation project(':modules:mapper-extras');
4445
api(project(':x-pack:plugin:esql-core'))
4546
api(project(':x-pack:plugin:core'))
4647
api(project(':x-pack:plugin:esql'))

‎benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
3030
import org.elasticsearch.index.similarity.SimilarityService;
3131
import org.elasticsearch.indices.IndicesModule;
32+
import org.elasticsearch.plugins.MapperPlugin;
3233
import org.elasticsearch.script.Script;
3334
import org.elasticsearch.script.ScriptCompiler;
3435
import org.elasticsearch.script.ScriptContext;
@@ -38,11 +39,16 @@
3839
import java.io.IOException;
3940
import java.io.UncheckedIOException;
4041
import java.util.Collections;
42+
import java.util.List;
4143
import java.util.Map;
4244

4345
public class MapperServiceFactory {
4446

4547
public static MapperService create(String mappings) {
48+
return create(mappings, Collections.emptyList());
49+
}
50+
51+
public static MapperService create(String mappings, List<MapperPlugin> mapperPlugins) {
4652
Settings settings = Settings.builder()
4753
.put("index.number_of_replicas", 0)
4854
.put("index.number_of_shards", 1)
@@ -51,7 +57,7 @@ public static MapperService create(String mappings) {
5157
.build();
5258
IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();
5359
IndexSettings indexSettings = new IndexSettings(meta, settings);
54-
MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
60+
MapperRegistry mapperRegistry = new IndicesModule(mapperPlugins).getMapperRegistry();
5561

5662
SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
5763
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);

‎benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/OptimizedTextBenchmark.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.common.logging.LogConfigurator;
1616
import org.elasticsearch.index.mapper.MapperService;
1717
import org.elasticsearch.index.mapper.SourceToParse;
18+
import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
1819
import org.elasticsearch.xcontent.XContentBuilder;
1920
import org.elasticsearch.xcontent.XContentFactory;
2021
import org.elasticsearch.xcontent.XContentType;
@@ -34,6 +35,7 @@
3435
import org.openjdk.jmh.infra.Blackhole;
3536

3637
import java.io.IOException;
38+
import java.util.List;
3739
import java.util.Random;
3840
import java.util.concurrent.TimeUnit;
3941

@@ -66,7 +68,7 @@ public class OptimizedTextBenchmark {
6668
private SourceToParse[] sources;
6769

6870
private String randomValue(int length) {
69-
final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
71+
final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ";
7072
Random random = new Random();
7173
StringBuilder builder = new StringBuilder(length);
7274
for (int i = 0; i < length; i++) {
@@ -83,17 +85,17 @@ public void setup() throws IOException {
8385
"dynamic": false,
8486
"properties": {
8587
"field": {
86-
"type": "keyword"
88+
"type": "match_only_text"
8789
}
8890
}
8991
}
9092
}
91-
""");
93+
""", List.of(new MapperExtrasPlugin()));
9294

9395
sources = new SourceToParse[nDocs];
9496
for (int i = 0; i < nDocs; i++) {
9597
XContentBuilder b = XContentFactory.jsonBuilder();
96-
b.startObject().field("field", randomValue(8)).endObject();
98+
b.startObject().field("field", randomValue(512)).endObject();
9799
sources[i] = new SourceToParse(UUIDs.randomBase64UUID(), BytesReference.bytes(b), XContentType.JSON);
98100
}
99101
}

‎docs/changelog/127636.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
pr: 127636
2+
summary: Disallow mixed quoted/unquoted patterns in FROM
3+
area: ES|QL
4+
type: breaking
5+
issues:
6+
- 122651
7+
breaking:
8+
title: Disallow mixed quoted/unquoted patterns in FROM
9+
area: ES|QL
10+
details: "Previously, the ES|QL grammar allowed users to individually quote constituent strings in index patterns\
11+
\ such as \"remote_cluster\":\"index_name\". This would allow users to write complex malformed index patterns\
12+
\ that often slip through grammar and the subsequent validation. This could result in runtime errors\
13+
\ that can be misleading. This change simplifies the grammar to early reject such malformed index patterns\
14+
\ at the parsing stage, allowing users to write simpler queries and see more relevant and meaningful\
15+
\ errors."
16+
impact: "Users can write queries with simpler index patterns and see more meaningful and relevant errors."
17+
notable: false

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public void testRolloverClusterStateForDataStream() throws Exception {
6565
final DataStream dataStream = DataStream.builder(
6666
dataStreamName,
6767
List.of(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, 1, now.toEpochMilli()), "uuid"))
68-
).setIndexMode(IndexMode.TIME_SERIES).build();
68+
).setIndexMode(IndexMode.TIME_SERIES).setTimeProvider(now::toEpochMilli).build();
6969
ComposableIndexTemplate template = ComposableIndexTemplate.builder()
7070
.indexPatterns(List.of(dataStream.getName() + "*"))
7171
.template(
@@ -123,8 +123,16 @@ public void testRolloverClusterStateForDataStream() throws Exception {
123123
);
124124
long after = testThreadPool.absoluteTimeInMillis();
125125

126-
String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
127-
String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
126+
String sourceIndexName = DataStream.getDefaultBackingIndexName(
127+
dataStream.getName(),
128+
dataStream.getGeneration(),
129+
now.toEpochMilli()
130+
);
131+
String newIndexName = DataStream.getDefaultBackingIndexName(
132+
dataStream.getName(),
133+
dataStream.getGeneration() + 1,
134+
now.toEpochMilli()
135+
);
128136
assertEquals(sourceIndexName, rolloverResult.sourceIndexName());
129137
assertEquals(newIndexName, rolloverResult.rolloverIndexName());
130138
ProjectMetadata rolloverMetadata = rolloverResult.clusterState().metadata().getProject(projectId);
@@ -219,8 +227,16 @@ public void testRolloverAndMigrateDataStream() throws Exception {
219227
false
220228
);
221229

222-
String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
223-
String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
230+
String sourceIndexName = DataStream.getDefaultBackingIndexName(
231+
dataStream.getName(),
232+
dataStream.getGeneration(),
233+
now.toEpochMilli()
234+
);
235+
String newIndexName = DataStream.getDefaultBackingIndexName(
236+
dataStream.getName(),
237+
dataStream.getGeneration() + 1,
238+
now.toEpochMilli()
239+
);
224240
assertEquals(sourceIndexName, rolloverResult.sourceIndexName());
225241
assertEquals(newIndexName, rolloverResult.rolloverIndexName());
226242
ProjectMetadata rolloverMetadata = rolloverResult.clusterState().metadata().getProject(projectId);
@@ -308,8 +324,16 @@ public void testChangingIndexModeFromTimeSeriesToSomethingElseNoEffectOnExisting
308324
false
309325
);
310326

311-
String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
312-
String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
327+
String sourceIndexName = DataStream.getDefaultBackingIndexName(
328+
dataStream.getName(),
329+
dataStream.getGeneration(),
330+
now.toEpochMilli()
331+
);
332+
String newIndexName = DataStream.getDefaultBackingIndexName(
333+
dataStream.getName(),
334+
dataStream.getGeneration() + 1,
335+
now.toEpochMilli()
336+
);
313337
assertEquals(sourceIndexName, rolloverResult.sourceIndexName());
314338
assertEquals(newIndexName, rolloverResult.rolloverIndexName());
315339
ProjectMetadata rolloverMetadata = rolloverResult.clusterState().metadata().getProject(projectId);
@@ -379,8 +403,16 @@ public void testRolloverClusterStateWithBrokenOlderTsdbDataStream() throws Excep
379403
);
380404
long after = testThreadPool.absoluteTimeInMillis();
381405

382-
String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
383-
String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
406+
String sourceIndexName = DataStream.getDefaultBackingIndexName(
407+
dataStream.getName(),
408+
dataStream.getGeneration(),
409+
now.toEpochMilli()
410+
);
411+
String newIndexName = DataStream.getDefaultBackingIndexName(
412+
dataStream.getName(),
413+
dataStream.getGeneration() + 1,
414+
now.toEpochMilli()
415+
);
384416
assertEquals(sourceIndexName, rolloverResult.sourceIndexName());
385417
assertEquals(newIndexName, rolloverResult.rolloverIndexName());
386418
ProjectMetadata rolloverMetadata = rolloverResult.clusterState().metadata().getProject(projectId);

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ public void testRetentionSkippedWhilstDownsamplingInProgress() {
394394
{
395395
ProjectMetadata.Builder newProjectBuilder = ProjectMetadata.builder(project);
396396

397-
String firstBackingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
397+
String firstBackingIndex = dataStream.getIndices().getFirst().getName();
398398
IndexMetadata indexMetadata = project.index(firstBackingIndex);
399399
IndexMetadata.Builder indexMetaBuilder = IndexMetadata.builder(indexMetadata);
400400
indexMetaBuilder.settings(
@@ -418,15 +418,15 @@ public void testRetentionSkippedWhilstDownsamplingInProgress() {
418418
assertThat(clientSeenRequests.get(1), instanceOf(DeleteIndexRequest.class));
419419
assertThat(
420420
((DeleteIndexRequest) clientSeenRequests.get(1)).indices()[0],
421-
is(DataStream.getDefaultBackingIndexName(dataStreamName, 2))
421+
DataStreamTestHelper.backingIndexEqualTo(dataStreamName, 2)
422422
);
423423
}
424424

425425
{
426426
// a lack of downsample status (i.e. the default `UNKNOWN`) must not prevent retention
427427
ProjectMetadata.Builder newProjectBuilder = ProjectMetadata.builder(project);
428428

429-
String firstBackingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
429+
String firstBackingIndex = dataStream.getIndices().getFirst().getName();
430430
IndexMetadata indexMetadata = project.index(firstBackingIndex);
431431
IndexMetadata.Builder indexMetaBuilder = IndexMetadata.builder(indexMetadata);
432432
indexMetaBuilder.settings(
@@ -441,12 +441,12 @@ public void testRetentionSkippedWhilstDownsamplingInProgress() {
441441
assertThat(clientSeenRequests.get(1), instanceOf(DeleteIndexRequest.class));
442442
assertThat(
443443
((DeleteIndexRequest) clientSeenRequests.get(1)).indices()[0],
444-
is(DataStream.getDefaultBackingIndexName(dataStreamName, 2))
444+
DataStreamTestHelper.backingIndexEqualTo(dataStreamName, 2)
445445
);
446446
assertThat(clientSeenRequests.get(2), instanceOf(DeleteIndexRequest.class));
447447
assertThat(
448448
((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0],
449-
is(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
449+
DataStreamTestHelper.backingIndexEqualTo(dataStreamName, 1)
450450
);
451451
}
452452
}
@@ -1195,8 +1195,8 @@ public void testDownsampling() throws Exception {
11951195
nodesBuilder.masterNodeId(nodeId);
11961196
ClusterState state = ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(builder).nodes(nodesBuilder).build();
11971197
setState(clusterService, state);
1198-
String firstGenIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
1199-
Index firstGenIndex = clusterService.state().metadata().getProject(projectId).index(firstGenIndexName).getIndex();
1198+
Index firstGenIndex = dataStream.getIndices().getFirst();
1199+
String firstGenIndexName = firstGenIndex.getName();
12001200
Set<Index> affectedIndices = dataStreamLifecycleService.maybeExecuteDownsampling(
12011201
clusterService.state().projectState(projectId),
12021202
dataStream,
@@ -1342,7 +1342,7 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc
13421342
DiscoveryNodes.Builder nodesBuilder = buildNodes(nodeId);
13431343
// we are the master node
13441344
nodesBuilder.masterNodeId(nodeId);
1345-
String firstGenIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
1345+
String firstGenIndexName = dataStream.getIndices().getFirst().getName();
13461346

13471347
// mark the first generation as read-only already
13481348
IndexMetadata indexMetadata = builder.get(firstGenIndexName);
@@ -1551,7 +1551,7 @@ public void testMaybeExecuteRetentionSuccessfulDownsampledIndex() {
15511551
ClusterState state = downsampleSetup(projectId, dataStreamName, SUCCESS);
15521552
final var project = state.metadata().getProject(projectId);
15531553
DataStream dataStream = project.dataStreams().get(dataStreamName);
1554-
String firstGenIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
1554+
String firstGenIndexName = dataStream.getIndices().getFirst().getName();
15551555
TimeValue dataRetention = dataStream.getDataLifecycle().dataRetention();
15561556

15571557
// Executing the method to be tested:
@@ -1590,7 +1590,7 @@ public void testMaybeExecuteRetentionDownsampledUnknown() {
15901590
ClusterState state = downsampleSetup(projectId, dataStreamName, UNKNOWN);
15911591
final var project = state.metadata().getProject(projectId);
15921592
DataStream dataStream = project.dataStreams().get(dataStreamName);
1593-
String firstGenIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
1593+
String firstGenIndexName = dataStream.getIndices().getFirst().getName();
15941594
TimeValue dataRetention = dataStream.getDataLifecycle().dataRetention();
15951595

15961596
// Executing the method to be tested:
@@ -1624,7 +1624,7 @@ private ClusterState downsampleSetup(ProjectId projectId, String dataStreamName,
16241624
builder.put(dataStream);
16251625

16261626
// Update the first backing index so that is appears to have been downsampled:
1627-
String firstGenIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
1627+
String firstGenIndexName = dataStream.getIndices().getFirst().getName();
16281628
var imd = builder.get(firstGenIndexName);
16291629
var imdBuilder = new IndexMetadata.Builder(imd);
16301630
imdBuilder.settings(Settings.builder().put(imd.getSettings()).put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey(), status).build());

0 commit comments

Comments
 (0)