Skip to content

Commit aba37e1

Browse files
masseykemasseyke
authored
Merge branch 'main' into using-temp-indexservice-for-template-validation
2 parents 203f376 + 9f78d11 commit aba37e1

File tree

64 files changed

+2386
-2185
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+2386
-2185
lines changed

‎.buildkite/scripts/release-tests.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,13 @@ export BEATS_DIR=$(pwd)/distribution/docker/build/artifacts/beats
99
mkdir -p ${BEATS_DIR}
1010
curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
1111
curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
12-
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
12+
curl --fail -o "${BEATS_DIR}/metricbeat-fips-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-fips-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
13+
curl --fail -o "${BEATS_DIR}/metricbeat-fips-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-fips-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
14+
15+
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
1316
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
17+
curl --fail -o "${BEATS_DIR}/filebeat-fips-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-fips-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
18+
curl --fail -o "${BEATS_DIR}/filebeat-fips-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-fips-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
1419

1520
# Fetch ML artifacts
1621
export ML_IVY_REPO=$(mktemp -d)

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public void testRolloverClusterStateForDataStream() throws Exception {
6565
final DataStream dataStream = DataStream.builder(
6666
dataStreamName,
6767
List.of(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, 1, now.toEpochMilli()), "uuid"))
68-
).setIndexMode(IndexMode.TIME_SERIES).build();
68+
).setIndexMode(IndexMode.TIME_SERIES).setTimeProvider(now::toEpochMilli).build();
6969
ComposableIndexTemplate template = ComposableIndexTemplate.builder()
7070
.indexPatterns(List.of(dataStream.getName() + "*"))
7171
.template(
@@ -123,8 +123,16 @@ public void testRolloverClusterStateForDataStream() throws Exception {
123123
);
124124
long after = testThreadPool.absoluteTimeInMillis();
125125

126-
String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
127-
String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
126+
String sourceIndexName = DataStream.getDefaultBackingIndexName(
127+
dataStream.getName(),
128+
dataStream.getGeneration(),
129+
now.toEpochMilli()
130+
);
131+
String newIndexName = DataStream.getDefaultBackingIndexName(
132+
dataStream.getName(),
133+
dataStream.getGeneration() + 1,
134+
now.toEpochMilli()
135+
);
128136
assertEquals(sourceIndexName, rolloverResult.sourceIndexName());
129137
assertEquals(newIndexName, rolloverResult.rolloverIndexName());
130138
ProjectMetadata rolloverMetadata = rolloverResult.clusterState().metadata().getProject(projectId);
@@ -219,8 +227,16 @@ public void testRolloverAndMigrateDataStream() throws Exception {
219227
false
220228
);
221229

222-
String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
223-
String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
230+
String sourceIndexName = DataStream.getDefaultBackingIndexName(
231+
dataStream.getName(),
232+
dataStream.getGeneration(),
233+
now.toEpochMilli()
234+
);
235+
String newIndexName = DataStream.getDefaultBackingIndexName(
236+
dataStream.getName(),
237+
dataStream.getGeneration() + 1,
238+
now.toEpochMilli()
239+
);
224240
assertEquals(sourceIndexName, rolloverResult.sourceIndexName());
225241
assertEquals(newIndexName, rolloverResult.rolloverIndexName());
226242
ProjectMetadata rolloverMetadata = rolloverResult.clusterState().metadata().getProject(projectId);
@@ -308,8 +324,16 @@ public void testChangingIndexModeFromTimeSeriesToSomethingElseNoEffectOnExisting
308324
false
309325
);
310326

311-
String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
312-
String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
327+
String sourceIndexName = DataStream.getDefaultBackingIndexName(
328+
dataStream.getName(),
329+
dataStream.getGeneration(),
330+
now.toEpochMilli()
331+
);
332+
String newIndexName = DataStream.getDefaultBackingIndexName(
333+
dataStream.getName(),
334+
dataStream.getGeneration() + 1,
335+
now.toEpochMilli()
336+
);
313337
assertEquals(sourceIndexName, rolloverResult.sourceIndexName());
314338
assertEquals(newIndexName, rolloverResult.rolloverIndexName());
315339
ProjectMetadata rolloverMetadata = rolloverResult.clusterState().metadata().getProject(projectId);
@@ -379,8 +403,16 @@ public void testRolloverClusterStateWithBrokenOlderTsdbDataStream() throws Excep
379403
);
380404
long after = testThreadPool.absoluteTimeInMillis();
381405

382-
String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
383-
String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
406+
String sourceIndexName = DataStream.getDefaultBackingIndexName(
407+
dataStream.getName(),
408+
dataStream.getGeneration(),
409+
now.toEpochMilli()
410+
);
411+
String newIndexName = DataStream.getDefaultBackingIndexName(
412+
dataStream.getName(),
413+
dataStream.getGeneration() + 1,
414+
now.toEpochMilli()
415+
);
384416
assertEquals(sourceIndexName, rolloverResult.sourceIndexName());
385417
assertEquals(newIndexName, rolloverResult.rolloverIndexName());
386418
ProjectMetadata rolloverMetadata = rolloverResult.clusterState().metadata().getProject(projectId);

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ public void testRetentionSkippedWhilstDownsamplingInProgress() {
394394
{
395395
ProjectMetadata.Builder newProjectBuilder = ProjectMetadata.builder(project);
396396

397-
String firstBackingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
397+
String firstBackingIndex = dataStream.getIndices().getFirst().getName();
398398
IndexMetadata indexMetadata = project.index(firstBackingIndex);
399399
IndexMetadata.Builder indexMetaBuilder = IndexMetadata.builder(indexMetadata);
400400
indexMetaBuilder.settings(
@@ -418,15 +418,15 @@ public void testRetentionSkippedWhilstDownsamplingInProgress() {
418418
assertThat(clientSeenRequests.get(1), instanceOf(DeleteIndexRequest.class));
419419
assertThat(
420420
((DeleteIndexRequest) clientSeenRequests.get(1)).indices()[0],
421-
is(DataStream.getDefaultBackingIndexName(dataStreamName, 2))
421+
DataStreamTestHelper.backingIndexEqualTo(dataStreamName, 2)
422422
);
423423
}
424424

425425
{
426426
// a lack of downsample status (i.e. the default `UNKNOWN`) must not prevent retention
427427
ProjectMetadata.Builder newProjectBuilder = ProjectMetadata.builder(project);
428428

429-
String firstBackingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
429+
String firstBackingIndex = dataStream.getIndices().getFirst().getName();
430430
IndexMetadata indexMetadata = project.index(firstBackingIndex);
431431
IndexMetadata.Builder indexMetaBuilder = IndexMetadata.builder(indexMetadata);
432432
indexMetaBuilder.settings(
@@ -441,12 +441,12 @@ public void testRetentionSkippedWhilstDownsamplingInProgress() {
441441
assertThat(clientSeenRequests.get(1), instanceOf(DeleteIndexRequest.class));
442442
assertThat(
443443
((DeleteIndexRequest) clientSeenRequests.get(1)).indices()[0],
444-
is(DataStream.getDefaultBackingIndexName(dataStreamName, 2))
444+
DataStreamTestHelper.backingIndexEqualTo(dataStreamName, 2)
445445
);
446446
assertThat(clientSeenRequests.get(2), instanceOf(DeleteIndexRequest.class));
447447
assertThat(
448448
((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0],
449-
is(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
449+
DataStreamTestHelper.backingIndexEqualTo(dataStreamName, 1)
450450
);
451451
}
452452
}
@@ -1195,8 +1195,8 @@ public void testDownsampling() throws Exception {
11951195
nodesBuilder.masterNodeId(nodeId);
11961196
ClusterState state = ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(builder).nodes(nodesBuilder).build();
11971197
setState(clusterService, state);
1198-
String firstGenIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
1199-
Index firstGenIndex = clusterService.state().metadata().getProject(projectId).index(firstGenIndexName).getIndex();
1198+
Index firstGenIndex = dataStream.getIndices().getFirst();
1199+
String firstGenIndexName = firstGenIndex.getName();
12001200
Set<Index> affectedIndices = dataStreamLifecycleService.maybeExecuteDownsampling(
12011201
clusterService.state().projectState(projectId),
12021202
dataStream,
@@ -1342,7 +1342,7 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc
13421342
DiscoveryNodes.Builder nodesBuilder = buildNodes(nodeId);
13431343
// we are the master node
13441344
nodesBuilder.masterNodeId(nodeId);
1345-
String firstGenIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
1345+
String firstGenIndexName = dataStream.getIndices().getFirst().getName();
13461346

13471347
// mark the first generation as read-only already
13481348
IndexMetadata indexMetadata = builder.get(firstGenIndexName);
@@ -1551,7 +1551,7 @@ public void testMaybeExecuteRetentionSuccessfulDownsampledIndex() {
15511551
ClusterState state = downsampleSetup(projectId, dataStreamName, SUCCESS);
15521552
final var project = state.metadata().getProject(projectId);
15531553
DataStream dataStream = project.dataStreams().get(dataStreamName);
1554-
String firstGenIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
1554+
String firstGenIndexName = dataStream.getIndices().getFirst().getName();
15551555
TimeValue dataRetention = dataStream.getDataLifecycle().dataRetention();
15561556

15571557
// Executing the method to be tested:
@@ -1590,7 +1590,7 @@ public void testMaybeExecuteRetentionDownsampledUnknown() {
15901590
ClusterState state = downsampleSetup(projectId, dataStreamName, UNKNOWN);
15911591
final var project = state.metadata().getProject(projectId);
15921592
DataStream dataStream = project.dataStreams().get(dataStreamName);
1593-
String firstGenIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
1593+
String firstGenIndexName = dataStream.getIndices().getFirst().getName();
15941594
TimeValue dataRetention = dataStream.getDataLifecycle().dataRetention();
15951595

15961596
// Executing the method to be tested:
@@ -1624,7 +1624,7 @@ private ClusterState downsampleSetup(ProjectId projectId, String dataStreamName,
16241624
builder.put(dataStream);
16251625

16261626
// Update the first backing index so that is appears to have been downsampled:
1627-
String firstGenIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
1627+
String firstGenIndexName = dataStream.getIndices().getFirst().getName();
16281628
var imd = builder.get(firstGenIndexName);
16291629
var imdBuilder = new IndexMetadata.Builder(imd);
16301630
imdBuilder.settings(Settings.builder().put(imd.getSettings()).put(IndexMetadata.INDEX_DOWNSAMPLE_STATUS.getKey(), status).build());

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDSTests.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ public void testDownsampleIndexMissingIsNoOp() {
5858
builder.put(dataStream);
5959
ClusterState previousState = ClusterState.builder(ClusterName.DEFAULT).putProjectMetadata(builder).build();
6060

61-
String firstGeneration = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
61+
String firstGeneration = dataStream.getIndices().getFirst().getName();
6262
ClusterState newState = new DeleteSourceAndAddDownsampleToDS(
6363
Settings.EMPTY,
6464
builder.getId(),
@@ -83,7 +83,7 @@ public void testDownsampleIsAddedToDSEvenIfSourceDeleted() {
8383
DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(),
8484
now
8585
);
86-
String firstGenIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
86+
String firstGenIndex = dataStream.getIndices().getFirst().getName();
8787
String downsampleIndex = "downsample-1s-" + firstGenIndex;
8888
IndexMetadata.Builder downsampleIndexMeta = IndexMetadata.builder(downsampleIndex)
8989
.settings(settings(IndexVersion.current()))
@@ -129,7 +129,7 @@ public void testSourceIndexIsWriteIndexThrowsException() {
129129
now
130130
);
131131
builder.put(dataStream);
132-
String writeIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 3);
132+
String writeIndex = dataStream.getIndices().getLast().getName();
133133
String downsampleIndex = "downsample-1s-" + writeIndex;
134134
IndexMetadata.Builder downsampleIndexMeta = IndexMetadata.builder(downsampleIndex)
135135
.settings(settings(IndexVersion.current()))
@@ -162,7 +162,7 @@ public void testSourceIsDeleteAndDownsampleOriginationDateIsConfigured() {
162162
DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(),
163163
now
164164
);
165-
String firstGenIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
165+
String firstGenIndex = dataStream.getIndices().getFirst().getName();
166166
String downsampleIndex = "downsample-1s-" + firstGenIndex;
167167
IndexMetadata.Builder downsampleIndexMeta = IndexMetadata.builder(downsampleIndex)
168168
.settings(settings(IndexVersion.current()))
@@ -224,7 +224,7 @@ public void testSourceWithoutLifecycleMetaAndDestWithOriginationDateAlreadyConfi
224224
DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(),
225225
now
226226
);
227-
String firstGenIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
227+
String firstGenIndex = dataStream.getIndices().getFirst().getName();
228228
String downsampleIndex = "downsample-1s-" + firstGenIndex;
229229
long downsampleOriginationDate = now - randomLongBetween(10_000, 12_000);
230230
IndexMetadata.Builder downsampleIndexMeta = IndexMetadata.builder(downsampleIndex)
@@ -275,7 +275,7 @@ public void testSourceIndexIsDeleteEvenIfNotPartOfDSAnymore() {
275275
DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(),
276276
now
277277
);
278-
String firstGenIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
278+
String firstGenIndex = dataStream.getIndices().getFirst().getName();
279279
String downsampleIndex = "downsample-1s-" + firstGenIndex;
280280
IndexMetadata.Builder downsampleIndexMeta = IndexMetadata.builder(downsampleIndex)
281281
.settings(settings(IndexVersion.current()))

‎server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceAutoShardingTests.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -578,16 +578,16 @@ private void assertRolloverResult(
578578
List<Condition<?>> metConditions,
579579
int expectedNumberOfShards
580580
) {
581-
String sourceIndexName = DataStream.getDefaultBackingIndexName(
582-
preRolloverDataStream.getName(),
583-
preRolloverDataStream.getGeneration()
581+
String sourceIndexName = rolloverResult.sourceIndexName();
582+
assertThat(
583+
sourceIndexName,
584+
DataStreamTestHelper.backingIndexEqualTo(preRolloverDataStream.getName(), (int) preRolloverDataStream.getGeneration())
584585
);
585-
String newIndexName = DataStream.getDefaultBackingIndexName(
586-
preRolloverDataStream.getName(),
587-
preRolloverDataStream.getGeneration() + 1
586+
String newIndexName = rolloverResult.rolloverIndexName();
587+
assertThat(
588+
newIndexName,
589+
DataStreamTestHelper.backingIndexEqualTo(preRolloverDataStream.getName(), (int) preRolloverDataStream.getGeneration() + 1)
588590
);
589-
assertEquals(sourceIndexName, rolloverResult.sourceIndexName());
590-
assertEquals(newIndexName, rolloverResult.rolloverIndexName());
591591
ProjectMetadata rolloverMetadata = rolloverResult.clusterState().metadata().getProject(projectId);
592592
assertEquals(preRolloverDataStream.getIndices().size() + 1, rolloverMetadata.indices().size());
593593
IndexMetadata rolloverIndexMetadata = rolloverMetadata.index(newIndexName);

0 commit comments

Comments
 (0)