Skip to content

Commit 6d03314

Browse files
gmarouligmarouli
committed
Refactor data stream lifecycle to use the template paradigm (elastic#124593)
(cherry picked from commit ce04da7) # Conflicts: # modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java # modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java # modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java # server/src/main/java/org/elasticsearch/TransportVersions.java # server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java # server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java # server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java # server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java # server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java # server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java # server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
1 parent ca54d79 commit 6d03314

File tree

45 files changed

+1142
-755
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1142
-755
lines changed

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1372,7 +1372,7 @@ public void testSearchAllResolvesDataStreams() throws Exception {
13721372

13731373
public void testGetDataStream() throws Exception {
13741374
Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build();
1375-
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build();
1375+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
13761376
putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false);
13771377
int numDocsFoo = randomIntBetween(2, 16);
13781378
indexDocs("metrics-foo", numDocsFoo);
@@ -1388,7 +1388,7 @@ public void testGetDataStream() throws Exception {
13881388
assertThat(metricsFooDataStream.getDataStreamStatus(), is(ClusterHealthStatus.YELLOW));
13891389
assertThat(metricsFooDataStream.getIndexTemplate(), is("template_for_foo"));
13901390
assertThat(metricsFooDataStream.getIlmPolicy(), is(nullValue()));
1391-
assertThat(dataStream.getLifecycle(), is(lifecycle));
1391+
assertThat(dataStream.getLifecycle(), is(lifecycle.toDataStreamLifecycle()));
13921392
assertThat(metricsFooDataStream.templatePreferIlmValue(), is(true));
13931393
GetDataStreamAction.Response.IndexProperties indexProperties = metricsFooDataStream.getIndexSettingsValues()
13941394
.get(dataStream.getWriteIndex());
@@ -2435,7 +2435,7 @@ static void putComposableIndexTemplate(
24352435
@Nullable Settings settings,
24362436
@Nullable Map<String, Object> metadata,
24372437
@Nullable Map<String, AliasMetadata> aliases,
2438-
@Nullable DataStreamLifecycle lifecycle,
2438+
@Nullable DataStreamLifecycle.Template lifecycle,
24392439
boolean withFailureStore
24402440
) throws IOException {
24412441
TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import java.util.List;
2626

2727
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.putComposableIndexTemplate;
28-
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomLifecycle;
28+
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomLifecycleTemplate;
2929
import static org.hamcrest.Matchers.equalTo;
3030
import static org.hamcrest.Matchers.is;
3131
import static org.hamcrest.Matchers.notNullValue;
@@ -39,7 +39,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
3939
}
4040

4141
public void testGetLifecycle() throws Exception {
42-
DataStreamLifecycle lifecycle = randomLifecycle();
42+
DataStreamLifecycle.Template lifecycle = randomLifecycleTemplate();
4343
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
4444
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
4545
{
@@ -82,9 +82,9 @@ public void testGetLifecycle() throws Exception {
8282
).get();
8383
assertThat(response.getDataStreamLifecycles().size(), equalTo(3));
8484
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
85-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
85+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
8686
assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
87-
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
87+
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
8888
assertThat(response.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("without-lifecycle"));
8989
assertThat(response.getDataStreamLifecycles().get(2).lifecycle(), is(nullValue()));
9090
assertThat(response.getRolloverConfiguration(), nullValue());
@@ -102,9 +102,9 @@ public void testGetLifecycle() throws Exception {
102102
).get();
103103
assertThat(response.getDataStreamLifecycles().size(), equalTo(2));
104104
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
105-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
105+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
106106
assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
107-
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), is(lifecycle));
107+
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), is(lifecycle.toDataStreamLifecycle()));
108108
assertThat(response.getRolloverConfiguration(), nullValue());
109109
}
110110

@@ -120,7 +120,7 @@ public void testGetLifecycle() throws Exception {
120120
).get();
121121
assertThat(response.getDataStreamLifecycles().size(), equalTo(2));
122122
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
123-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
123+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
124124
assertThat(response.getRolloverConfiguration(), nullValue());
125125
}
126126

@@ -135,9 +135,9 @@ public void testGetLifecycle() throws Exception {
135135
).get();
136136
assertThat(responseWithRollover.getDataStreamLifecycles().size(), equalTo(3));
137137
assertThat(responseWithRollover.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
138-
assertThat(responseWithRollover.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
138+
assertThat(responseWithRollover.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
139139
assertThat(responseWithRollover.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
140-
assertThat(responseWithRollover.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
140+
assertThat(responseWithRollover.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
141141
assertThat(responseWithRollover.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("without-lifecycle"));
142142
assertThat(responseWithRollover.getDataStreamLifecycles().get(2).lifecycle(), is(nullValue()));
143143
assertThat(responseWithRollover.getRolloverConfiguration(), notNullValue());
@@ -192,8 +192,8 @@ public void testPutLifecycle() throws Exception {
192192
).get();
193193
assertThat(response.getDataStreamLifecycles().size(), equalTo(1));
194194
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("my-data-stream"));
195-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().getDataStreamRetention(), equalTo(dataRetention));
196-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().isEnabled(), equalTo(true));
195+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().dataRetention(), equalTo(dataRetention));
196+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().enabled(), equalTo(true));
197197
}
198198

199199
// Disable the lifecycle
@@ -220,13 +220,13 @@ public void testPutLifecycle() throws Exception {
220220
).get();
221221
assertThat(response.getDataStreamLifecycles().size(), equalTo(1));
222222
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("my-data-stream"));
223-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().getDataStreamRetention(), equalTo(dataRetention));
224-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().isEnabled(), equalTo(false));
223+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().dataRetention(), equalTo(dataRetention));
224+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().enabled(), equalTo(false));
225225
}
226226
}
227227

228228
public void testDeleteLifecycle() throws Exception {
229-
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build();
229+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
230230
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
231231
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
232232
{

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
204204
Template.builder()
205205
.settings(Settings.EMPTY)
206206
.mappings(mappings)
207-
.lifecycle(DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build())
207+
.lifecycle(DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build())
208208
)
209209
.dataStreamTemplate(new DataStreamTemplate())
210210
.build(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ public void cleanup() {
146146

147147
public void testRolloverLifecycle() throws Exception {
148148
// empty lifecycle contains the default rollover
149-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
149+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
150150

151151
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
152152
String dataStreamName = "metrics-foo";
@@ -178,7 +178,7 @@ public void testRolloverLifecycle() throws Exception {
178178
}
179179

180180
public void testRolloverAndRetention() throws Exception {
181-
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(0).build();
181+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.ZERO).build();
182182

183183
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
184184

@@ -321,7 +321,7 @@ public void testOriginationDate() throws Exception {
321321
* days ago, and one with an origination date 1 day ago. After data stream lifecycle runs, we expect the one with the old
322322
* origination date to have been deleted, and the one with the newer origination date to remain.
323323
*/
324-
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(TimeValue.timeValueDays(7)).build();
324+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.timeValueDays(7)).build();
325325

326326
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
327327

@@ -393,7 +393,7 @@ public void testOriginationDate() throws Exception {
393393
}
394394

395395
public void testUpdatingLifecycleAppliesToAllBackingIndices() throws Exception {
396-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
396+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
397397

398398
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
399399

@@ -437,7 +437,7 @@ public void testAutomaticForceMerge() throws Exception {
437437
* because all necessary merging has already happened automatically. So in order to detect whether forcemerge has been called, we
438438
* use a SendRequestBehavior in the MockTransportService to detect it.
439439
*/
440-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
440+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
441441
disableDataStreamLifecycle();
442442
String dataStreamName = "metrics-foo";
443443
putComposableIndexTemplate(
@@ -539,7 +539,7 @@ private static void disableDataStreamLifecycle() {
539539

540540
public void testErrorRecordingOnRollover() throws Exception {
541541
// empty lifecycle contains the default rollover
542-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
542+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
543543
/*
544544
* We set index.auto_expand_replicas to 0-1 so that if we get a single-node cluster it is not yellow. The cluster being yellow
545545
* could result in data stream lifecycle's automatic forcemerge failing, which would result in an unexpected error in the error
@@ -697,7 +697,7 @@ public void testErrorRecordingOnRollover() throws Exception {
697697
public void testErrorRecordingOnRetention() throws Exception {
698698
// starting with a lifecycle without retention so we can rollover the data stream and manipulate the second generation index such
699699
// that its retention execution fails
700-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
700+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
701701

702702
/*
703703
* We set index.auto_expand_replicas to 0-1 so that if we get a single-node cluster it is not yellow. The cluster being yellow
@@ -871,7 +871,7 @@ public void testErrorRecordingOnRetention() throws Exception {
871871
}
872872

873873
public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception {
874-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
874+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
875875

876876
putComposableIndexTemplate(
877877
"id1",
@@ -970,7 +970,7 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception
970970

971971
public void testReenableDataStreamLifecycle() throws Exception {
972972
// start with a lifecycle that's not enabled
973-
DataStreamLifecycle lifecycle = new DataStreamLifecycle(null, null, false);
973+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().enabled(false).build();
974974

975975
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
976976
String dataStreamName = "metrics-foo";
@@ -1029,15 +1029,13 @@ public void testReenableDataStreamLifecycle() throws Exception {
10291029

10301030
public void testLifecycleAppliedToFailureStore() throws Exception {
10311031
// We configure a lifecycle with downsampling to ensure it doesn't fail
1032-
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder()
1033-
.dataRetention(20_000)
1032+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
1033+
.dataRetention(TimeValue.timeValueSeconds(20))
10341034
.downsampling(
1035-
new DataStreamLifecycle.Downsampling(
1036-
List.of(
1037-
new DataStreamLifecycle.Downsampling.Round(
1038-
TimeValue.timeValueMillis(10),
1039-
new DownsampleConfig(new DateHistogramInterval("10m"))
1040-
)
1035+
List.of(
1036+
new DataStreamLifecycle.DownsamplingRound(
1037+
TimeValue.timeValueMillis(10),
1038+
new DownsampleConfig(new DateHistogramInterval("10m"))
10411039
)
10421040
)
10431041
)
@@ -1202,7 +1200,7 @@ static void putComposableIndexTemplate(
12021200
List<String> patterns,
12031201
@Nullable Settings settings,
12041202
@Nullable Map<String, Object> metadata,
1205-
@Nullable DataStreamLifecycle lifecycle,
1203+
@Nullable DataStreamLifecycle.Template lifecycle,
12061204
boolean withFailureStore
12071205
) throws IOException {
12081206
TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id);
@@ -1265,7 +1263,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
12651263
Template.builder()
12661264
.settings(Settings.EMPTY)
12671265
.lifecycle(
1268-
DataStreamLifecycle.newBuilder()
1266+
DataStreamLifecycle.Template.builder()
12691267
.dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
12701268
)
12711269
)

0 commit comments

Comments
 (0)