Skip to content

Commit 16cfcbb

Browse files
gmarouliomricohenn
gmarouli
authored and
omricohenn
committed
Refactor data stream lifecycle to use the template paradigm (elastic#124593)
1 parent 95a1eb4 commit 16cfcbb

File tree

46 files changed

+1148
-756
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+1148
-756
lines changed

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1384,7 +1384,7 @@ public void testSearchAllResolvesDataStreams() throws Exception {
13841384

13851385
public void testGetDataStream() throws Exception {
13861386
Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build();
1387-
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build();
1387+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
13881388
putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false);
13891389
int numDocsFoo = randomIntBetween(2, 16);
13901390
indexDocs("metrics-foo", numDocsFoo);
@@ -1400,7 +1400,7 @@ public void testGetDataStream() throws Exception {
14001400
assertThat(metricsFooDataStream.getDataStreamStatus(), is(ClusterHealthStatus.YELLOW));
14011401
assertThat(metricsFooDataStream.getIndexTemplate(), is("template_for_foo"));
14021402
assertThat(metricsFooDataStream.getIlmPolicy(), is(nullValue()));
1403-
assertThat(dataStream.getLifecycle(), is(lifecycle));
1403+
assertThat(dataStream.getLifecycle(), is(lifecycle.toDataStreamLifecycle()));
14041404
assertThat(metricsFooDataStream.templatePreferIlmValue(), is(true));
14051405
GetDataStreamAction.Response.IndexProperties indexProperties = metricsFooDataStream.getIndexSettingsValues()
14061406
.get(dataStream.getWriteIndex());
@@ -2450,7 +2450,7 @@ static void putComposableIndexTemplate(
24502450
@Nullable Settings settings,
24512451
@Nullable Map<String, Object> metadata,
24522452
@Nullable Map<String, AliasMetadata> aliases,
2453-
@Nullable DataStreamLifecycle lifecycle,
2453+
@Nullable DataStreamLifecycle.Template lifecycle,
24542454
boolean withFailureStore
24552455
) throws IOException {
24562456
TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import java.util.List;
2626

2727
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.putComposableIndexTemplate;
28-
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomLifecycle;
28+
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomLifecycleTemplate;
2929
import static org.hamcrest.Matchers.equalTo;
3030
import static org.hamcrest.Matchers.is;
3131
import static org.hamcrest.Matchers.notNullValue;
@@ -39,7 +39,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
3939
}
4040

4141
public void testGetLifecycle() throws Exception {
42-
DataStreamLifecycle lifecycle = randomLifecycle();
42+
DataStreamLifecycle.Template lifecycle = randomLifecycleTemplate();
4343
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
4444
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
4545
{
@@ -82,9 +82,9 @@ public void testGetLifecycle() throws Exception {
8282
).get();
8383
assertThat(response.getDataStreamLifecycles().size(), equalTo(3));
8484
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
85-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
85+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
8686
assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
87-
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
87+
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
8888
assertThat(response.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("without-lifecycle"));
8989
assertThat(response.getDataStreamLifecycles().get(2).lifecycle(), is(nullValue()));
9090
assertThat(response.getRolloverConfiguration(), nullValue());
@@ -102,9 +102,9 @@ public void testGetLifecycle() throws Exception {
102102
).get();
103103
assertThat(response.getDataStreamLifecycles().size(), equalTo(2));
104104
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
105-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
105+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
106106
assertThat(response.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
107-
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), is(lifecycle));
107+
assertThat(response.getDataStreamLifecycles().get(1).lifecycle(), is(lifecycle.toDataStreamLifecycle()));
108108
assertThat(response.getRolloverConfiguration(), nullValue());
109109
}
110110

@@ -120,7 +120,7 @@ public void testGetLifecycle() throws Exception {
120120
).get();
121121
assertThat(response.getDataStreamLifecycles().size(), equalTo(2));
122122
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
123-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
123+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
124124
assertThat(response.getRolloverConfiguration(), nullValue());
125125
}
126126

@@ -135,9 +135,9 @@ public void testGetLifecycle() throws Exception {
135135
).get();
136136
assertThat(responseWithRollover.getDataStreamLifecycles().size(), equalTo(3));
137137
assertThat(responseWithRollover.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("with-lifecycle-1"));
138-
assertThat(responseWithRollover.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle));
138+
assertThat(responseWithRollover.getDataStreamLifecycles().get(0).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
139139
assertThat(responseWithRollover.getDataStreamLifecycles().get(1).dataStreamName(), equalTo("with-lifecycle-2"));
140-
assertThat(responseWithRollover.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle));
140+
assertThat(responseWithRollover.getDataStreamLifecycles().get(1).lifecycle(), equalTo(lifecycle.toDataStreamLifecycle()));
141141
assertThat(responseWithRollover.getDataStreamLifecycles().get(2).dataStreamName(), equalTo("without-lifecycle"));
142142
assertThat(responseWithRollover.getDataStreamLifecycles().get(2).lifecycle(), is(nullValue()));
143143
assertThat(responseWithRollover.getRolloverConfiguration(), notNullValue());
@@ -192,8 +192,8 @@ public void testPutLifecycle() throws Exception {
192192
).get();
193193
assertThat(response.getDataStreamLifecycles().size(), equalTo(1));
194194
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("my-data-stream"));
195-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().getDataStreamRetention(), equalTo(dataRetention));
196-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().isEnabled(), equalTo(true));
195+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().dataRetention(), equalTo(dataRetention));
196+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().enabled(), equalTo(true));
197197
}
198198

199199
// Disable the lifecycle
@@ -220,13 +220,13 @@ public void testPutLifecycle() throws Exception {
220220
).get();
221221
assertThat(response.getDataStreamLifecycles().size(), equalTo(1));
222222
assertThat(response.getDataStreamLifecycles().get(0).dataStreamName(), equalTo("my-data-stream"));
223-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().getDataStreamRetention(), equalTo(dataRetention));
224-
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().isEnabled(), equalTo(false));
223+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().dataRetention(), equalTo(dataRetention));
224+
assertThat(response.getDataStreamLifecycles().get(0).lifecycle().enabled(), equalTo(false));
225225
}
226226
}
227227

228228
public void testDeleteLifecycle() throws Exception {
229-
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build();
229+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
230230
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
231231
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
232232
{

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
207207
Template.builder()
208208
.settings(Settings.EMPTY)
209209
.mappings(mappings)
210-
.lifecycle(DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build())
210+
.lifecycle(DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build())
211211
)
212212
.dataStreamTemplate(new DataStreamTemplate())
213213
.build(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ public void cleanup() {
146146

147147
public void testRolloverLifecycle() throws Exception {
148148
// empty lifecycle contains the default rollover
149-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
149+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
150150

151151
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
152152
String dataStreamName = "metrics-foo";
@@ -178,7 +178,7 @@ public void testRolloverLifecycle() throws Exception {
178178
}
179179

180180
public void testRolloverAndRetention() throws Exception {
181-
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(0).build();
181+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.ZERO).build();
182182

183183
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
184184

@@ -321,7 +321,7 @@ public void testOriginationDate() throws Exception {
321321
* days ago, and one with an origination date 1 day ago. After data stream lifecycle runs, we expect the one with the old
322322
* origination date to have been deleted, and the one with the newer origination date to remain.
323323
*/
324-
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder().dataRetention(TimeValue.timeValueDays(7)).build();
324+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.timeValueDays(7)).build();
325325

326326
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
327327

@@ -393,7 +393,7 @@ public void testOriginationDate() throws Exception {
393393
}
394394

395395
public void testUpdatingLifecycleAppliesToAllBackingIndices() throws Exception {
396-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
396+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
397397

398398
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
399399

@@ -437,7 +437,7 @@ public void testAutomaticForceMerge() throws Exception {
437437
* because all necessary merging has already happened automatically. So in order to detect whether forcemerge has been called, we
438438
* use a SendRequestBehavior in the MockTransportService to detect it.
439439
*/
440-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
440+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
441441
disableDataStreamLifecycle();
442442
String dataStreamName = "metrics-foo";
443443
putComposableIndexTemplate(
@@ -539,7 +539,7 @@ private static void disableDataStreamLifecycle() {
539539

540540
public void testErrorRecordingOnRollover() throws Exception {
541541
// empty lifecycle contains the default rollover
542-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
542+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
543543
/*
544544
* We set index.auto_expand_replicas to 0-1 so that if we get a single-node cluster it is not yellow. The cluster being yellow
545545
* could result in data stream lifecycle's automatic forcemerge failing, which would result in an unexpected error in the error
@@ -697,7 +697,7 @@ public void testErrorRecordingOnRollover() throws Exception {
697697
public void testErrorRecordingOnRetention() throws Exception {
698698
// starting with a lifecycle without retention so we can rollover the data stream and manipulate the second generation index such
699699
// that its retention execution fails
700-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
700+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
701701

702702
/*
703703
* We set index.auto_expand_replicas to 0-1 so that if we get a single-node cluster it is not yellow. The cluster being yellow
@@ -871,7 +871,7 @@ public void testErrorRecordingOnRetention() throws Exception {
871871
}
872872

873873
public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception {
874-
DataStreamLifecycle lifecycle = new DataStreamLifecycle();
874+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.DEFAULT;
875875

876876
putComposableIndexTemplate(
877877
"id1",
@@ -972,7 +972,7 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception
972972

973973
public void testReenableDataStreamLifecycle() throws Exception {
974974
// start with a lifecycle that's not enabled
975-
DataStreamLifecycle lifecycle = new DataStreamLifecycle(null, null, false);
975+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().enabled(false).build();
976976

977977
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
978978
String dataStreamName = "metrics-foo";
@@ -1031,15 +1031,13 @@ public void testReenableDataStreamLifecycle() throws Exception {
10311031

10321032
public void testLifecycleAppliedToFailureStore() throws Exception {
10331033
// We configure a lifecycle with downsampling to ensure it doesn't fail
1034-
DataStreamLifecycle lifecycle = DataStreamLifecycle.newBuilder()
1035-
.dataRetention(20_000)
1034+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
1035+
.dataRetention(TimeValue.timeValueSeconds(20))
10361036
.downsampling(
1037-
new DataStreamLifecycle.Downsampling(
1038-
List.of(
1039-
new DataStreamLifecycle.Downsampling.Round(
1040-
TimeValue.timeValueMillis(10),
1041-
new DownsampleConfig(new DateHistogramInterval("10m"))
1042-
)
1037+
List.of(
1038+
new DataStreamLifecycle.DownsamplingRound(
1039+
TimeValue.timeValueMillis(10),
1040+
new DownsampleConfig(new DateHistogramInterval("10m"))
10431041
)
10441042
)
10451043
)
@@ -1205,7 +1203,7 @@ static void putComposableIndexTemplate(
12051203
List<String> patterns,
12061204
@Nullable Settings settings,
12071205
@Nullable Map<String, Object> metadata,
1208-
@Nullable DataStreamLifecycle lifecycle,
1206+
@Nullable DataStreamLifecycle.Template lifecycle,
12091207
boolean withFailureStore
12101208
) throws IOException {
12111209
TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(id);
@@ -1268,7 +1266,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
12681266
Template.builder()
12691267
.settings(Settings.EMPTY)
12701268
.lifecycle(
1271-
DataStreamLifecycle.newBuilder()
1269+
DataStreamLifecycle.Template.builder()
12721270
.dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
12731271
)
12741272
)

0 commit comments

Comments
 (0)