Skip to content

Commit 42a6952

Browse files
gmarouligmarouli
authored
Create dedicated factory methods for data lifecycle (#126487) (#127226)
The class `DataStreamLifecycle` is currently capturing the lifecycle configuration that currently manages all data stream indices, but soon enough it will be split into two variants, the data and the failures lifecycle. Some pre-work has been done already but as we are progressing in our POC, we see that it will be really useful if the `DataStreamLifecycle` is "aware" of the target index component. This will allow us to correctly apply global retention or to throw an error if a downsampling configuration is provided to a failure lifecycle. In this PR, we perform a small refactoring to reduce the noise in #125658. Here we introduce the following: - A factory method that creates a data lifecycle, for now it's trivial but it will be more useful soon. - We rename the "empty" builder to explicitly mention the index component it refers to. (cherry picked from commit b991708) # Conflicts: # modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java # server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java # server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
1 parent dac64a1 commit 42a6952

File tree

35 files changed

+225
-164
lines changed

35 files changed

+225
-164
lines changed

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1372,7 +1372,9 @@ public void testSearchAllResolvesDataStreams() throws Exception {
13721372

13731373
public void testGetDataStream() throws Exception {
13741374
Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build();
1375-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate();
1375+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
1376+
.dataRetention(randomPositiveTimeValue())
1377+
.buildTemplate();
13761378
putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false);
13771379
int numDocsFoo = randomIntBetween(2, 16);
13781380
indexDocs("metrics-foo", numDocsFoo);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -398,17 +398,23 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
398398
Type.EXTERNAL,
399399
ComposableIndexTemplate.builder()
400400
.indexPatterns(List.of(".test-failure-store"))
401-
.template(Template.builder().mappings(new CompressedXContent("""
402-
{
403-
"properties": {
404-
"@timestamp" : {
405-
"type": "date"
406-
},
407-
"count": {
408-
"type": "long"
409-
}
410-
}
411-
}""")).dataStreamOptions(new DataStreamOptions.Template(new DataStreamFailureStore.Template(true))))
401+
.template(
402+
Template.builder()
403+
.mappings(new CompressedXContent("""
404+
{
405+
"properties": {
406+
"@timestamp" : {
407+
"type": "date"
408+
},
409+
"count": {
410+
"type": "long"
411+
}
412+
}
413+
}"""))
414+
.dataStreamOptions(
415+
new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate())
416+
)
417+
)
412418
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
413419
.build(),
414420
Map.of(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import java.util.List;
2626

2727
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.putComposableIndexTemplate;
28-
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomLifecycleTemplate;
28+
import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomDataLifecycleTemplate;
2929
import static org.hamcrest.Matchers.equalTo;
3030
import static org.hamcrest.Matchers.is;
3131
import static org.hamcrest.Matchers.notNullValue;
@@ -39,7 +39,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
3939
}
4040

4141
public void testGetLifecycle() throws Exception {
42-
DataStreamLifecycle.Template lifecycle = randomLifecycleTemplate();
42+
DataStreamLifecycle.Template lifecycle = randomDataLifecycleTemplate();
4343
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
4444
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
4545
{
@@ -226,7 +226,9 @@ public void testPutLifecycle() throws Exception {
226226
}
227227

228228
public void testDeleteLifecycle() throws Exception {
229-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate();
229+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
230+
.dataRetention(randomPositiveTimeValue())
231+
.buildTemplate();
230232
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
231233
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
232234
{

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
204204
Template.builder()
205205
.settings(Settings.EMPTY)
206206
.mappings(mappings)
207-
.lifecycle(DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()))
207+
.lifecycle(DataStreamLifecycle.dataLifecycleBuilder().dataRetention(randomPositiveTimeValue()))
208208
)
209209
.dataStreamTemplate(new DataStreamTemplate())
210210
.build(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ public void testRolloverLifecycle() throws Exception {
178178
}
179179

180180
public void testRolloverAndRetention() throws Exception {
181-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).buildTemplate();
181+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.ZERO).buildTemplate();
182182

183183
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
184184

@@ -321,7 +321,9 @@ public void testOriginationDate() throws Exception {
321321
* days ago, and one with an origination date 1 day ago. After data stream lifecycle runs, we expect the one with the old
322322
* origination date to have been deleted, and the one with the newer origination date to remain.
323323
*/
324-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(7)).buildTemplate();
324+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
325+
.dataRetention(TimeValue.timeValueDays(7))
326+
.buildTemplate();
325327

326328
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
327329

@@ -970,7 +972,7 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception
970972

971973
public void testReenableDataStreamLifecycle() throws Exception {
972974
// start with a lifecycle that's not enabled
973-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().enabled(false).buildTemplate();
975+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder().enabled(false).buildTemplate();
974976

975977
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
976978
String dataStreamName = "metrics-foo";
@@ -1029,7 +1031,7 @@ public void testReenableDataStreamLifecycle() throws Exception {
10291031

10301032
public void testLifecycleAppliedToFailureStore() throws Exception {
10311033
// We configure a lifecycle with downsampling to ensure it doesn't fail
1032-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder()
1034+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
10331035
.dataRetention(TimeValue.timeValueSeconds(20))
10341036
.downsampling(
10351037
List.of(
@@ -1263,7 +1265,8 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
12631265
Template.builder()
12641266
.settings(Settings.EMPTY)
12651267
.lifecycle(
1266-
DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
1268+
DataStreamLifecycle.dataLifecycleBuilder()
1269+
.dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
12671270
)
12681271
)
12691272
.build(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ public void testExplainFailuresLifecycle() throws Exception {
265265
null,
266266
null,
267267
DataStreamLifecycle.Template.DATA_DEFAULT,
268-
new DataStreamOptions.Template(new DataStreamFailureStore.Template(true))
268+
new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate())
269269
);
270270
String dataStreamName = "metrics-foo";
271271
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
@@ -400,7 +400,7 @@ public void testExplainLifecycleForIndicesWithErrors() throws Exception {
400400
null,
401401
null,
402402
lifecycle,
403-
new DataStreamOptions.Template(new DataStreamFailureStore.Template(true))
403+
new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate())
404404
);
405405
String dataStreamName = "metrics-foo";
406406
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
@@ -501,7 +501,7 @@ public void testExplainDataStreamLifecycleForUnmanagedIndices() throws Exception
501501
List.of("metrics-foo*"),
502502
null,
503503
null,
504-
DataStreamLifecycle.builder().enabled(false).buildTemplate()
504+
DataStreamLifecycle.dataLifecycleBuilder().enabled(false).buildTemplate()
505505
);
506506
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
507507
TEST_REQUEST_TIMEOUT,

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
2828
import org.elasticsearch.cluster.metadata.DataStreamFailureStore;
2929
import org.elasticsearch.cluster.metadata.DataStreamOptions;
30-
import org.elasticsearch.cluster.metadata.ResettableValue;
3130
import org.elasticsearch.cluster.metadata.Template;
3231
import org.elasticsearch.common.compress.CompressedXContent;
3332
import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
@@ -271,11 +270,9 @@ private String createDataStream() throws Exception {
271270

272271
private String createDataStream(boolean hidden, boolean failureStore) throws Exception {
273272
String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault());
274-
ResettableValue<DataStreamOptions.Template> failureStoreOptions = failureStore == false
275-
? ResettableValue.undefined()
276-
: ResettableValue.create(
277-
new DataStreamOptions.Template(ResettableValue.create(new DataStreamFailureStore.Template(ResettableValue.create(true))))
278-
);
273+
DataStreamOptions.Template failureStoreOptions = failureStore == false
274+
? null
275+
: new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate());
279276
Template idxTemplate = new Template(null, new CompressedXContent("""
280277
{"properties":{"@timestamp":{"type":"date"},"data":{"type":"keyword"}}}
281278
"""), null, null, failureStoreOptions);

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,11 @@ public void testLifecycleComposition() {
145145
}
146146
// One lifecycle results to this lifecycle as the final
147147
{
148-
DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(true, randomRetention(), randomDownsampling());
148+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate(
149+
true,
150+
randomRetention(),
151+
randomDownsampling()
152+
);
149153
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle);
150154
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
151155
// Defaults to true
@@ -156,7 +160,11 @@ public void testLifecycleComposition() {
156160
// If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones
157161
// Enabled is always true unless it's explicitly set to false
158162
{
159-
DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
163+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate(
164+
false,
165+
randomPositiveTimeValue(),
166+
randomRounds()
167+
);
160168
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DATA_DEFAULT);
161169
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
162170
assertThat(result.enabled(), equalTo(true));
@@ -165,8 +173,16 @@ public void testLifecycleComposition() {
165173
}
166174
// If both lifecycle have all properties, then the latest one overwrites all the others
167175
{
168-
DataStreamLifecycle.Template lifecycle1 = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
169-
DataStreamLifecycle.Template lifecycle2 = new DataStreamLifecycle.Template(true, randomPositiveTimeValue(), randomRounds());
176+
DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.createDataLifecycleTemplate(
177+
false,
178+
randomPositiveTimeValue(),
179+
randomRounds()
180+
);
181+
DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.createDataLifecycleTemplate(
182+
true,
183+
randomPositiveTimeValue(),
184+
randomRounds()
185+
);
170186
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle1, lifecycle2);
171187
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
172188
assertThat(result.enabled(), equalTo(lifecycle2.enabled()));

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
187187
.setGeneration(3)
188188
.setAllowCustomRouting(true)
189189
.setIndexMode(IndexMode.STANDARD)
190-
.setLifecycle(new DataStreamLifecycle(false, null, null))
190+
.setLifecycle(DataStreamLifecycle.createDataLifecycle(false, null, null))
191191
.setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
192192
.setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build())
193193
.build();

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,8 @@ static void putComposableIndexTemplate(
128128
assertTrue(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet().isAcknowledged());
129129
}
130130

131-
static DataStreamLifecycle.Template randomLifecycleTemplate() {
132-
return new DataStreamLifecycle.Template(
131+
static DataStreamLifecycle.Template randomDataLifecycleTemplate() {
132+
return DataStreamLifecycle.createDataLifecycleTemplate(
133133
frequently(),
134134
randomResettable(ESTestCase::randomTimeValue),
135135
randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds)

0 commit comments

Comments
 (0)