From a15bc6231e853d6662600f2eb9eb1d9275e7cdc7 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 23 Apr 2025 13:00:25 +0300 Subject: [PATCH] Create dedicated factory methods for data lifecycle (#126487) The class `DataStreamLifecycle` is currently capturing the lifecycle configuration that currently manages all data stream indices, but soon enough it will be split into two variants, the data and the failures lifecycle. Some pre-work has been done already but as we are progressing in our POC, we see that it will be really useful if the `DataStreamLifecycle` is "aware" of the target index component. This will allow us to correctly apply global retention or to throw an error if a downsampling configuration is provided to a failure lifecycle. In this PR, we perform a small refactoring to reduce the noise in https://github.com/elastic/elasticsearch/pull/125658. Here we introduce the following: - A factory method that creates a data lifecycle, for now it's trivial but it will be more useful soon. - We rename the "empty" builder to explicitly mention the index component it refers to. (cherry picked from commit b9917086e1c1bf4ab1948476164277bdea1f8440) # Conflicts: # modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java # server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java # server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java --- .../datastreams/DataStreamIT.java | 4 +- .../datastreams/SystemDataStreamIT.java | 28 +++++---- .../lifecycle/CrudDataStreamLifecycleIT.java | 8 ++- .../CrudSystemDataStreamLifecycleIT.java | 2 +- .../DataStreamLifecycleServiceIT.java | 13 ++-- .../ExplainDataStreamLifecycleIT.java | 6 +- .../datastreams/DataStreamsStatsTests.java | 9 +-- .../MetadataIndexTemplateServiceTests.java | 24 +++++-- .../action/GetDataStreamsResponseTests.java | 2 +- .../DataStreamLifecycleFixtures.java | 4 +- .../DataStreamLifecycleServiceTests.java | 34 +++++----- ...etDataStreamLifecycleStatsActionTests.java | 4 +- ...DeleteSourceAndAddDownsampleToDSTests.java | 12 ++-- .../PutDataStreamLifecycleAction.java | 2 +- .../cluster/metadata/DataStreamLifecycle.java | 62 +++++++++++++------ .../cluster/metadata/Template.java | 4 +- .../datastreams/GetDataStreamActionTests.java | 2 +- .../ExplainIndexDataStreamLifecycleTests.java | 2 +- .../GetDataStreamLifecycleActionTests.java | 2 +- .../metadata/ComponentTemplateTests.java | 9 +-- .../ComposableIndexTemplateTests.java | 11 ++-- .../DataStreamLifecycleTemplateTests.java | 18 +++--- .../metadata/DataStreamLifecycleTests.java | 26 ++++---- ...amLifecycleWithRetentionWarningsTests.java | 10 +-- .../cluster/metadata/DataStreamTests.java | 24 +++---- .../MetadataDataStreamsServiceTests.java | 2 +- .../MetadataIndexTemplateServiceTests.java | 25 +++++--- .../metadata/DataStreamTestHelper.java | 2 +- ...StreamLifecycleUsageTransportActionIT.java | 4 +- ...taStreamLifecycleFeatureSetUsageTests.java | 6 +- ...StreamLifecycleDownsampleDisruptionIT.java | 2 +- .../DataStreamLifecycleDownsampleIT.java | 8 +-- ...ataStreamAndIndexLifecycleMixingTests.java | 4 +- ...StreamLifecycleDownsamplingSecurityIT.java | 4 +- ...reamLifecycleServiceRuntimeSecurityIT.java | 10 ++- 35 files changed, 225 insertions(+), 164 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index a220182180c9e..c79bb15daeac1 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1372,7 +1372,9 @@ public void testSearchAllResolvesDataStreams() throws Exception { public void testGetDataStream() throws Exception { Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build(); - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate(); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(randomPositiveTimeValue()) + .buildTemplate(); putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false); int numDocsFoo = randomIntBetween(2, 16); indexDocs("metrics-foo", numDocsFoo); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java index 36e109cf113d3..2c7a5862e4371 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/SystemDataStreamIT.java @@ -398,17 +398,23 @@ public Collection getSystemDataStreamDescriptors() { Type.EXTERNAL, ComposableIndexTemplate.builder() .indexPatterns(List.of(".test-failure-store")) - .template(Template.builder().mappings(new CompressedXContent(""" - { - "properties": { - "@timestamp" : { - "type": "date" - }, - "count": { - "type": "long" - } - } - }""")).dataStreamOptions(new DataStreamOptions.Template(new DataStreamFailureStore.Template(true)))) + .template( + Template.builder() + .mappings(new CompressedXContent(""" + { + "properties": { + "@timestamp" : { + "type": "date" + }, + "count": { + "type": "long" + } + } + }""")) + .dataStreamOptions( + new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate()) + ) + ) .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) .build(), Map.of(), diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java index 7ab58163f0a2f..b82ee87952568 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java @@ -25,7 +25,7 @@ import java.util.List; import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.putComposableIndexTemplate; -import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomLifecycleTemplate; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.randomDataLifecycleTemplate; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -39,7 +39,7 @@ protected Collection> nodePlugins() { } public void testGetLifecycle() throws Exception { - DataStreamLifecycle.Template lifecycle = randomLifecycleTemplate(); + DataStreamLifecycle.Template lifecycle = randomDataLifecycleTemplate(); putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle); putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null); { @@ -226,7 +226,9 @@ public void testPutLifecycle() throws Exception { } public void testDeleteLifecycle() throws Exception { - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate(); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(randomPositiveTimeValue()) + .buildTemplate(); putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle); putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null); { diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java index 80e553ac7a08e..0fc04560cf32b 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java @@ -204,7 +204,7 @@ public Collection getSystemDataStreamDescriptors() { Template.builder() .settings(Settings.EMPTY) .mappings(mappings) - .lifecycle(DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue())) + .lifecycle(DataStreamLifecycle.dataLifecycleBuilder().dataRetention(randomPositiveTimeValue())) ) .dataStreamTemplate(new DataStreamTemplate()) .build(), diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index b5472a5b83fe4..f4f7a4abc742e 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -178,7 +178,7 @@ public void testRolloverLifecycle() throws Exception { } public void testRolloverAndRetention() throws Exception { - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).buildTemplate(); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.ZERO).buildTemplate(); putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false); @@ -321,7 +321,9 @@ public void testOriginationDate() throws Exception { * days ago, and one with an origination date 1 day ago. After data stream lifecycle runs, we expect the one with the old * origination date to have been deleted, and the one with the newer origination date to remain. */ - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(7)).buildTemplate(); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(TimeValue.timeValueDays(7)) + .buildTemplate(); putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false); @@ -970,7 +972,7 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception public void testReenableDataStreamLifecycle() throws Exception { // start with a lifecycle that's not enabled - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().enabled(false).buildTemplate(); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder().enabled(false).buildTemplate(); putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false); String dataStreamName = "metrics-foo"; @@ -1029,7 +1031,7 @@ public void testReenableDataStreamLifecycle() throws Exception { public void testLifecycleAppliedToFailureStore() throws Exception { // We configure a lifecycle with downsampling to ensure it doesn't fail - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder() + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(TimeValue.timeValueSeconds(20)) .downsampling( List.of( @@ -1263,7 +1265,8 @@ public Collection getSystemDataStreamDescriptors() { Template.builder() .settings(Settings.EMPTY) .lifecycle( - DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS)) + DataStreamLifecycle.dataLifecycleBuilder() + .dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS)) ) ) .build(), diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java index 7f5fd14fd45d6..87fd46d715329 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java @@ -265,7 +265,7 @@ public void testExplainFailuresLifecycle() throws Exception { null, null, DataStreamLifecycle.Template.DATA_DEFAULT, - new DataStreamOptions.Template(new DataStreamFailureStore.Template(true)) + new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate()) ); String dataStreamName = "metrics-foo"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request( @@ -400,7 +400,7 @@ public void testExplainLifecycleForIndicesWithErrors() throws Exception { null, null, lifecycle, - new DataStreamOptions.Template(new DataStreamFailureStore.Template(true)) + new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate()) ); String dataStreamName = "metrics-foo"; CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request( @@ -501,7 +501,7 @@ public void testExplainDataStreamLifecycleForUnmanagedIndices() throws Exception List.of("metrics-foo*"), null, null, - DataStreamLifecycle.builder().enabled(false).buildTemplate() + DataStreamLifecycle.dataLifecycleBuilder().enabled(false).buildTemplate() ); CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request( TEST_REQUEST_TIMEOUT, diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java index e8fbb46b430ba..6f0f2935449da 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamsStatsTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStreamFailureStore; import org.elasticsearch.cluster.metadata.DataStreamOptions; -import org.elasticsearch.cluster.metadata.ResettableValue; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; @@ -271,11 +270,9 @@ private String createDataStream() throws Exception { private String createDataStream(boolean hidden, boolean failureStore) throws Exception { String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.getDefault()); - ResettableValue failureStoreOptions = failureStore == false - ? ResettableValue.undefined() - : ResettableValue.create( - new DataStreamOptions.Template(ResettableValue.create(new DataStreamFailureStore.Template(ResettableValue.create(true)))) - ); + DataStreamOptions.Template failureStoreOptions = failureStore == false + ? null + : new DataStreamOptions.Template(DataStreamFailureStore.builder().enabled(true).buildTemplate()); Template idxTemplate = new Template(null, new CompressedXContent(""" {"properties":{"@timestamp":{"type":"date"},"data":{"type":"keyword"}}} """), null, null, failureStoreOptions); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index 6b8a63df5f513..05ce8eb26b567 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -145,7 +145,11 @@ public void testLifecycleComposition() { } // One lifecycle results to this lifecycle as the final { - DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(true, randomRetention(), randomDownsampling()); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate( + true, + randomRetention(), + randomDownsampling() + ); List lifecycles = List.of(lifecycle); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); // Defaults to true @@ -156,7 +160,11 @@ public void testLifecycleComposition() { // If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones // Enabled is always true unless it's explicitly set to false { - DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds()); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate( + false, + randomPositiveTimeValue(), + randomRounds() + ); List lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DATA_DEFAULT); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(true)); @@ -165,8 +173,16 @@ public void testLifecycleComposition() { } // If both lifecycle have all properties, then the latest one overwrites all the others { - DataStreamLifecycle.Template lifecycle1 = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds()); - DataStreamLifecycle.Template lifecycle2 = new DataStreamLifecycle.Template(true, randomPositiveTimeValue(), randomRounds()); + DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.createDataLifecycleTemplate( + false, + randomPositiveTimeValue(), + randomRounds() + ); + DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.createDataLifecycleTemplate( + true, + randomPositiveTimeValue(), + randomRounds() + ); List lifecycles = List.of(lifecycle1, lifecycle2); DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(lifecycle2.enabled())); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index fc9719c6329fb..bedce9e0730f8 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -187,7 +187,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti .setGeneration(3) .setAllowCustomRouting(true) .setIndexMode(IndexMode.STANDARD) - .setLifecycle(new DataStreamLifecycle(false, null, null)) + .setLifecycle(DataStreamLifecycle.createDataLifecycle(false, null, null)) .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED) .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build()) .build(); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java index 0da3e5db18e48..59a852220c759 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java @@ -128,8 +128,8 @@ static void putComposableIndexTemplate( assertTrue(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet().isAcknowledged()); } - static DataStreamLifecycle.Template randomLifecycleTemplate() { - return new DataStreamLifecycle.Template( + static DataStreamLifecycle.Template randomDataLifecycleTemplate() { + return DataStreamLifecycle.createDataLifecycleTemplate( frequently(), randomResettable(ESTestCase::randomTimeValue), randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 8d84e476684cf..8bcca57e6d380 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -213,7 +213,7 @@ public void testOperationsExecutedOnce() { numBackingIndices, 2, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.ZERO).build(), now ); builder.put(dataStream); @@ -276,7 +276,7 @@ public void testRetentionNotExecutedDueToAge() { numBackingIndices, numFailureIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(700)).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.timeValueDays(700)).build(), now ); builder.put(dataStream); @@ -309,7 +309,7 @@ public void testRetentionNotExecutedForTSIndicesWithinTimeBounds() { dataStream.copy() .setName(dataStreamName) .setGeneration(dataStream.getGeneration() + 1) - .setLifecycle(DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build()) + .setLifecycle(DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.ZERO).build()) .build() ); clusterState = ClusterState.builder(clusterState).metadata(builder).build(); @@ -346,7 +346,7 @@ public void testMergePolicyNotExecutedForTSIndicesWithinTimeBounds() { dataStream.copy() .setName(dataStreamName) .setGeneration(dataStream.getGeneration() + 1) - .setLifecycle(DataStreamLifecycle.builder().build()) + .setLifecycle(DataStreamLifecycle.dataLifecycleBuilder().build()) .build() ); clusterState = ClusterState.builder(clusterState).metadata(builder).build(); @@ -378,7 +378,7 @@ public void testRetentionSkippedWhilstDownsamplingInProgress() { dataStreamName, numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueMillis(0)).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.timeValueMillis(0)).build(), now ); builder.put(dataStream); @@ -458,7 +458,7 @@ public void testIlmManagedIndicesAreSkipped() { Settings.builder() .put(IndexMetadata.LIFECYCLE_NAME, "ILM_policy") .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.ZERO).build(), now ); builder.put(dataStream); @@ -547,7 +547,7 @@ public void testErrorStoreIsClearedOnBackingIndexBecomingUnmanaged() { dataStreamName, numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(700)).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.timeValueDays(700)).build(), now ); // all backing indices are in the error store @@ -585,7 +585,7 @@ public void testBackingIndicesFromMultipleDataStreamsInErrorStore() { ilmManagedDataStreamName, 3, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(700)).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.timeValueDays(700)).build(), now ); // all backing indices are in the error store @@ -598,7 +598,7 @@ public void testBackingIndicesFromMultipleDataStreamsInErrorStore() { dataStreamWithBackingIndicesInErrorState, 5, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(700)).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.timeValueDays(700)).build(), now ); // put all backing indices in the error store @@ -649,7 +649,7 @@ public void testForceMerge() throws Exception { numBackingIndices, settings(IndexVersion.current()).put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), ONE_HUNDRED_MB) .put(MergePolicyConfig.INDEX_MERGE_POLICY_MERGE_FACTOR_SETTING.getKey(), TARGET_MERGE_FACTOR_VALUE), - DataStreamLifecycle.builder().dataRetention(TimeValue.MAX_VALUE).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(), now ); builder.put(dataStream); @@ -760,7 +760,7 @@ public void testForceMergeRetries() throws Exception { numBackingIndices, settings(IndexVersion.current()).put(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING.getKey(), ONE_HUNDRED_MB) .put(MergePolicyConfig.INDEX_MERGE_POLICY_MERGE_FACTOR_SETTING.getKey(), TARGET_MERGE_FACTOR_VALUE), - DataStreamLifecycle.builder().dataRetention(TimeValue.MAX_VALUE).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(), now ); builder.put(dataStream); @@ -932,7 +932,7 @@ public void testForceMergeDedup() throws Exception { dataStreamName, numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.MAX_VALUE).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(), now ); builder.put(dataStream); @@ -1124,7 +1124,7 @@ public void testMergePolicySettingsAreConfiguredBeforeForcemerge() throws Except dataStreamName, numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.MAX_VALUE).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(), now ); builder.put(dataStream); @@ -1203,7 +1203,7 @@ public void testDownsampling() throws Exception { .put(MergePolicyConfig.INDEX_MERGE_POLICY_MERGE_FACTOR_SETTING.getKey(), TARGET_MERGE_FACTOR_VALUE) .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), - DataStreamLifecycle.builder() + DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) ) @@ -1339,7 +1339,7 @@ public void testDownsamplingWhenTargetIndexNameClashYieldsException() throws Exc .put(MergePolicyConfig.INDEX_MERGE_POLICY_MERGE_FACTOR_SETTING.getKey(), TARGET_MERGE_FACTOR_VALUE) .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), - DataStreamLifecycle.builder() + DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) ) @@ -1526,7 +1526,7 @@ public void testFailureStoreIsManagedEvenWhenDisabled() { numBackingIndices, 2, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.ZERO).build(), now ).copy().setDataStreamOptions(DataStreamOptions.FAILURE_STORE_DISABLED).build(); // failure store is managed even when disabled builder.put(dataStream); @@ -1589,7 +1589,7 @@ private ClusterState downsampleSetup(String dataStreamName, IndexMetadata.Downsa 2, settings(IndexVersion.current()).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) .put("index.routing_path", "@timestamp"), - DataStreamLifecycle.builder() + DataStreamLifecycle.dataLifecycleBuilder() .downsampling( List.of(new DownsamplingRound(TimeValue.timeValueMillis(0), new DownsampleConfig(new DateHistogramInterval("5m")))) ) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java index 3be1be12487b9..89bcb36a92030 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java @@ -94,7 +94,7 @@ public void testMixedDataStreams() { "dsl-managed-index", numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(10)).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.timeValueDays(10)).build(), Clock.systemUTC().millis() ); indicesInError.add(dslDataStream.getIndices().get(randomInt(numBackingIndices - 1)).getName()); @@ -130,7 +130,7 @@ public void testMixedDataStreams() { IndexMetadata indexMetadata = indexMetaBuilder.build(); builder.put(indexMetadata, false); backingIndices.add(indexMetadata.getIndex()); - builder.put(newInstance(dataStreamName, backingIndices, 3, null, false, DataStreamLifecycle.builder().build())); + builder.put(newInstance(dataStreamName, backingIndices, 3, null, false, DataStreamLifecycle.dataLifecycleBuilder().build())); } ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(builder).build(); when(errorStore.getAllIndices()).thenReturn(indicesInError); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDSTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDSTests.java index dbd904ca3926a..4c359c8eaf037 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDSTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/downsampling/DeleteSourceAndAddDownsampleToDSTests.java @@ -52,7 +52,7 @@ public void testDownsampleIndexMissingIsNoOp() { dataStreamName, numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.MAX_VALUE).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(), now ); builder.put(dataStream); @@ -79,7 +79,7 @@ public void testDownsampleIsAddedToDSEvenIfSourceDeleted() { dataStreamName, numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.MAX_VALUE).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(), now ); String firstGenIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1); @@ -115,7 +115,7 @@ public void testSourceIndexIsWriteIndexThrowsException() { dataStreamName, numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.MAX_VALUE).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(), now ); builder.put(dataStream); @@ -150,7 +150,7 @@ public void testSourceIsDeleteAndDownsampleOriginationDateIsConfigured() { dataStreamName, numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.MAX_VALUE).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(), now ); String firstGenIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1); @@ -203,7 +203,7 @@ public void testSourceWithoutLifecycleMetaAndDestWithOriginationDateAlreadyConfi dataStreamName, numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.MAX_VALUE).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(), now ); String firstGenIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1); @@ -245,7 +245,7 @@ public void testSourceIndexIsDeleteEvenIfNotPartOfDSAnymore() { dataStreamName, numBackingIndices, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(TimeValue.MAX_VALUE).build(), + DataStreamLifecycle.dataLifecycleBuilder().dataRetention(TimeValue.MAX_VALUE).build(), now ); String firstGenIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java index 1dbfb97851aa6..45b71fd63cb16 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java @@ -149,7 +149,7 @@ public Request( ) { super(masterNodeTimeout, ackTimeout); this.names = names; - this.lifecycle = DataStreamLifecycle.builder() + this.lifecycle = DataStreamLifecycle.dataLifecycleBuilder() .dataRetention(dataRetention) .enabled(enabled == null || enabled) .downsampling(downsampling) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index ab63c57a66271..971a65e963e6c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -86,7 +86,7 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { Setting.Property.NodeScope ); - public static final DataStreamLifecycle DEFAULT_DATA_LIFECYCLE = new DataStreamLifecycle(null, null, null); + public static final DataStreamLifecycle DEFAULT_DATA_LIFECYCLE = DataStreamLifecycle.createDataLifecycle(null, null, null); public static final String DATA_STREAM_LIFECYCLE_ORIGIN = "data_stream_lifecycle"; @@ -132,17 +132,27 @@ public static boolean isDataStreamsLifecycleOnlyMode(final Settings settings) { @Nullable private final List downsampling; - public DataStreamLifecycle( - @Nullable Boolean enabled, - @Nullable TimeValue dataRetention, - @Nullable List downsampling - ) { + // Visible for testing, preferably use the factory methods that are specialised by lifecycle type, + // for example for the data component. + DataStreamLifecycle(@Nullable Boolean enabled, @Nullable TimeValue dataRetention, @Nullable List downsampling) { this.enabled = enabled == null || enabled; this.dataRetention = dataRetention; DownsamplingRound.validateRounds(downsampling); this.downsampling = downsampling; } + /** + * This factory method creates a lifecycle applicable for the data index component of a data stream. This + * means it supports all configuration applicable for backing indices. + */ + public static DataStreamLifecycle createDataLifecycle( + @Nullable Boolean enabled, + @Nullable TimeValue dataRetention, + @Nullable List downsampling + ) { + return new DataStreamLifecycle(enabled, dataRetention, downsampling); + } + /** * Returns true, if this data stream lifecycle configuration is enabled, false otherwise */ @@ -538,6 +548,22 @@ public String toString() { } } + public static Template createDataLifecycleTemplate( + boolean enabled, + TimeValue dataRetention, + List downsampling + ) { + return new Template(enabled, ResettableValue.create(dataRetention), ResettableValue.create(downsampling)); + } + + public static Template createDataLifecycleTemplate( + boolean enabled, + ResettableValue dataRetention, + ResettableValue> downsampling + ) { + return new Template(enabled, dataRetention, downsampling); + } + /** * Represents the template configuration of a lifecycle. It supports explicitly resettable values * to allow value reset during template composition. @@ -548,7 +574,7 @@ public record Template( ResettableValue> downsampling ) implements ToXContentObject, Writeable { - public Template(boolean enabled, TimeValue dataRetention, List downsampling) { + Template(boolean enabled, TimeValue dataRetention, List downsampling) { this(enabled, ResettableValue.create(dataRetention), ResettableValue.create(downsampling)); } @@ -728,8 +754,8 @@ public static Builder builder(Template template) { return new Builder(template); } - public static Builder builder() { - return new Builder((DataStreamLifecycle) null); + public static Builder dataLifecycleBuilder() { + return new Builder(); } /** @@ -742,20 +768,18 @@ public static class Builder { @Nullable private List downsampling = null; + private Builder() {} + private Builder(DataStreamLifecycle.Template template) { - if (template != null) { - enabled = template.enabled(); - dataRetention = template.dataRetention().get(); - downsampling = template.downsampling().get(); - } + enabled = template.enabled(); + dataRetention = template.dataRetention().get(); + downsampling = template.downsampling().get(); } private Builder(DataStreamLifecycle lifecycle) { - if (lifecycle != null) { - enabled = lifecycle.enabled(); - dataRetention = lifecycle.dataRetention(); - downsampling = lifecycle.downsampling(); - } + enabled = lifecycle.enabled(); + dataRetention = lifecycle.dataRetention(); + downsampling = lifecycle.downsampling(); } public Builder composeTemplate(DataStreamLifecycle.Template template) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java index 6f413cbbedece..36c470d17d9a0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java @@ -61,7 +61,9 @@ public class Template implements SimpleDiffable