diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index e16d34d597759..125d3fedec60b 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1384,7 +1384,7 @@ public void testSearchAllResolvesDataStreams() throws Exception { public void testGetDataStream() throws Exception { Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build(); - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build(); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate(); putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false); int numDocsFoo = randomIntBetween(2, 16); indexDocs("metrics-foo", numDocsFoo); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java index d6edd671131e9..7ab58163f0a2f 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java @@ -226,7 +226,7 @@ public void testPutLifecycle() throws Exception { } public void testDeleteLifecycle() throws Exception { - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build(); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate(); putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle); putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null); { diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java index b6c5b554a6451..aeba15563b991 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java @@ -207,7 +207,7 @@ public Collection getSystemDataStreamDescriptors() { Template.builder() .settings(Settings.EMPTY) .mappings(mappings) - .lifecycle(DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build()) + .lifecycle(DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue())) ) .dataStreamTemplate(new DataStreamTemplate()) .build(), diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index b2b5563059930..04f1d73c09b44 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -179,7 +179,7 @@ public void testRolloverLifecycle() throws Exception { } public void testRolloverAndRetention() throws Exception { - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.ZERO).build(); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).buildTemplate(); putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false); @@ -322,7 +322,7 @@ public void testOriginationDate() throws Exception { * days ago, and one with an origination date 1 day ago. After data stream lifecycle runs, we expect the one with the old * origination date to have been deleted, and the one with the newer origination date to remain. */ - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.timeValueDays(7)).build(); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(7)).buildTemplate(); putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false); @@ -974,7 +974,7 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception public void testReenableDataStreamLifecycle() throws Exception { // start with a lifecycle that's not enabled - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().enabled(false).build(); + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().enabled(false).buildTemplate(); putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false); String dataStreamName = "metrics-foo"; @@ -1033,7 +1033,7 @@ public void testReenableDataStreamLifecycle() throws Exception { public void testLifecycleAppliedToFailureStore() throws Exception { // We configure a lifecycle with downsampling to ensure it doesn't fail - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder() + DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder() .dataRetention(TimeValue.timeValueSeconds(20)) .downsampling( List.of( @@ -1043,7 +1043,7 @@ public void testLifecycleAppliedToFailureStore() throws Exception { ) ) ) - .build(); + .buildTemplate(); putComposableIndexTemplate("id1", """ { @@ -1268,8 +1268,7 @@ public Collection getSystemDataStreamDescriptors() { Template.builder() .settings(Settings.EMPTY) .lifecycle( - DataStreamLifecycle.Template.builder() - .dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS)) + DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS)) ) ) .build(), diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java index 62b68b2cc69bb..75b268f6ffd2f 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java @@ -374,7 +374,7 @@ public void testExplainDataStreamLifecycleForUnmanagedIndices() throws Exception List.of("metrics-foo*"), null, null, - DataStreamLifecycle.Template.builder().enabled(false).build() + DataStreamLifecycle.builder().enabled(false).buildTemplate() ); CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request( TEST_REQUEST_TIMEOUT, diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index 23eb8e87b6622..835cff8a081fa 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -144,12 +144,9 @@ public void testLifecycleComposition() { } // One lifecycle results to this lifecycle as the final { - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder() - .dataRetention(randomRetention()) - .downsampling(randomDownsampling()) - .build(); + DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(true, randomRetention(), randomDownsampling()); List lifecycles = List.of(lifecycle); - DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle(); + DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); // Defaults to true assertThat(result.enabled(), equalTo(true)); assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get())); @@ -158,31 +155,19 @@ public void testLifecycleComposition() { // If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones // Enabled is always true unless it's explicitly set to false { - DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder() - .enabled(false) - .dataRetention(randomPositiveTimeValue()) - .downsampling(randomRounds()) - .build(); + DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds()); List lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DEFAULT); - DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle(); + DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(true)); assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get())); assertThat(result.downsampling(), equalTo(lifecycle.downsampling().get())); } // If both lifecycle have all properties, then the latest one overwrites all the others { - DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.Template.builder() - .enabled(false) - .dataRetention(randomPositiveTimeValue()) - .downsampling(randomRounds()) - .build(); - DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.Template.builder() - .enabled(true) - .dataRetention(randomPositiveTimeValue()) - .downsampling(randomRounds()) - .build(); + DataStreamLifecycle.Template lifecycle1 = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds()); + DataStreamLifecycle.Template lifecycle2 = new DataStreamLifecycle.Template(true, randomPositiveTimeValue(), randomRounds()); List lifecycles = List.of(lifecycle1, lifecycle2); - DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle(); + DataStreamLifecycle result = composeDataLifecycles(lifecycles).build(); assertThat(result.enabled(), equalTo(lifecycle2.enabled())); assertThat(result.dataRetention(), equalTo(lifecycle2.dataRetention().get())); assertThat(result.downsampling(), equalTo(lifecycle2.downsampling().get())); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java index cdcb4dbcad930..0c64dba41c894 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java @@ -171,11 +171,11 @@ static void putComposableIndexTemplate( } static DataStreamLifecycle.Template randomLifecycleTemplate() { - return DataStreamLifecycle.Template.builder() - .dataRetention(randomResettable(ESTestCase::randomTimeValue)) - .downsampling(randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds)) - .enabled(frequently()) - .build(); + return new DataStreamLifecycle.Template( + frequently(), + randomResettable(ESTestCase::randomTimeValue), + randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds) + ); } private static ResettableValue randomResettable(Supplier supplier) { diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 2598735f270d5..05fc75c145286 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -205,7 +205,7 @@ public void testOperationsExecutedOnce() { numBackingIndices, 2, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(0).build(), + DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(), now ); builder.put(dataStream); @@ -301,7 +301,7 @@ public void testRetentionNotExecutedForTSIndicesWithinTimeBounds() { dataStream.copy() .setName(dataStreamName) .setGeneration(dataStream.getGeneration() + 1) - .setLifecycle(DataStreamLifecycle.builder().dataRetention(0L).build()) + .setLifecycle(DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build()) .build() ); clusterState = ClusterState.builder(clusterState).metadata(builder).build(); @@ -453,7 +453,7 @@ public void testIlmManagedIndicesAreSkipped() { Settings.builder() .put(IndexMetadata.LIFECYCLE_NAME, "ILM_policy") .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(0).build(), + DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(), now ); builder.put(dataStream); @@ -1555,7 +1555,7 @@ public void testFailureStoreIsManagedEvenWhenDisabled() { numBackingIndices, 2, settings(IndexVersion.current()), - DataStreamLifecycle.builder().dataRetention(0).build(), + DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(), now ).copy().setDataStreamOptions(DataStreamOptions.FAILURE_STORE_DISABLED).build(); // failure store is managed even when disabled builder.put(dataStream); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 2299975949498..81fceefc93830 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; @@ -344,16 +345,18 @@ public static Template resolveTemplate( ); Settings settings = Settings.builder().put(additionalSettings.build()).put(templateSettings).build(); - DataStreamLifecycle.Template lifecycle = resolveLifecycle(simulatedProject, matchingTemplate); + DataStreamLifecycle.Builder lifecycleBuilder = resolveLifecycle(simulatedProject, matchingTemplate); + DataStreamLifecycle.Template lifecycle = lifecycleBuilder == null ? null : lifecycleBuilder.buildTemplate(); if (template.getDataStreamTemplate() != null && lifecycle == null && isDslOnlyMode) { lifecycle = DataStreamLifecycle.Template.DEFAULT; } + DataStreamOptions.Builder optionsBuilder = resolveDataStreamOptions(simulatedProject, matchingTemplate); return new Template( settings, mergedMapping, aliasesByName, lifecycle, - resolveDataStreamOptions(simulatedProject, matchingTemplate) + optionsBuilder == null ? null : optionsBuilder.buildTemplate() ); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index bbdc4d2734d5c..5002628c083b4 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -680,12 +680,12 @@ private Boolean resolveFailureStoreFromTemplate(String indexName, ProjectMetadat ComposableIndexTemplate composableIndexTemplate = projectMetadata.templatesV2().get(template); if (composableIndexTemplate.getDataStreamTemplate() != null) { // Check if the data stream has the failure store enabled - DataStreamOptions dataStreamOptions = MetadataIndexTemplateService.resolveDataStreamOptions( + DataStreamOptions.Builder dataStreamOptionsBuilder = MetadataIndexTemplateService.resolveDataStreamOptions( composableIndexTemplate, projectMetadata.componentTemplates() - ).mapAndGet(DataStreamOptions.Template::toDataStreamOptions); + ); return DataStream.isFailureStoreEffectivelyEnabled( - dataStreamOptions, + dataStreamOptionsBuilder == null ? null : dataStreamOptionsBuilder.build(), dataStreamFailureStoreSettings, IndexNameExpressionResolver.resolveDateMathExpression(indexName, epochMillis), systemIndices diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java index 5a6217eea8f7b..f5ae5747e088c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java @@ -89,8 +89,7 @@ public static DataStreamFailureStore fromXContent(XContentParser parser) throws /** * This class is only used in template configuration. It wraps the fields of {@link DataStreamFailureStore} with {@link ResettableValue} - * to allow a user to signal when they want to reset any previously encountered values during template composition. Furthermore, it - * provides the method {@link #merge(Template, Template)} that dictates how two templates can be composed. + * to allow a user to signal when they want to reset any previously encountered values during template composition. */ public record Template(ResettableValue enabled) implements Writeable, ToXContentObject { @@ -112,6 +111,10 @@ public record Template(ResettableValue enabled) implements Writeable, T ); } + public Template(Boolean enabled) { + this(ResettableValue.create(enabled)); + } + public Template { if (enabled.get() == null) { throw new IllegalArgumentException("Failure store configuration should have at least one non-null configuration value."); @@ -144,15 +147,6 @@ public static Template fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } - /** - * Returns a template which has the value of the initial template updated with the values of the update. - * Note: for now it's a trivial composition because we have only one non-null field. - * @return the composed template - */ - public static Template merge(Template ignored, Template update) { - return update; - } - public DataStreamFailureStore toFailureStore() { return new DataStreamFailureStore(enabled.get()); } @@ -162,4 +156,65 @@ public String toString() { return Strings.toString(this, true, true); } } + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(Template template) { + return new Builder(template); + } + + public static Builder builder(DataStreamFailureStore failureStore) { + return new Builder(failureStore); + } + + /** + * Builder that is able to create either a DataStreamFailureStore or its respective Template. + * Furthermore, its update methods can be used to compose templates. + */ + public static class Builder { + private Boolean enabled = null; + + private Builder() {} + + private Builder(Template template) { + if (template != null) { + enabled = template.enabled.get(); + } + } + + private Builder(DataStreamFailureStore failureStore) { + if (failureStore != null) { + enabled = failureStore.enabled; + } + } + + public Builder enabled(Boolean enabled) { + this.enabled = enabled; + return this; + } + + public Builder enabled(ResettableValue enabled) { + if (enabled.shouldReset()) { + this.enabled = null; + } else if (enabled.isDefined()) { + this.enabled = enabled.get(); + } + return this; + } + + public Builder composeTemplate(DataStreamFailureStore.Template failureStore) { + this.enabled(failureStore.enabled()); + return this; + } + + public DataStreamFailureStore build() { + return new DataStreamFailureStore(enabled); + } + + public DataStreamFailureStore.Template buildTemplate() { + return new Template(enabled); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index 6860211b6ad46..1f408b191bf1b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -430,57 +430,6 @@ public static ToXContent.Params addEffectiveRetentionParams(ToXContent.Params pa return new DelegatingMapParams(INCLUDE_EFFECTIVE_RETENTION_PARAMS, params); } - public static Builder builder(DataStreamLifecycle lifecycle) { - return new Builder(lifecycle); - } - - public static Builder builder() { - return new Builder(null); - } - - /** - * This builder helps during the composition of the data stream lifecycle templates. - */ - public static class Builder { - private boolean enabled = true; - @Nullable - private TimeValue dataRetention = null; - @Nullable - private List downsampling = null; - - private Builder(@Nullable DataStreamLifecycle lifecycle) { - if (lifecycle != null) { - enabled = lifecycle.enabled; - dataRetention = lifecycle.dataRetention; - downsampling = lifecycle.downsampling; - } - } - - public Builder enabled(boolean value) { - enabled = value; - return this; - } - - public Builder dataRetention(@Nullable TimeValue value) { - dataRetention = value; - return this; - } - - public Builder dataRetention(long value) { - dataRetention = TimeValue.timeValueMillis(value); - return this; - } - - public Builder downsampling(@Nullable List rounds) { - downsampling = rounds; - return this; - } - - public DataStreamLifecycle build() { - return new DataStreamLifecycle(enabled, dataRetention, downsampling); - } - } - /** * This enum represents all configuration sources that can influence the retention of a data stream. */ @@ -603,6 +552,10 @@ public record Template( ResettableValue> downsampling ) implements ToXContentObject, Writeable { + public Template(boolean enabled, TimeValue dataRetention, List downsampling) { + this(enabled, ResettableValue.create(dataRetention), ResettableValue.create(downsampling)); + } + public Template { if (downsampling.isDefined() && downsampling.get() != null) { DownsamplingRound.validateRounds(downsampling.get()); @@ -761,64 +714,96 @@ public XContentBuilder toXContent( return builder; } - public static Builder builder(DataStreamLifecycle.Template template) { - return new Builder(template); + public DataStreamLifecycle toDataStreamLifecycle() { + return new DataStreamLifecycle(enabled, dataRetention.get(), downsampling.get()); } - public static Builder builder() { - return new Builder(null); + @Override + public String toString() { + return Strings.toString(this, true, true); } + } - public static class Builder { - private boolean enabled = true; - private ResettableValue dataRetention = ResettableValue.undefined(); - private ResettableValue> downsampling = ResettableValue.undefined(); + public static Builder builder(DataStreamLifecycle lifecycle) { + return new Builder(lifecycle); + } - private Builder(Template template) { - if (template != null) { - enabled = template.enabled(); - dataRetention = template.dataRetention(); - downsampling = template.downsampling(); - } - } + public static Builder builder(Template template) { + return new Builder(template); + } - public Builder enabled(boolean enabled) { - this.enabled = enabled; - return this; - } + public static Builder builder() { + return new Builder((DataStreamLifecycle) null); + } - public Builder dataRetention(ResettableValue dataRetention) { - this.dataRetention = dataRetention; - return this; - } + /** + * Builds and composes the data stream lifecycle or the respective template. + */ + public static class Builder { + private boolean enabled = true; + @Nullable + private TimeValue dataRetention = null; + @Nullable + private List downsampling = null; - public Builder dataRetention(@Nullable TimeValue dataRetention) { - this.dataRetention = ResettableValue.create(dataRetention); - return this; + private Builder(DataStreamLifecycle.Template template) { + if (template != null) { + enabled = template.enabled(); + dataRetention = template.dataRetention().get(); + downsampling = template.downsampling().get(); } + } - public Builder downsampling(ResettableValue> downsampling) { - this.downsampling = downsampling; - return this; + private Builder(DataStreamLifecycle lifecycle) { + if (lifecycle != null) { + enabled = lifecycle.enabled(); + dataRetention = lifecycle.dataRetention(); + downsampling = lifecycle.downsampling(); } + } + + public Builder composeTemplate(DataStreamLifecycle.Template template) { + enabled(template.enabled()); + dataRetention(template.dataRetention()); + downsampling(template.downsampling()); + return this; + } + + public Builder enabled(boolean enabled) { + this.enabled = enabled; + return this; + } - public Builder downsampling(@Nullable List downsampling) { - this.downsampling = ResettableValue.create(downsampling); - return this; + public Builder dataRetention(ResettableValue dataRetention) { + if (dataRetention.isDefined()) { + this.dataRetention = dataRetention.get(); } + return this; + } - public Template build() { - return new Template(enabled, dataRetention, downsampling); + public Builder dataRetention(@Nullable TimeValue dataRetention) { + this.dataRetention = dataRetention; + return this; + } + + public Builder downsampling(ResettableValue> downsampling) { + if (downsampling.isDefined()) { + this.downsampling = downsampling.get(); } + return this; } - public DataStreamLifecycle toDataStreamLifecycle() { - return new DataStreamLifecycle(enabled, dataRetention.get(), downsampling.get()); + public Builder downsampling(@Nullable List downsampling) { + this.downsampling = downsampling; + return this; } - @Override - public String toString() { - return Strings.toString(this, true, true); + public DataStreamLifecycle build() { + return new DataStreamLifecycle(enabled, dataRetention, downsampling); + } + + public Template buildTemplate() { + return new Template(enabled, dataRetention, downsampling); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java index 423b698442581..645924a54a7d8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamOptions.java @@ -96,8 +96,7 @@ public static DataStreamOptions fromXContent(XContentParser parser) throws IOExc /** * This class is only used in template configuration. It wraps the fields of {@link DataStreamOptions} with {@link ResettableValue} - * to allow a user to signal when they want to reset any previously encountered values during template composition. Furthermore, it - * provides the {@link Template.Builder} that dictates how two templates can be composed. + * to allow a user to signal when they want to reset any previously encountered values during template composition. */ public record Template(ResettableValue failureStore) implements Writeable, ToXContentObject { public static final Template EMPTY = new Template(ResettableValue.undefined()); @@ -120,6 +119,10 @@ public record Template(ResettableValue failureS ); } + public Template(DataStreamFailureStore.Template template) { + this(ResettableValue.create(template)); + } + public Template { assert failureStore != null : "Template does not accept null values, please use Resettable.undefined()"; } @@ -150,43 +153,65 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } - public DataStreamOptions toDataStreamOptions() { - return new DataStreamOptions(failureStore.mapAndGet(DataStreamFailureStore.Template::toFailureStore)); + @Override + public String toString() { + return Strings.toString(this, true, true); } + } - public static Builder builder(Template template) { - return new Builder(template); + public static Builder builder(Template template) { + return new Builder(template); + } + + /** + * Builds and composes the data stream options or the respective template. + */ + public static class Builder { + private DataStreamFailureStore.Builder failureStore = null; + + public Builder(Template template) { + if (template != null && template.failureStore().get() != null) { + failureStore = DataStreamFailureStore.builder(template.failureStore().get()); + } + } + + public Builder(DataStreamOptions options) { + if (options != null && options.failureStore() != null) { + failureStore = DataStreamFailureStore.builder(options.failureStore()); + } } /** - * Builds and composes a data stream template. + * Updates this builder with the values of the provided template. This is not a replacement necessarily, the + * inner values will be merged. */ - public static class Builder { - private ResettableValue failureStore = ResettableValue.undefined(); + public Builder composeTemplate(DataStreamOptions.Template options) { + return failureStore(options.failureStore()); + } - public Builder(Template template) { - if (template != null) { - failureStore = template.failureStore(); + /** + * Updates the current failure store configuration with the provided value. This is not a replacement necessarily, if both + * instance contain data the configurations are merged. + */ + public Builder failureStore(ResettableValue newFailureStore) { + if (newFailureStore.shouldReset()) { + failureStore = null; + } else if (newFailureStore.isDefined()) { + if (failureStore == null) { + failureStore = DataStreamFailureStore.builder(newFailureStore.get()); + } else { + failureStore.composeTemplate(newFailureStore.get()); } } + return this; + } - /** - * Updates the current failure store configuration with the provided value. This is not a replacement necessarily, if both - * instance contain data the configurations are merged. - */ - public Builder updateFailureStore(ResettableValue newFailureStore) { - failureStore = ResettableValue.merge(failureStore, newFailureStore, DataStreamFailureStore.Template::merge); - return this; - } - - public Template build() { - return new Template(failureStore); - } + public Template buildTemplate() { + return new Template(failureStore == null ? null : failureStore.buildTemplate()); } - @Override - public String toString() { - return Strings.toString(this, true, true); + public DataStreamOptions build() { + return new DataStreamOptions(failureStore == null ? null : failureStore.build()); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index c00ee039eeafa..46e5aa4ec1a38 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -265,10 +265,12 @@ static ClusterState createDataStream( // This is not a problem as both have different prefixes (`.ds-` vs `.fs-`) and both will be using the same `generation` field // when rolling over in the future. final long initialGeneration = 1; - ResettableValue dataStreamOptionsTemplate = isSystem - ? MetadataIndexTemplateService.resolveDataStreamOptions(template, systemDataStreamDescriptor.getComponentTemplates()) - : MetadataIndexTemplateService.resolveDataStreamOptions(template, currentProject.componentTemplates()); - final DataStreamOptions dataStreamOptions = dataStreamOptionsTemplate.mapAndGet(DataStreamOptions.Template::toDataStreamOptions); + final DataStreamOptions dataStreamOptions = resolveDataStreamOptions( + currentProject, + systemDataStreamDescriptor, + template, + isSystem + ); // If we need to create a failure store, do so first. Do not reroute during the creation since we will do // that as part of creating the backing index if required. N.B. This is done if initializeFailureStore, @@ -325,10 +327,7 @@ static ClusterState createDataStream( dsBackingIndices.add(writeIndex.getIndex()); boolean hidden = isSystem || template.getDataStreamTemplate().isHidden(); final IndexMode indexMode = newProject.retrieveIndexModeFromTemplate(template); - final DataStreamLifecycle.Template lifecycleTemplate = isSystem - ? MetadataIndexTemplateService.resolveLifecycle(template, systemDataStreamDescriptor.getComponentTemplates()) - : MetadataIndexTemplateService.resolveLifecycle(template, newProject.componentTemplates()); - final DataStreamLifecycle lifecycle = lifecycleTemplate == null ? null : lifecycleTemplate.toDataStreamLifecycle(); + final DataStreamLifecycle lifecycle = resolveDataStreamLifecycle(currentProject, systemDataStreamDescriptor, template, isSystem); List failureIndices = failureStoreIndex == null ? List.of() : List.of(failureStoreIndex.getIndex()); DataStream newDataStream = new DataStream( dataStreamName, @@ -489,4 +488,28 @@ public static void validateTimestampFieldMapping(MappingLookup mappingLookup) th // Sanity check (this validation logic should already have been executed when merging mappings): fieldMapper.validate(mappingLookup); } + + private static DataStreamOptions resolveDataStreamOptions( + ProjectMetadata project, + SystemDataStreamDescriptor systemDataStreamDescriptor, + ComposableIndexTemplate template, + boolean isSystem + ) { + DataStreamOptions.Builder builder = isSystem + ? MetadataIndexTemplateService.resolveDataStreamOptions(template, systemDataStreamDescriptor.getComponentTemplates()) + : MetadataIndexTemplateService.resolveDataStreamOptions(template, project.componentTemplates()); + return builder == null ? null : builder.build(); + } + + private static DataStreamLifecycle resolveDataStreamLifecycle( + ProjectMetadata project, + SystemDataStreamDescriptor systemDataStreamDescriptor, + ComposableIndexTemplate template, + boolean isSystem + ) { + DataStreamLifecycle.Builder builder = isSystem + ? MetadataIndexTemplateService.resolveLifecycle(template, systemDataStreamDescriptor.getComponentTemplates()) + : MetadataIndexTemplateService.resolveLifecycle(template, project.componentTemplates()); + return builder == null ? null : builder.build(); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 2260c23edb659..4e92f87f990c1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -819,8 +819,8 @@ static void validateLifecycle( ComposableIndexTemplate template, @Nullable DataStreamGlobalRetention globalRetention ) { - DataStreamLifecycle.Template lifecycle = resolveLifecycle(template, project.componentTemplates()); - if (lifecycle != null) { + DataStreamLifecycle.Builder builder = resolveLifecycle(template, project.componentTemplates()); + if (builder != null) { if (template.getDataStreamTemplate() == null) { throw new IllegalArgumentException( "index template [" @@ -832,18 +832,15 @@ static void validateLifecycle( // We cannot know for sure if the template will apply to internal data streams, so we use a simpler heuristic: // If all the index patterns start with a dot, we consider that all the connected data streams are internal. boolean isInternalDataStream = template.indexPatterns().stream().allMatch(indexPattern -> indexPattern.charAt(0) == '.'); - lifecycle.toDataStreamLifecycle().addWarningHeaderIfDataRetentionNotEffective(globalRetention, isInternalDataStream); + builder.build().addWarningHeaderIfDataRetentionNotEffective(globalRetention, isInternalDataStream); } } } // Visible for testing static void validateDataStreamOptions(ProjectMetadata projectMetadata, String indexTemplateName, ComposableIndexTemplate template) { - ResettableValue dataStreamOptions = resolveDataStreamOptions( - template, - projectMetadata.componentTemplates() - ); - if (dataStreamOptions.get() != null) { + DataStreamOptions.Builder dataStreamOptionsBuilder = resolveDataStreamOptions(template, projectMetadata.componentTemplates()); + if (dataStreamOptionsBuilder != null) { if (template.getDataStreamTemplate() == null) { throw new IllegalArgumentException( "index template [" @@ -1623,7 +1620,7 @@ static List> resolveAliases( * Resolve the given v2 template into a {@link DataStreamLifecycle} object */ @Nullable - public static DataStreamLifecycle.Template resolveLifecycle(ProjectMetadata metadata, final String templateName) { + public static DataStreamLifecycle.Builder resolveLifecycle(ProjectMetadata metadata, final String templateName) { final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); assert template != null : "attempted to resolve lifecycle for a template [" + templateName + "] that did not exist in the cluster state"; @@ -1637,7 +1634,7 @@ public static DataStreamLifecycle.Template resolveLifecycle(ProjectMetadata meta * Resolve the provided v2 template and component templates into a {@link DataStreamLifecycle} object */ @Nullable - public static DataStreamLifecycle.Template resolveLifecycle( + public static DataStreamLifecycle.Builder resolveLifecycle( ComposableIndexTemplate template, Map componentTemplates ) { @@ -1697,47 +1694,42 @@ public static DataStreamLifecycle.Template resolveLifecycle( * The result will be { "lifecycle": { "enabled": true, "data_retention" : "10d"} } because the latest lifecycle does not have any * information on retention. * @param lifecycles a sorted list of lifecycles in the order that they will be composed - * @return the final lifecycle + * @return the builder that will build the final lifecycle or the template */ @Nullable - public static DataStreamLifecycle.Template composeDataLifecycles(List lifecycles) { - DataStreamLifecycle.Template.Builder builder = null; + public static DataStreamLifecycle.Builder composeDataLifecycles(List lifecycles) { + DataStreamLifecycle.Builder builder = null; for (DataStreamLifecycle.Template current : lifecycles) { if (builder == null) { - builder = DataStreamLifecycle.Template.builder(current); + builder = DataStreamLifecycle.builder(current); } else { - builder.enabled(current.enabled()); - if (current.dataRetention().isDefined()) { - builder.dataRetention(current.dataRetention()); - } - if (current.downsampling().isDefined()) { - builder.downsampling(current.downsampling()); - } + builder.composeTemplate(current); } } - return builder == null ? null : builder.build(); + return builder; } /** - * Resolve the given v2 template into a {@link ResettableValue} object + * Resolve the given v2 template into a {@link DataStreamOptions.Builder} object that can be built to either a + * {@link DataStreamOptions} or the equivalent {@link DataStreamOptions.Template}. */ - public static ResettableValue resolveDataStreamOptions( - final ProjectMetadata projectMetadata, - final String templateName - ) { + @Nullable + public static DataStreamOptions.Builder resolveDataStreamOptions(final ProjectMetadata projectMetadata, final String templateName) { final ComposableIndexTemplate template = projectMetadata.templatesV2().get(templateName); assert template != null : "attempted to resolve data stream options for a template [" + templateName + "] that did not exist in the cluster state"; if (template == null) { - return ResettableValue.undefined(); + return null; } return resolveDataStreamOptions(template, projectMetadata.componentTemplates()); } /** - * Resolve the provided v2 template and component templates into a {@link ResettableValue} object + * Resolve the provided v2 template and component templates into a {@link DataStreamOptions.Builder} object that can be built to + * either a {@link DataStreamOptions} or the equivalent {@link DataStreamOptions.Template}. */ - public static ResettableValue resolveDataStreamOptions( + @Nullable + public static DataStreamOptions.Builder resolveDataStreamOptions( ComposableIndexTemplate template, Map componentTemplates ) { @@ -1764,19 +1756,18 @@ public static ResettableValue resolveDataStreamOptio } /** - * This method composes a series of data streams options to a final one. Since currently the data stream options - * contains only the failure store configuration which also contains only one field, the composition is a bit trivial. - * But we introduce the mechanics that will help extend it really easily. + * This method composes a series of data streams options to a final one. * @param dataStreamOptionsList a sorted list of data stream options in the order that they will be composed * @return the final data stream option configuration */ - public static ResettableValue composeDataStreamOptions( + @Nullable + public static DataStreamOptions.Builder composeDataStreamOptions( List> dataStreamOptionsList ) { if (dataStreamOptionsList.isEmpty()) { - return ResettableValue.undefined(); + return null; } - DataStreamOptions.Template.Builder builder = null; + DataStreamOptions.Builder builder = null; for (ResettableValue current : dataStreamOptionsList) { if (current.isDefined() == false) { continue; @@ -1786,14 +1777,13 @@ public static ResettableValue composeDataStreamOptio } else { DataStreamOptions.Template currentTemplate = current.get(); if (builder == null) { - builder = DataStreamOptions.Template.builder(currentTemplate); + builder = DataStreamOptions.builder(currentTemplate); } else { - // Currently failure store has only one field that needs to be defined so the composing of the failure store is trivial - builder.updateFailureStore(currentTemplate.failureStore()); + builder.composeTemplate(currentTemplate); } } } - return builder == null ? ResettableValue.undefined() : ResettableValue.create(builder.build()); + return builder; } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java index e6ab7d1722e60..2bb149c131ff4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ResettableValue.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; -import java.util.function.BiFunction; import java.util.function.Function; /** @@ -154,23 +153,6 @@ public ResettableValue map(Function mapper) { return ResettableValue.create(mapper.apply(value)); } - /** - * Ιt merges the values of the ResettableValue's when they are defined using the provided mergeFunction. - */ - public static ResettableValue merge(ResettableValue initial, ResettableValue update, BiFunction mergeFunction) { - if (update.shouldReset()) { - return undefined(); - } - if (update.isDefined() == false) { - return initial; - } - if (initial.isDefined() == false || initial.shouldReset()) { - return update; - } - // Because we checked that's defined and not in reset state, we can directly apply the merge function. - return ResettableValue.create(mergeFunction.apply(initial.value, update.value)); - } - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params, String field) throws IOException { return toXContent(builder, params, field, Function.identity()); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java index 656bd442a3eb5..6f413cbbedece 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java @@ -61,6 +61,7 @@ public class Template implements SimpleDiffable