Unify template builders for data stream options, failure store and data stream lifecycle

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -1384,7 +1384,7 @@ public void testSearchAllResolvesDataStreams() throws Exception {
 
     public void testGetDataStream() throws Exception {
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build();
-        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
+        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate();
         putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false);
         int numDocsFoo = randomIntBetween(2, 16);
         indexDocs("metrics-foo", numDocsFoo);

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java

@@ -226,7 +226,7 @@ public void testPutLifecycle() throws Exception {
     }
 
     public void testDeleteLifecycle() throws Exception {
-        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
+        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate();
         putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
         putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
         {

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java

@@ -207,7 +207,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
                                 Template.builder()
                                     .settings(Settings.EMPTY)
                                     .mappings(mappings)
-                                    .lifecycle(DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build())
+                                    .lifecycle(DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()))
                             )
                             .dataStreamTemplate(new DataStreamTemplate())
                             .build(),

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java

@@ -179,7 +179,7 @@ public void testRolloverLifecycle() throws Exception {
     }
 
     public void testRolloverAndRetention() throws Exception {
-        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.ZERO).build();
+        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).buildTemplate();
 
         putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
 
@@ -322,7 +322,7 @@ public void testOriginationDate() throws Exception {
          * days ago, and one with an origination date 1 day ago. After data stream lifecycle runs, we expect the one with the old
          * origination date to have been deleted, and the one with the newer origination date to remain.
          */
-        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.timeValueDays(7)).build();
+        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(7)).buildTemplate();
 
         putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
 
@@ -974,7 +974,7 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception
 
     public void testReenableDataStreamLifecycle() throws Exception {
         // start with a lifecycle that's not enabled
-        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().enabled(false).build();
+        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().enabled(false).buildTemplate();
 
         putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
         String dataStreamName = "metrics-foo";
@@ -1033,7 +1033,7 @@ public void testReenableDataStreamLifecycle() throws Exception {
 
     public void testLifecycleAppliedToFailureStore() throws Exception {
         // We configure a lifecycle with downsampling to ensure it doesn't fail
-        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
+        DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder()
             .dataRetention(TimeValue.timeValueSeconds(20))
             .downsampling(
                 List.of(
@@ -1043,7 +1043,7 @@ public void testLifecycleAppliedToFailureStore() throws Exception {
                     )
                 )
             )
-            .build();
+            .buildTemplate();
 
         putComposableIndexTemplate("id1", """
             {
@@ -1268,8 +1268,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
                             Template.builder()
                                 .settings(Settings.EMPTY)
                                 .lifecycle(
-                                    DataStreamLifecycle.Template.builder()
-                                        .dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
+                                    DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
                                 )
                         )
                         .build(),

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java

@@ -374,7 +374,7 @@ public void testExplainDataStreamLifecycleForUnmanagedIndices() throws Exception
             List.of("metrics-foo*"),
             null,
             null,
-            DataStreamLifecycle.Template.builder().enabled(false).build()
+            DataStreamLifecycle.builder().enabled(false).buildTemplate()
         );
         CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
             TEST_REQUEST_TIMEOUT,

modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java

@@ -144,12 +144,9 @@ public void testLifecycleComposition() {
         }
         // One lifecycle results to this lifecycle as the final
         {
-            DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
-                .dataRetention(randomRetention())
-                .downsampling(randomDownsampling())
-                .build();
+            DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(true, randomRetention(), randomDownsampling());
             List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle);
-            DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle();
+            DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
             // Defaults to true
             assertThat(result.enabled(), equalTo(true));
             assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get()));
@@ -158,31 +155,19 @@ public void testLifecycleComposition() {
         // If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones
         // Enabled is always true unless it's explicitly set to false
         {
-            DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
-                .enabled(false)
-                .dataRetention(randomPositiveTimeValue())
-                .downsampling(randomRounds())
-                .build();
+            DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
             List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DEFAULT);
-            DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle();
+            DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
             assertThat(result.enabled(), equalTo(true));
             assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get()));
             assertThat(result.downsampling(), equalTo(lifecycle.downsampling().get()));
         }
         // If both lifecycle have all properties, then the latest one overwrites all the others
         {
-            DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.Template.builder()
-                .enabled(false)
-                .dataRetention(randomPositiveTimeValue())
-                .downsampling(randomRounds())
-                .build();
-            DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.Template.builder()
-                .enabled(true)
-                .dataRetention(randomPositiveTimeValue())
-                .downsampling(randomRounds())
-                .build();
+            DataStreamLifecycle.Template lifecycle1 = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
+            DataStreamLifecycle.Template lifecycle2 = new DataStreamLifecycle.Template(true, randomPositiveTimeValue(), randomRounds());
             List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle1, lifecycle2);
-            DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle();
+            DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
             assertThat(result.enabled(), equalTo(lifecycle2.enabled()));
             assertThat(result.dataRetention(), equalTo(lifecycle2.dataRetention().get()));
             assertThat(result.downsampling(), equalTo(lifecycle2.downsampling().get()));

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java

@@ -171,11 +171,11 @@ static void putComposableIndexTemplate(
     }
 
     static DataStreamLifecycle.Template randomLifecycleTemplate() {
-        return DataStreamLifecycle.Template.builder()
-            .dataRetention(randomResettable(ESTestCase::randomTimeValue))
-            .downsampling(randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds))
-            .enabled(frequently())
-            .build();
+        return new DataStreamLifecycle.Template(
+            frequently(),
+            randomResettable(ESTestCase::randomTimeValue),
+            randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds)
+        );
     }
 
     private static <T> ResettableValue<T> randomResettable(Supplier<T> supplier) {

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java

@@ -205,7 +205,7 @@ public void testOperationsExecutedOnce() {
             numBackingIndices,
             2,
             settings(IndexVersion.current()),
-            DataStreamLifecycle.builder().dataRetention(0).build(),
+            DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(),
             now
         );
         builder.put(dataStream);
@@ -301,7 +301,7 @@ public void testRetentionNotExecutedForTSIndicesWithinTimeBounds() {
             dataStream.copy()
                 .setName(dataStreamName)
                 .setGeneration(dataStream.getGeneration() + 1)
-                .setLifecycle(DataStreamLifecycle.builder().dataRetention(0L).build())
+                .setLifecycle(DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build())
                 .build()
         );
         clusterState = ClusterState.builder(clusterState).metadata(builder).build();
@@ -453,7 +453,7 @@ public void testIlmManagedIndicesAreSkipped() {
             Settings.builder()
                 .put(IndexMetadata.LIFECYCLE_NAME, "ILM_policy")
                 .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()),
-            DataStreamLifecycle.builder().dataRetention(0).build(),
+            DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(),
             now
         );
         builder.put(dataStream);
@@ -1555,7 +1555,7 @@ public void testFailureStoreIsManagedEvenWhenDisabled() {
             numBackingIndices,
             2,
             settings(IndexVersion.current()),
-            DataStreamLifecycle.builder().dataRetention(0).build(),
+            DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(),
             now
         ).copy().setDataStreamOptions(DataStreamOptions.FAILURE_STORE_DISABLED).build(); // failure store is managed even when disabled
         builder.put(dataStream);

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
+import org.elasticsearch.cluster.metadata.DataStreamOptions;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
@@ -344,16 +345,18 @@ public static Template resolveTemplate(
         );
 
         Settings settings = Settings.builder().put(additionalSettings.build()).put(templateSettings).build();
-        DataStreamLifecycle.Template lifecycle = resolveLifecycle(simulatedProject, matchingTemplate);
+        DataStreamLifecycle.Builder lifecycleBuilder = resolveLifecycle(simulatedProject, matchingTemplate);
+        DataStreamLifecycle.Template lifecycle = lifecycleBuilder == null ? null : lifecycleBuilder.buildTemplate();
         if (template.getDataStreamTemplate() != null && lifecycle == null && isDslOnlyMode) {
             lifecycle = DataStreamLifecycle.Template.DEFAULT;
         }
+        DataStreamOptions.Builder optionsBuilder = resolveDataStreamOptions(simulatedProject, matchingTemplate);
         return new Template(
             settings,
             mergedMapping,
             aliasesByName,
             lifecycle,
-            resolveDataStreamOptions(simulatedProject, matchingTemplate)
+            optionsBuilder == null ? null : optionsBuilder.buildTemplate()
         );
     }
 

server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java

@@ -680,12 +680,12 @@ private Boolean resolveFailureStoreFromTemplate(String indexName, ProjectMetadat
             ComposableIndexTemplate composableIndexTemplate = projectMetadata.templatesV2().get(template);
             if (composableIndexTemplate.getDataStreamTemplate() != null) {
                 // Check if the data stream has the failure store enabled
-                DataStreamOptions dataStreamOptions = MetadataIndexTemplateService.resolveDataStreamOptions(
+                DataStreamOptions.Builder dataStreamOptionsBuilder = MetadataIndexTemplateService.resolveDataStreamOptions(
                     composableIndexTemplate,
                     projectMetadata.componentTemplates()
-                ).mapAndGet(DataStreamOptions.Template::toDataStreamOptions);
+                );
                 return DataStream.isFailureStoreEffectivelyEnabled(
-                    dataStreamOptions,
+                    dataStreamOptionsBuilder == null ? null : dataStreamOptionsBuilder.build(),
                     dataStreamFailureStoreSettings,
                     IndexNameExpressionResolver.resolveDateMathExpression(indexName, epochMillis),
                     systemIndices

server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamFailureStore.java

@@ -89,8 +89,7 @@ public static DataStreamFailureStore fromXContent(XContentParser parser) throws
 
     /**
      * This class is only used in template configuration. It wraps the fields of {@link DataStreamFailureStore} with {@link ResettableValue}
-     * to allow a user to signal when they want to reset any previously encountered values during template composition. Furthermore, it
-     * provides the method {@link #merge(Template, Template)} that dictates how two templates can be composed.
+     * to allow a user to signal when they want to reset any previously encountered values during template composition.
      */
     public record Template(ResettableValue<Boolean> enabled) implements Writeable, ToXContentObject {
 
@@ -112,6 +111,10 @@ public record Template(ResettableValue<Boolean> enabled) implements Writeable, T
             );
         }
 
+        public Template(Boolean enabled) {
+            this(ResettableValue.create(enabled));
+        }
+
         public Template {
             if (enabled.get() == null) {
                 throw new IllegalArgumentException("Failure store configuration should have at least one non-null configuration value.");
@@ -144,15 +147,6 @@ public static Template fromXContent(XContentParser parser) throws IOException {
             return PARSER.parse(parser, null);
         }
 
-        /**
-         * Returns a template which has the value of the initial template updated with the values of the update.
-         * Note: for now it's a trivial composition because we have only one non-null field.
-         * @return the composed template
-         */
-        public static Template merge(Template ignored, Template update) {
-            return update;
-        }
-
         public DataStreamFailureStore toFailureStore() {
             return new DataStreamFailureStore(enabled.get());
         }
@@ -162,4 +156,65 @@ public String toString() {
             return Strings.toString(this, true, true);
         }
     }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static Builder builder(Template template) {
+        return new Builder(template);
+    }
+
+    public static Builder builder(DataStreamFailureStore failureStore) {
+        return new Builder(failureStore);
+    }
+
+    /**
+     * Builder that is able to create either a DataStreamFailureStore or its respective Template.
+     * Furthermore, its update methods can be used to compose templates.
+     */
+    public static class Builder {
+        private Boolean enabled = null;
+
+        private Builder() {}
+
+        private Builder(Template template) {
+            if (template != null) {
+                enabled = template.enabled.get();
+            }
+        }
+
+        private Builder(DataStreamFailureStore failureStore) {
+            if (failureStore != null) {
+                enabled = failureStore.enabled;
+            }
+        }
+
+        public Builder enabled(Boolean enabled) {
+            this.enabled = enabled;
+            return this;
+        }
+
+        public Builder enabled(ResettableValue<Boolean> enabled) {
+            if (enabled.shouldReset()) {
+                this.enabled = null;
+            } else if (enabled.isDefined()) {
+                this.enabled = enabled.get();
+            }
+            return this;
+        }
+
+        public Builder composeTemplate(DataStreamFailureStore.Template failureStore) {
+            this.enabled(failureStore.enabled());
+            return this;
+        }
+
+        public DataStreamFailureStore build() {
+            return new DataStreamFailureStore(enabled);
+        }
+
+        public DataStreamFailureStore.Template buildTemplate() {
+            return new Template(enabled);
+        }
+    }
 }