Skip to content

Commit dfcf9d6

Browse files
gmarouligmarouli
committed
Unify template builders for data stream options, failure store and data stream lifecycle (elastic#125293)
(cherry picked from commit 2c377f9) # Conflicts: # server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java # server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java # server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java # server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java # server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java # server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
1 parent 0301c86 commit dfcf9d6

File tree

32 files changed

+400
-381
lines changed

32 files changed

+400
-381
lines changed

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1372,7 +1372,7 @@ public void testSearchAllResolvesDataStreams() throws Exception {
13721372

13731373
public void testGetDataStream() throws Exception {
13741374
Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build();
1375-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
1375+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate();
13761376
putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false);
13771377
int numDocsFoo = randomIntBetween(2, 16);
13781378
indexDocs("metrics-foo", numDocsFoo);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ public void testPutLifecycle() throws Exception {
226226
}
227227

228228
public void testDeleteLifecycle() throws Exception {
229-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
229+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate();
230230
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
231231
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
232232
{

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
204204
Template.builder()
205205
.settings(Settings.EMPTY)
206206
.mappings(mappings)
207-
.lifecycle(DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build())
207+
.lifecycle(DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()))
208208
)
209209
.dataStreamTemplate(new DataStreamTemplate())
210210
.build(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ public void testRolloverLifecycle() throws Exception {
178178
}
179179

180180
public void testRolloverAndRetention() throws Exception {
181-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.ZERO).build();
181+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).buildTemplate();
182182

183183
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
184184

@@ -321,7 +321,7 @@ public void testOriginationDate() throws Exception {
321321
* days ago, and one with an origination date 1 day ago. After data stream lifecycle runs, we expect the one with the old
322322
* origination date to have been deleted, and the one with the newer origination date to remain.
323323
*/
324-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.timeValueDays(7)).build();
324+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(7)).buildTemplate();
325325

326326
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
327327

@@ -970,7 +970,7 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception
970970

971971
public void testReenableDataStreamLifecycle() throws Exception {
972972
// start with a lifecycle that's not enabled
973-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().enabled(false).build();
973+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().enabled(false).buildTemplate();
974974

975975
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
976976
String dataStreamName = "metrics-foo";
@@ -1029,7 +1029,7 @@ public void testReenableDataStreamLifecycle() throws Exception {
10291029

10301030
public void testLifecycleAppliedToFailureStore() throws Exception {
10311031
// We configure a lifecycle with downsampling to ensure it doesn't fail
1032-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
1032+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder()
10331033
.dataRetention(TimeValue.timeValueSeconds(20))
10341034
.downsampling(
10351035
List.of(
@@ -1039,7 +1039,7 @@ public void testLifecycleAppliedToFailureStore() throws Exception {
10391039
)
10401040
)
10411041
)
1042-
.build();
1042+
.buildTemplate();
10431043

10441044
putComposableIndexTemplate("id1", """
10451045
{
@@ -1263,8 +1263,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
12631263
Template.builder()
12641264
.settings(Settings.EMPTY)
12651265
.lifecycle(
1266-
DataStreamLifecycle.Template.builder()
1267-
.dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
1266+
DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
12681267
)
12691268
)
12701269
.build(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ public void testExplainDataStreamLifecycleForUnmanagedIndices() throws Exception
374374
List.of("metrics-foo*"),
375375
null,
376376
null,
377-
DataStreamLifecycle.Template.builder().enabled(false).build()
377+
DataStreamLifecycle.builder().enabled(false).buildTemplate()
378378
);
379379
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
380380
TEST_REQUEST_TIMEOUT,

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java‎

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -145,12 +145,9 @@ public void testLifecycleComposition() {
145145
}
146146
// One lifecycle results to this lifecycle as the final
147147
{
148-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
149-
.dataRetention(randomRetention())
150-
.downsampling(randomDownsampling())
151-
.build();
148+
DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(true, randomRetention(), randomDownsampling());
152149
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle);
153-
DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle();
150+
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
154151
// Defaults to true
155152
assertThat(result.enabled(), equalTo(true));
156153
assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get()));
@@ -159,31 +156,19 @@ public void testLifecycleComposition() {
159156
// If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones
160157
// Enabled is always true unless it's explicitly set to false
161158
{
162-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
163-
.enabled(false)
164-
.dataRetention(randomPositiveTimeValue())
165-
.downsampling(randomRounds())
166-
.build();
159+
DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
167160
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DEFAULT);
168-
DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle();
161+
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
169162
assertThat(result.enabled(), equalTo(true));
170163
assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get()));
171164
assertThat(result.downsampling(), equalTo(lifecycle.downsampling().get()));
172165
}
173166
// If both lifecycle have all properties, then the latest one overwrites all the others
174167
{
175-
DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.Template.builder()
176-
.enabled(false)
177-
.dataRetention(randomPositiveTimeValue())
178-
.downsampling(randomRounds())
179-
.build();
180-
DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.Template.builder()
181-
.enabled(true)
182-
.dataRetention(randomPositiveTimeValue())
183-
.downsampling(randomRounds())
184-
.build();
168+
DataStreamLifecycle.Template lifecycle1 = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
169+
DataStreamLifecycle.Template lifecycle2 = new DataStreamLifecycle.Template(true, randomPositiveTimeValue(), randomRounds());
185170
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle1, lifecycle2);
186-
DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle();
171+
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
187172
assertThat(result.enabled(), equalTo(lifecycle2.enabled()));
188173
assertThat(result.dataRetention(), equalTo(lifecycle2.dataRetention().get()));
189174
assertThat(result.downsampling(), equalTo(lifecycle2.downsampling().get()));

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -129,11 +129,11 @@ static void putComposableIndexTemplate(
129129
}
130130

131131
static DataStreamLifecycle.Template randomLifecycleTemplate() {
132-
return DataStreamLifecycle.Template.builder()
133-
.dataRetention(randomResettable(ESTestCase::randomTimeValue))
134-
.downsampling(randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds))
135-
.enabled(frequently())
136-
.build();
132+
return new DataStreamLifecycle.Template(
133+
frequently(),
134+
randomResettable(ESTestCase::randomTimeValue),
135+
randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds)
136+
);
137137
}
138138

139139
private static <T> ResettableValue<T> randomResettable(Supplier<T> supplier) {

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ public void testOperationsExecutedOnce() {
213213
numBackingIndices,
214214
2,
215215
settings(IndexVersion.current()),
216-
DataStreamLifecycle.builder().dataRetention(0).build(),
216+
DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(),
217217
now
218218
);
219219
builder.put(dataStream);
@@ -309,7 +309,7 @@ public void testRetentionNotExecutedForTSIndicesWithinTimeBounds() {
309309
dataStream.copy()
310310
.setName(dataStreamName)
311311
.setGeneration(dataStream.getGeneration() + 1)
312-
.setLifecycle(DataStreamLifecycle.builder().dataRetention(0L).build())
312+
.setLifecycle(DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build())
313313
.build()
314314
);
315315
clusterState = ClusterState.builder(clusterState).metadata(builder).build();
@@ -458,7 +458,7 @@ public void testIlmManagedIndicesAreSkipped() {
458458
Settings.builder()
459459
.put(IndexMetadata.LIFECYCLE_NAME, "ILM_policy")
460460
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()),
461-
DataStreamLifecycle.builder().dataRetention(0).build(),
461+
DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(),
462462
now
463463
);
464464
builder.put(dataStream);
@@ -1526,7 +1526,7 @@ public void testFailureStoreIsManagedEvenWhenDisabled() {
15261526
numBackingIndices,
15271527
2,
15281528
settings(IndexVersion.current()),
1529-
DataStreamLifecycle.builder().dataRetention(0).build(),
1529+
DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(),
15301530
now
15311531
).copy().setDataStreamOptions(DataStreamOptions.FAILURE_STORE_DISABLED).build(); // failure store is managed even when disabled
15321532
builder.put(dataStream);

‎server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
2020
import org.elasticsearch.cluster.metadata.DataStream;
2121
import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
22+
import org.elasticsearch.cluster.metadata.DataStreamOptions;
2223
import org.elasticsearch.cluster.metadata.IndexMetadata;
2324
import org.elasticsearch.cluster.metadata.Metadata;
2425
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
@@ -342,16 +343,18 @@ public static Template resolveTemplate(
342343
);
343344

344345
Settings settings = Settings.builder().put(additionalSettings.build()).put(templateSettings).build();
345-
DataStreamLifecycle.Template lifecycle = resolveLifecycle(simulatedState.metadata(), matchingTemplate);
346+
DataStreamLifecycle.Builder lifecycleBuilder = resolveLifecycle(simulatedState.metadata(), matchingTemplate);
347+
DataStreamLifecycle.Template lifecycle = lifecycleBuilder == null ? null : lifecycleBuilder.buildTemplate();
346348
if (template.getDataStreamTemplate() != null && lifecycle == null && isDslOnlyMode) {
347349
lifecycle = DataStreamLifecycle.Template.DEFAULT;
348350
}
351+
DataStreamOptions.Builder optionsBuilder = resolveDataStreamOptions(simulatedState.metadata(), matchingTemplate);
349352
return new Template(
350353
settings,
351354
mergedMapping,
352355
aliasesByName,
353356
lifecycle,
354-
resolveDataStreamOptions(simulatedState.metadata(), matchingTemplate)
357+
optionsBuilder == null ? null : optionsBuilder.buildTemplate()
355358
);
356359
}
357360

‎server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -667,12 +667,12 @@ private Boolean resolveFailureStoreFromTemplate(String indexName, Metadata metad
667667
ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(template);
668668
if (composableIndexTemplate.getDataStreamTemplate() != null) {
669669
// Check if the data stream has the failure store enabled
670-
DataStreamOptions dataStreamOptions = MetadataIndexTemplateService.resolveDataStreamOptions(
670+
DataStreamOptions.Builder dataStreamOptionsBuilder = MetadataIndexTemplateService.resolveDataStreamOptions(
671671
composableIndexTemplate,
672672
metadata.componentTemplates()
673-
).mapAndGet(DataStreamOptions.Template::toDataStreamOptions);
673+
);
674674
return DataStream.isFailureStoreEffectivelyEnabled(
675-
dataStreamOptions,
675+
dataStreamOptionsBuilder == null ? null : dataStreamOptionsBuilder.build(),
676676
dataStreamFailureStoreSettings,
677677
IndexNameExpressionResolver.resolveDateMathExpression(indexName, epochMillis),
678678
systemIndices

0 commit comments

Comments
 (0)