Skip to content

Commit a3fc085

Browse files
gmaroulismalyshev
gmarouli
authored and
smalyshev
committed
Unify template builders for data stream options, failure store and data stream lifecycle (elastic#125293)
1 parent 1ce3c4c commit a3fc085

File tree

32 files changed

+402
-392
lines changed

32 files changed

+402
-392
lines changed

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1384,7 +1384,7 @@ public void testSearchAllResolvesDataStreams() throws Exception {
13841384

13851385
public void testGetDataStream() throws Exception {
13861386
Settings settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, maximumNumberOfReplicas() + 2).build();
1387-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
1387+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate();
13881388
putComposableIndexTemplate("template_for_foo", null, List.of("metrics-foo*"), settings, null, null, lifecycle, false);
13891389
int numDocsFoo = randomIntBetween(2, 16);
13901390
indexDocs("metrics-foo", numDocsFoo);

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ public void testPutLifecycle() throws Exception {
226226
}
227227

228228
public void testDeleteLifecycle() throws Exception {
229-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build();
229+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()).buildTemplate();
230230
putComposableIndexTemplate("id1", null, List.of("with-lifecycle*"), null, null, lifecycle);
231231
putComposableIndexTemplate("id2", null, List.of("without-lifecycle*"), null, null, null);
232232
{

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
207207
Template.builder()
208208
.settings(Settings.EMPTY)
209209
.mappings(mappings)
210-
.lifecycle(DataStreamLifecycle.Template.builder().dataRetention(randomPositiveTimeValue()).build())
210+
.lifecycle(DataStreamLifecycle.builder().dataRetention(randomPositiveTimeValue()))
211211
)
212212
.dataStreamTemplate(new DataStreamTemplate())
213213
.build(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ public void testRolloverLifecycle() throws Exception {
179179
}
180180

181181
public void testRolloverAndRetention() throws Exception {
182-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.ZERO).build();
182+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).buildTemplate();
183183

184184
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
185185

@@ -322,7 +322,7 @@ public void testOriginationDate() throws Exception {
322322
* days ago, and one with an origination date 1 day ago. After data stream lifecycle runs, we expect the one with the old
323323
* origination date to have been deleted, and the one with the newer origination date to remain.
324324
*/
325-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().dataRetention(TimeValue.timeValueDays(7)).build();
325+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(7)).buildTemplate();
326326

327327
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
328328

@@ -974,7 +974,7 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception
974974

975975
public void testReenableDataStreamLifecycle() throws Exception {
976976
// start with a lifecycle that's not enabled
977-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder().enabled(false).build();
977+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder().enabled(false).buildTemplate();
978978

979979
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle, false);
980980
String dataStreamName = "metrics-foo";
@@ -1033,7 +1033,7 @@ public void testReenableDataStreamLifecycle() throws Exception {
10331033

10341034
public void testLifecycleAppliedToFailureStore() throws Exception {
10351035
// We configure a lifecycle with downsampling to ensure it doesn't fail
1036-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
1036+
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.builder()
10371037
.dataRetention(TimeValue.timeValueSeconds(20))
10381038
.downsampling(
10391039
List.of(
@@ -1043,7 +1043,7 @@ public void testLifecycleAppliedToFailureStore() throws Exception {
10431043
)
10441044
)
10451045
)
1046-
.build();
1046+
.buildTemplate();
10471047

10481048
putComposableIndexTemplate("id1", """
10491049
{
@@ -1268,8 +1268,7 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
12681268
Template.builder()
12691269
.settings(Settings.EMPTY)
12701270
.lifecycle(
1271-
DataStreamLifecycle.Template.builder()
1272-
.dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
1271+
DataStreamLifecycle.builder().dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
12731272
)
12741273
)
12751274
.build(),

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ public void testExplainDataStreamLifecycleForUnmanagedIndices() throws Exception
374374
List.of("metrics-foo*"),
375375
null,
376376
null,
377-
DataStreamLifecycle.Template.builder().enabled(false).build()
377+
DataStreamLifecycle.builder().enabled(false).buildTemplate()
378378
);
379379
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
380380
TEST_REQUEST_TIMEOUT,

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java‎

Lines changed: 7 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -144,12 +144,9 @@ public void testLifecycleComposition() {
144144
}
145145
// One lifecycle results to this lifecycle as the final
146146
{
147-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
148-
.dataRetention(randomRetention())
149-
.downsampling(randomDownsampling())
150-
.build();
147+
DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(true, randomRetention(), randomDownsampling());
151148
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle);
152-
DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle();
149+
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
153150
// Defaults to true
154151
assertThat(result.enabled(), equalTo(true));
155152
assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get()));
@@ -158,31 +155,19 @@ public void testLifecycleComposition() {
158155
// If the last lifecycle is missing a property (apart from enabled) we keep the latest from the previous ones
159156
// Enabled is always true unless it's explicitly set to false
160157
{
161-
DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.Template.builder()
162-
.enabled(false)
163-
.dataRetention(randomPositiveTimeValue())
164-
.downsampling(randomRounds())
165-
.build();
158+
DataStreamLifecycle.Template lifecycle = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
166159
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DEFAULT);
167-
DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle();
160+
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
168161
assertThat(result.enabled(), equalTo(true));
169162
assertThat(result.dataRetention(), equalTo(lifecycle.dataRetention().get()));
170163
assertThat(result.downsampling(), equalTo(lifecycle.downsampling().get()));
171164
}
172165
// If both lifecycle have all properties, then the latest one overwrites all the others
173166
{
174-
DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.Template.builder()
175-
.enabled(false)
176-
.dataRetention(randomPositiveTimeValue())
177-
.downsampling(randomRounds())
178-
.build();
179-
DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.Template.builder()
180-
.enabled(true)
181-
.dataRetention(randomPositiveTimeValue())
182-
.downsampling(randomRounds())
183-
.build();
167+
DataStreamLifecycle.Template lifecycle1 = new DataStreamLifecycle.Template(false, randomPositiveTimeValue(), randomRounds());
168+
DataStreamLifecycle.Template lifecycle2 = new DataStreamLifecycle.Template(true, randomPositiveTimeValue(), randomRounds());
184169
List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle1, lifecycle2);
185-
DataStreamLifecycle result = composeDataLifecycles(lifecycles).toDataStreamLifecycle();
170+
DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
186171
assertThat(result.enabled(), equalTo(lifecycle2.enabled()));
187172
assertThat(result.dataRetention(), equalTo(lifecycle2.dataRetention().get()));
188173
assertThat(result.downsampling(), equalTo(lifecycle2.downsampling().get()));

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -171,11 +171,11 @@ static void putComposableIndexTemplate(
171171
}
172172

173173
static DataStreamLifecycle.Template randomLifecycleTemplate() {
174-
return DataStreamLifecycle.Template.builder()
175-
.dataRetention(randomResettable(ESTestCase::randomTimeValue))
176-
.downsampling(randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds))
177-
.enabled(frequently())
178-
.build();
174+
return new DataStreamLifecycle.Template(
175+
frequently(),
176+
randomResettable(ESTestCase::randomTimeValue),
177+
randomResettable(DataStreamLifecycleFixtures::randomDownsamplingRounds)
178+
);
179179
}
180180

181181
private static <T> ResettableValue<T> randomResettable(Supplier<T> supplier) {

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ public void testOperationsExecutedOnce() {
205205
numBackingIndices,
206206
2,
207207
settings(IndexVersion.current()),
208-
DataStreamLifecycle.builder().dataRetention(0).build(),
208+
DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(),
209209
now
210210
);
211211
builder.put(dataStream);
@@ -301,7 +301,7 @@ public void testRetentionNotExecutedForTSIndicesWithinTimeBounds() {
301301
dataStream.copy()
302302
.setName(dataStreamName)
303303
.setGeneration(dataStream.getGeneration() + 1)
304-
.setLifecycle(DataStreamLifecycle.builder().dataRetention(0L).build())
304+
.setLifecycle(DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build())
305305
.build()
306306
);
307307
clusterState = ClusterState.builder(clusterState).metadata(builder).build();
@@ -453,7 +453,7 @@ public void testIlmManagedIndicesAreSkipped() {
453453
Settings.builder()
454454
.put(IndexMetadata.LIFECYCLE_NAME, "ILM_policy")
455455
.put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()),
456-
DataStreamLifecycle.builder().dataRetention(0).build(),
456+
DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(),
457457
now
458458
);
459459
builder.put(dataStream);
@@ -1555,7 +1555,7 @@ public void testFailureStoreIsManagedEvenWhenDisabled() {
15551555
numBackingIndices,
15561556
2,
15571557
settings(IndexVersion.current()),
1558-
DataStreamLifecycle.builder().dataRetention(0).build(),
1558+
DataStreamLifecycle.builder().dataRetention(TimeValue.ZERO).build(),
15591559
now
15601560
).copy().setDataStreamOptions(DataStreamOptions.FAILURE_STORE_DISABLED).build(); // failure store is managed even when disabled
15611561
builder.put(dataStream);

‎server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
2121
import org.elasticsearch.cluster.metadata.DataStream;
2222
import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
23+
import org.elasticsearch.cluster.metadata.DataStreamOptions;
2324
import org.elasticsearch.cluster.metadata.IndexMetadata;
2425
import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
2526
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
@@ -344,16 +345,18 @@ public static Template resolveTemplate(
344345
);
345346

346347
Settings settings = Settings.builder().put(additionalSettings.build()).put(templateSettings).build();
347-
DataStreamLifecycle.Template lifecycle = resolveLifecycle(simulatedProject, matchingTemplate);
348+
DataStreamLifecycle.Builder lifecycleBuilder = resolveLifecycle(simulatedProject, matchingTemplate);
349+
DataStreamLifecycle.Template lifecycle = lifecycleBuilder == null ? null : lifecycleBuilder.buildTemplate();
348350
if (template.getDataStreamTemplate() != null && lifecycle == null && isDslOnlyMode) {
349351
lifecycle = DataStreamLifecycle.Template.DEFAULT;
350352
}
353+
DataStreamOptions.Builder optionsBuilder = resolveDataStreamOptions(simulatedProject, matchingTemplate);
351354
return new Template(
352355
settings,
353356
mergedMapping,
354357
aliasesByName,
355358
lifecycle,
356-
resolveDataStreamOptions(simulatedProject, matchingTemplate)
359+
optionsBuilder == null ? null : optionsBuilder.buildTemplate()
357360
);
358361
}
359362

‎server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -680,12 +680,12 @@ private Boolean resolveFailureStoreFromTemplate(String indexName, ProjectMetadat
680680
ComposableIndexTemplate composableIndexTemplate = projectMetadata.templatesV2().get(template);
681681
if (composableIndexTemplate.getDataStreamTemplate() != null) {
682682
// Check if the data stream has the failure store enabled
683-
DataStreamOptions dataStreamOptions = MetadataIndexTemplateService.resolveDataStreamOptions(
683+
DataStreamOptions.Builder dataStreamOptionsBuilder = MetadataIndexTemplateService.resolveDataStreamOptions(
684684
composableIndexTemplate,
685685
projectMetadata.componentTemplates()
686-
).mapAndGet(DataStreamOptions.Template::toDataStreamOptions);
686+
);
687687
return DataStream.isFailureStoreEffectivelyEnabled(
688-
dataStreamOptions,
688+
dataStreamOptionsBuilder == null ? null : dataStreamOptionsBuilder.build(),
689689
dataStreamFailureStoreSettings,
690690
IndexNameExpressionResolver.resolveDateMathExpression(indexName, epochMillis),
691691
systemIndices

0 commit comments

Comments
 (0)