Skip to content

Commit 5d1c17c

Browse files
gmarouligmarouli
committed
Extend the APIs to use the respective global retention
1 parent 5c4273e commit 5d1c17c

File tree

9 files changed

+117
-45
lines changed

9 files changed

+117
-45
lines changed

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,8 @@ public int compareTo(IndexInfo o) {
370370
return new GetDataStreamAction.Response(
371371
dataStreamInfos,
372372
request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null,
373-
globalRetentionSettings.get()
373+
globalRetentionSettings.get(false),
374+
globalRetentionSettings.get(true)
374375
);
375376
}
376377

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,8 @@ protected void masterOperation(
123123
new ExplainDataStreamLifecycleAction.Response(
124124
explainIndices,
125125
request.includeDefaults() ? clusterSettings.get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING) : null,
126-
globalRetentionSettings.get()
126+
globalRetentionSettings.get(false),
127+
globalRetentionSettings.get(true)
127128
)
128129
);
129130
}

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -362,8 +362,8 @@ public void testPassingGlobalRetention() {
362362
new IndexSettingProviders(Set.of()),
363363
null
364364
);
365-
assertThat(response.getGlobalRetention(), nullValue());
366-
DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(
365+
assertThat(response.getDataGlobalRetention(), nullValue());
366+
DataStreamGlobalRetention dataGlobalRetention = new DataStreamGlobalRetention(
367367
TimeValue.timeValueDays(randomIntBetween(1, 5)),
368368
TimeValue.timeValueDays(randomIntBetween(5, 10))
369369
);
@@ -372,9 +372,9 @@ public void testPassingGlobalRetention() {
372372
Settings.builder()
373373
.put(
374374
DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey(),
375-
globalRetention.defaultRetention()
375+
dataGlobalRetention.defaultRetention()
376376
)
377-
.put(DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING.getKey(), globalRetention.maxRetention())
377+
.put(DataStreamGlobalRetentionSettings.DATA_STREAMS_MAX_RETENTION_SETTING.getKey(), dataGlobalRetention.maxRetention())
378378
.build()
379379
)
380380
);
@@ -389,7 +389,9 @@ public void testPassingGlobalRetention() {
389389
new IndexSettingProviders(Set.of()),
390390
null
391391
);
392-
assertThat(response.getGlobalRetention(), equalTo(globalRetention));
392+
assertThat(response.getDataGlobalRetention(), equalTo(dataGlobalRetention));
393+
// We used the default failures retention here which is greater than the max
394+
assertThat(response.getFailuresGlobalRetention(), equalTo(new DataStreamGlobalRetention(null, dataGlobalRetention.maxRetention())));
393395
}
394396

395397
public void testDataStreamIsFailureStoreEffectivelyEnabled_disabled() {

‎modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_failure_store_info.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,9 @@ teardown:
189189
- match: { data_streams.0.template: 'my-template1' }
190190
- match: { data_streams.0.failure_store.enabled: true }
191191
- match: { data_streams.0.failure_store.lifecycle.enabled: false }
192+
- is_false: data_streams.0.failure_store.lifecycle.data_retention
193+
- is_false: data_streams.0.failure_store.lifecycle.effective_retention
194+
- is_false: data_streams.0.failure_store.lifecycle.retention_determined_by
192195
- length: { data_streams.0.failure_store.indices: 1 }
193196
- match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
194197
- is_false: data_streams.0.failure_store.indices.0.prefer_ilm
@@ -212,6 +215,9 @@ teardown:
212215
- match: { data_streams.0.template: 'my-template2' }
213216
- match: { data_streams.0.failure_store.enabled: true }
214217
- match: { data_streams.0.failure_store.lifecycle.enabled: true }
218+
- is_false: data_streams.0.failure_store.lifecycle.data_retention
219+
- match: { data_streams.0.failure_store.lifecycle.effective_retention: '30d' }
220+
- match: { data_streams.0.failure_store.lifecycle.retention_determined_by: 'default_failures_retention' }
215221
- match: { data_streams.0.failure_store.indices: [] }
216222

217223
# Initialize failure store
@@ -234,6 +240,9 @@ teardown:
234240
- match: { data_streams.0.template: 'my-template2' }
235241
- match: { data_streams.0.failure_store.enabled: true }
236242
- match: { data_streams.0.failure_store.lifecycle.enabled: true }
243+
- is_false: data_streams.0.failure_store.lifecycle.data_retention
244+
- match: { data_streams.0.failure_store.lifecycle.effective_retention: '30d' }
245+
- match: { data_streams.0.failure_store.lifecycle.retention_determined_by: 'default_failures_retention' }
237246
- length: { data_streams.0.failure_store.indices: 1 }
238247
- match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-fs-default-data-stream-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
239248
- is_false: data_streams.0.failure_store.indices.0.prefer_ilm

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,7 @@ static TransportVersion def(int id) {
229229
public static final TransportVersion RANDOM_SAMPLER_QUERY_BUILDER = def(9_063_0_00);
230230
public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(9_064_0_00);
231231
public static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = def(9_065_0_00);
232+
public static final TransportVersion INTRODUCE_FAILURES_DEFAULT_RETENTION = def(9_066_0_00);
232233

233234
/*
234235
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java‎

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ public void writeTo(StreamOutput out) throws IOException {
351351

352352
@Override
353353
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
354-
return toXContent(builder, params, null, null);
354+
return toXContent(builder, params, null, null, null);
355355
}
356356

357357
/**
@@ -362,7 +362,8 @@ public XContentBuilder toXContent(
362362
XContentBuilder builder,
363363
Params params,
364364
@Nullable RolloverConfiguration rolloverConfiguration,
365-
@Nullable DataStreamGlobalRetention globalRetention
365+
@Nullable DataStreamGlobalRetention dataGlobalRetention,
366+
@Nullable DataStreamGlobalRetention failureGlobalRetention
366367
) throws IOException {
367368
builder.startObject();
368369
builder.field(DataStream.NAME_FIELD.getPreferredName(), dataStream.getName());
@@ -383,7 +384,7 @@ public XContentBuilder toXContent(
383384
if (dataStream.getDataLifecycle() != null) {
384385
builder.field(LIFECYCLE_FIELD.getPreferredName());
385386
dataStream.getDataLifecycle()
386-
.toXContent(builder, params, rolloverConfiguration, globalRetention, dataStream.isInternal());
387+
.toXContent(builder, params, rolloverConfiguration, dataGlobalRetention, dataStream.isInternal());
387388
}
388389
if (ilmPolicyName != null) {
389390
builder.field(ILM_POLICY_FIELD.getPreferredName(), ilmPolicyName);
@@ -425,7 +426,7 @@ public XContentBuilder toXContent(
425426
DataStreamLifecycle failuresLifecycle = dataStream.getFailuresLifecycle(failureStoreEffectivelyEnabled);
426427
if (failuresLifecycle != null) {
427428
builder.field(LIFECYCLE_FIELD.getPreferredName());
428-
failuresLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, dataStream.isInternal());
429+
failuresLifecycle.toXContent(builder, params, rolloverConfiguration, failureGlobalRetention, dataStream.isInternal());
429430
}
430431
builder.endObject();
431432
builder.endObject();
@@ -599,20 +600,24 @@ public void writeTo(StreamOutput out) throws IOException {
599600
@Nullable
600601
private final RolloverConfiguration rolloverConfiguration;
601602
@Nullable
602-
private final DataStreamGlobalRetention globalRetention;
603+
private final DataStreamGlobalRetention dataGlobalRetention;
604+
@Nullable
605+
private final DataStreamGlobalRetention failuresGlobalRetention;
603606

604607
public Response(List<DataStreamInfo> dataStreams) {
605-
this(dataStreams, null, null);
608+
this(dataStreams, null, null, null);
606609
}
607610

608611
public Response(
609612
List<DataStreamInfo> dataStreams,
610613
@Nullable RolloverConfiguration rolloverConfiguration,
611-
@Nullable DataStreamGlobalRetention globalRetention
614+
@Nullable DataStreamGlobalRetention dataGlobalRetention,
615+
@Nullable DataStreamGlobalRetention failuresGlobalRetention
612616
) {
613617
this.dataStreams = dataStreams;
614618
this.rolloverConfiguration = rolloverConfiguration;
615-
this.globalRetention = globalRetention;
619+
this.dataGlobalRetention = dataGlobalRetention;
620+
this.failuresGlobalRetention = failuresGlobalRetention;
616621
}
617622

618623
public List<DataStreamInfo> getDataStreams() {
@@ -625,8 +630,13 @@ public RolloverConfiguration getRolloverConfiguration() {
625630
}
626631

627632
@Nullable
628-
public DataStreamGlobalRetention getGlobalRetention() {
629-
return globalRetention;
633+
public DataStreamGlobalRetention getDataGlobalRetention() {
634+
return dataGlobalRetention;
635+
}
636+
637+
@Nullable
638+
public DataStreamGlobalRetention getFailuresGlobalRetention() {
639+
return failuresGlobalRetention;
630640
}
631641

632642
/**
@@ -640,8 +650,12 @@ public void writeTo(StreamOutput out) throws IOException {
640650
if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) {
641651
out.writeOptionalWriteable(rolloverConfiguration);
642652
}
643-
if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
644-
out.writeOptionalWriteable(globalRetention);
653+
if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.INTRODUCE_FAILURES_DEFAULT_RETENTION)) {
654+
out.writeOptionalWriteable(dataGlobalRetention);
655+
} else if (out.getTransportVersion().onOrAfter(TransportVersions.INTRODUCE_FAILURES_DEFAULT_RETENTION)) {
656+
out.writeOptionalTimeValue(dataGlobalRetention == null ? null : dataGlobalRetention.defaultRetention());
657+
out.writeOptionalTimeValue(dataGlobalRetention == null ? null : dataGlobalRetention.maxRetention());
658+
out.writeOptionalTimeValue(failuresGlobalRetention == null ? null : failuresGlobalRetention.defaultRetention());
645659
}
646660
}
647661

@@ -654,7 +668,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
654668
builder,
655669
DataStreamLifecycle.addEffectiveRetentionParams(params),
656670
rolloverConfiguration,
657-
globalRetention
671+
dataGlobalRetention,
672+
failuresGlobalRetention
658673
);
659674
}
660675
builder.endArray();
@@ -669,12 +684,13 @@ public boolean equals(Object o) {
669684
Response response = (Response) o;
670685
return dataStreams.equals(response.dataStreams)
671686
&& Objects.equals(rolloverConfiguration, response.rolloverConfiguration)
672-
&& Objects.equals(globalRetention, response.globalRetention);
687+
&& Objects.equals(dataGlobalRetention, response.dataGlobalRetention)
688+
&& Objects.equals(failuresGlobalRetention, response.failuresGlobalRetention);
673689
}
674690

675691
@Override
676692
public int hashCode() {
677-
return Objects.hash(dataStreams, rolloverConfiguration, globalRetention);
693+
return Objects.hash(dataStreams, rolloverConfiguration, dataGlobalRetention, failuresGlobalRetention);
678694
}
679695
}
680696

‎server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java‎

Lines changed: 43 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -147,24 +147,38 @@ public static class Response extends ActionResponse implements ChunkedToXContent
147147
@Nullable
148148
private final RolloverConfiguration rolloverConfiguration;
149149
@Nullable
150-
private final DataStreamGlobalRetention globalRetention;
150+
private final DataStreamGlobalRetention dataGlobalRetention;
151+
@Nullable
152+
private final DataStreamGlobalRetention failureGlobalRetention;
151153

152154
public Response(
153155
List<ExplainIndexDataStreamLifecycle> indices,
154156
@Nullable RolloverConfiguration rolloverConfiguration,
155-
@Nullable DataStreamGlobalRetention globalRetention
157+
@Nullable DataStreamGlobalRetention dataGlobalRetention,
158+
@Nullable DataStreamGlobalRetention failureGlobalRetention
156159
) {
157160
this.indices = indices;
158161
this.rolloverConfiguration = rolloverConfiguration;
159-
this.globalRetention = globalRetention;
162+
this.dataGlobalRetention = dataGlobalRetention;
163+
this.failureGlobalRetention = failureGlobalRetention;
160164
}
161165

162166
public Response(StreamInput in) throws IOException {
163167
this.indices = in.readCollectionAsList(ExplainIndexDataStreamLifecycle::new);
164168
this.rolloverConfiguration = in.readOptionalWriteable(RolloverConfiguration::new);
165-
this.globalRetention = in.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)
166-
? in.readOptionalWriteable(DataStreamGlobalRetention::read)
167-
: null;
169+
if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.INTRODUCE_FAILURES_DEFAULT_RETENTION)) {
170+
dataGlobalRetention = in.readOptionalWriteable(DataStreamGlobalRetention::read);
171+
failureGlobalRetention = dataGlobalRetention;
172+
} else if (in.getTransportVersion().onOrAfter(TransportVersions.INTRODUCE_FAILURES_DEFAULT_RETENTION)) {
173+
var defaultRetention = in.readOptionalTimeValue();
174+
var maxRetention = in.readOptionalTimeValue();
175+
var defaultFailuresRetention = in.readOptionalTimeValue();
176+
dataGlobalRetention = DataStreamGlobalRetention.create(defaultRetention, maxRetention);
177+
failureGlobalRetention = DataStreamGlobalRetention.create(defaultFailuresRetention, maxRetention);
178+
} else {
179+
dataGlobalRetention = null;
180+
failureGlobalRetention = null;
181+
}
168182
}
169183

170184
public List<ExplainIndexDataStreamLifecycle> getIndices() {
@@ -175,16 +189,31 @@ public RolloverConfiguration getRolloverConfiguration() {
175189
return rolloverConfiguration;
176190
}
177191

178-
public DataStreamGlobalRetention getGlobalRetention() {
179-
return globalRetention;
192+
public DataStreamGlobalRetention getDataGlobalRetention() {
193+
return dataGlobalRetention;
194+
}
195+
196+
public DataStreamGlobalRetention getFailuresGlobalRetention() {
197+
return failureGlobalRetention;
198+
}
199+
200+
private DataStreamGlobalRetention getGlobalRetentionForLifecycle(DataStreamLifecycle lifecycle) {
201+
if (lifecycle == null) {
202+
return null;
203+
}
204+
return lifecycle.targetsFailureStore() ? failureGlobalRetention : dataGlobalRetention;
180205
}
181206

182207
@Override
183208
public void writeTo(StreamOutput out) throws IOException {
184209
out.writeCollection(indices);
185210
out.writeOptionalWriteable(rolloverConfiguration);
186-
if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_14_0)) {
187-
out.writeOptionalWriteable(globalRetention);
211+
if (out.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.INTRODUCE_FAILURES_DEFAULT_RETENTION)) {
212+
out.writeOptionalWriteable(getDataGlobalRetention());
213+
} else if (out.getTransportVersion().onOrAfter(TransportVersions.INTRODUCE_FAILURES_DEFAULT_RETENTION)) {
214+
out.writeOptionalTimeValue(dataGlobalRetention == null ? null : dataGlobalRetention.defaultRetention());
215+
out.writeOptionalTimeValue(dataGlobalRetention == null ? null : dataGlobalRetention.maxRetention());
216+
out.writeOptionalTimeValue(failureGlobalRetention == null ? null : failureGlobalRetention.defaultRetention());
188217
}
189218
}
190219

@@ -199,12 +228,13 @@ public boolean equals(Object o) {
199228
Response response = (Response) o;
200229
return Objects.equals(indices, response.indices)
201230
&& Objects.equals(rolloverConfiguration, response.rolloverConfiguration)
202-
&& Objects.equals(globalRetention, response.globalRetention);
231+
&& Objects.equals(dataGlobalRetention, response.dataGlobalRetention)
232+
&& Objects.equals(failureGlobalRetention, response.failureGlobalRetention);
203233
}
204234

205235
@Override
206236
public int hashCode() {
207-
return Objects.hash(indices, rolloverConfiguration, globalRetention);
237+
return Objects.hash(indices, rolloverConfiguration, dataGlobalRetention, failureGlobalRetention);
208238
}
209239

210240
@Override
@@ -219,7 +249,7 @@ public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params outerP
219249
builder,
220250
DataStreamLifecycle.addEffectiveRetentionParams(outerParams),
221251
rolloverConfiguration,
222-
globalRetention
252+
getGlobalRetentionForLifecycle(explainIndexDataLifecycle.getLifecycle())
223253
);
224254
return builder;
225255
}), Iterators.single((builder, params) -> {

‎server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ private Map<String, Object> getXContentMap(
7575
ToXContent.Params params = new ToXContent.MapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS);
7676
RolloverConfiguration rolloverConfiguration = null;
7777
DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention(globalDefaultRetention, globalMaxRetention);
78-
dataStreamInfo.toXContent(builder, params, rolloverConfiguration, globalRetention);
78+
dataStreamInfo.toXContent(builder, params, rolloverConfiguration, globalRetention, globalRetention);
7979
String serialized = Strings.toString(builder);
8080
return XContentHelper.convertToMap(XContentType.JSON.xContent(), serialized, randomBoolean());
8181
}

0 commit comments

Comments
 (0)