Skip to content

Commit efcdbbc

Browse files
masseykemasseyke
authored
Adding origination date to DLM (#95113)
This adds support for an index's index.lifecycle.origination_date setting in DLM. If an index has a value for index.lifecycle.origination_date then it is used instead of the creation date or rollover date (except in the case of the write index when the write index has not been rolled over yet).
1 parent cffe0f0 commit efcdbbc

File tree

9 files changed

+517
-84
lines changed

9 files changed

+517
-84
lines changed

‎docs/changelog/95113.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 95113
2+
summary: Adding origination date to DLM
3+
area: DLM
4+
type: enhancement
5+
issues: []

‎docs/reference/dlm/apis/explain-data-lifecycle.asciidoc

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,10 @@ shown.
8282
data stream)
8383
<7> The generation time of the index represents the time since the index started progressing
8484
towards the user configurable / business specific parts of the lifecycle (e.g. retention).
85-
Every index will have to wait for it to be rolled over before being able to progress to the
86-
business-specific part of the lifecycle (i.e. the index advances in its lifecycle after it
87-
stops being the write index of a data stream). If the index has not been rolled over the
88-
`generation_time` will not be reported.
85+
The `generation_time` is calculated from the origination date if it exists, or from the
86+
rollover date if it exists, or from the creation date if neither of the other two exist.
87+
If the index is the write index the `generation_time` will not be reported because it is not
88+
eligible for retention or other parts of the lifecycle.
8989

9090
The `explain` will also report any errors related to the lifecycle execution for the target
9191
index:

‎modules/dlm/src/internalClusterTest/java/org/elasticsearch/dlm/DataLifecycleServiceIT.java

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@
1515
import org.elasticsearch.action.bulk.BulkResponse;
1616
import org.elasticsearch.action.datastreams.CreateDataStreamAction;
1717
import org.elasticsearch.action.datastreams.GetDataStreamAction;
18+
import org.elasticsearch.action.datastreams.ModifyDataStreamsAction;
1819
import org.elasticsearch.action.index.IndexRequest;
1920
import org.elasticsearch.action.support.master.AcknowledgedResponse;
2021
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
2122
import org.elasticsearch.cluster.metadata.DataLifecycle;
2223
import org.elasticsearch.cluster.metadata.DataStream;
24+
import org.elasticsearch.cluster.metadata.DataStreamAction;
2325
import org.elasticsearch.cluster.metadata.Template;
2426
import org.elasticsearch.common.compress.CompressedXContent;
2527
import org.elasticsearch.common.settings.Settings;
@@ -41,10 +43,13 @@
4143
import java.util.List;
4244
import java.util.Locale;
4345
import java.util.Map;
46+
import java.util.Set;
47+
import java.util.stream.Collectors;
4448

4549
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo;
4650
import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.READ_ONLY;
4751
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD;
52+
import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE;
4853
import static org.elasticsearch.indices.ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_NODE;
4954
import static org.hamcrest.Matchers.containsString;
5055
import static org.hamcrest.Matchers.equalTo;
@@ -130,6 +135,74 @@ public void testRolloverAndRetention() throws Exception {
130135
});
131136
}
132137

138+
public void testOriginationDate() throws Exception {
139+
/*
140+
* In this test, we set up a datastream with 7 day retention. Then we add two indices to it -- one with an origination date 365
141+
* days ago, and one with an origination date 1 day ago. After DLM runs, we expect the one with the old origination date to have
142+
* been deleted, and the one with the newer origination date to remain.
143+
*/
144+
DataLifecycle lifecycle = new DataLifecycle(TimeValue.timeValueDays(7).millis());
145+
146+
putComposableIndexTemplate("id1", null, List.of("metrics-foo*"), null, null, lifecycle);
147+
148+
String dataStreamName = "metrics-foo";
149+
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dataStreamName);
150+
client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
151+
152+
indexDocs(dataStreamName, 1);
153+
154+
String mapping = """
155+
{
156+
"properties":{
157+
"@timestamp": {
158+
"type": "date"
159+
}
160+
}
161+
}""";
162+
PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("id2");
163+
request.indexTemplate(
164+
new ComposableIndexTemplate(
165+
List.of("index_*"),
166+
new Template(null, CompressedXContent.fromJSON(mapping), null, null),
167+
null,
168+
null,
169+
null,
170+
null,
171+
null,
172+
null
173+
)
174+
);
175+
client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet();
176+
177+
String indexWithOldOriginationDate = "index_old";
178+
long originTimeMillis = System.currentTimeMillis() - TimeValue.timeValueDays(365).millis();
179+
createIndex(indexWithOldOriginationDate, Settings.builder().put(LIFECYCLE_ORIGINATION_DATE, originTimeMillis).build());
180+
client().execute(
181+
ModifyDataStreamsAction.INSTANCE,
182+
new ModifyDataStreamsAction.Request(List.of(DataStreamAction.addBackingIndex(dataStreamName, indexWithOldOriginationDate)))
183+
).get();
184+
185+
String indexWithNewOriginationDate = "index_new";
186+
originTimeMillis = System.currentTimeMillis() - TimeValue.timeValueDays(1).millis();
187+
createIndex(indexWithNewOriginationDate, Settings.builder().put(LIFECYCLE_ORIGINATION_DATE, originTimeMillis).build());
188+
client().execute(
189+
ModifyDataStreamsAction.INSTANCE,
190+
new ModifyDataStreamsAction.Request(List.of(DataStreamAction.addBackingIndex(dataStreamName, indexWithNewOriginationDate)))
191+
).get();
192+
193+
assertBusy(() -> {
194+
GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName });
195+
GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest)
196+
.actionGet();
197+
assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
198+
assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName));
199+
List<Index> backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices();
200+
Set<String> indexNames = backingIndices.stream().map(index -> index.getName()).collect(Collectors.toSet());
201+
assertTrue(indexNames.contains("index_new"));
202+
assertFalse(indexNames.contains("index_old"));
203+
});
204+
}
205+
133206
public void testUpdatingLifecycleAppliesToAllBackingIndices() throws Exception {
134207
DataLifecycle lifecycle = new DataLifecycle();
135208

‎modules/dlm/src/main/java/org/elasticsearch/dlm/action/TransportExplainDataLifecycleAction.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.elasticsearch.cluster.service.ClusterService;
2626
import org.elasticsearch.common.inject.Inject;
2727
import org.elasticsearch.common.settings.ClusterSettings;
28+
import org.elasticsearch.core.TimeValue;
2829
import org.elasticsearch.dlm.DataLifecycleErrorStore;
2930
import org.elasticsearch.tasks.Task;
3031
import org.elasticsearch.threadpool.ThreadPool;
@@ -87,16 +88,18 @@ protected void masterOperation(
8788
}
8889
DataStream parentDataStream = indexAbstraction.getParentDataStream();
8990
if (parentDataStream == null || parentDataStream.isIndexManagedByDLM(idxMetadata.getIndex(), metadata::index) == false) {
90-
explainIndices.add(new ExplainIndexDataLifecycle(index, false, null, null, null, null));
91+
explainIndices.add(new ExplainIndexDataLifecycle(index, false, null, null, null, null, null));
9192
continue;
9293
}
9394

9495
RolloverInfo rolloverInfo = idxMetadata.getRolloverInfos().get(parentDataStream.getName());
96+
TimeValue generationDate = parentDataStream.getGenerationLifecycleDate(idxMetadata);
9597
ExplainIndexDataLifecycle explainIndexDataLifecycle = new ExplainIndexDataLifecycle(
9698
index,
9799
true,
98100
idxMetadata.getCreationDate(),
99101
rolloverInfo == null ? null : rolloverInfo.getTime(),
102+
generationDate,
100103
parentDataStream.getLifecycle(),
101104
errorStore.getError(index)
102105
);

‎modules/dlm/src/test/java/org/elasticsearch/dlm/action/ExplainDataLifecycleResponseTests.java

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ public void testToXContent() throws IOException {
9494
explainIndexMap.get("time_since_rollover"),
9595
is(explainIndex.getTimeSinceRollover(() -> now).toHumanReadableString(2))
9696
);
97+
}
98+
if (explainIndex.getGenerationTime(() -> now) != null) {
9799
assertThat(
98100
explainIndexMap.get("generation_time"),
99101
is(explainIndex.getGenerationTime(() -> now).toHumanReadableString(2))
@@ -145,6 +147,8 @@ public void testToXContent() throws IOException {
145147
explainIndexMap.get("time_since_rollover"),
146148
is(explainIndex.getTimeSinceRollover(() -> now).toHumanReadableString(2))
147149
);
150+
}
151+
if (explainIndex.getGenerationTime(() -> now) != null) {
148152
assertThat(
149153
explainIndexMap.get("generation_time"),
150154
is(explainIndex.getGenerationTime(() -> now).toHumanReadableString(2))
@@ -161,6 +165,35 @@ public void testToXContent() throws IOException {
161165
assertThat(lifecycleRollover.get("max_primary_shard_docs"), is(9));
162166
}
163167
}
168+
{
169+
// Make sure generation_date is not present if it is null (which it is for a write index):
170+
ExplainIndexDataLifecycle explainIndexWithNullGenerationDate = new ExplainIndexDataLifecycle(
171+
randomAlphaOfLengthBetween(10, 30),
172+
true,
173+
now,
174+
randomBoolean() ? now + TimeValue.timeValueDays(1).getMillis() : null,
175+
null,
176+
lifecycle,
177+
randomBoolean() ? new NullPointerException("bad times").getMessage() : null
178+
);
179+
Response response = new Response(List.of(explainIndexWithNullGenerationDate), null);
180+
181+
XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
182+
response.toXContentChunked(EMPTY_PARAMS).forEachRemaining(xcontent -> {
183+
try {
184+
xcontent.toXContent(builder, EMPTY_PARAMS);
185+
} catch (IOException e) {
186+
logger.error(e.getMessage(), e);
187+
fail(e.getMessage());
188+
}
189+
});
190+
Map<String, Object> xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2();
191+
Map<String, Object> indices = (Map<String, Object>) xContentMap.get("indices");
192+
assertThat(indices.size(), is(1));
193+
Map<String, Object> explainIndexMap = (Map<String, Object>) indices.get(explainIndexWithNullGenerationDate.getIndex());
194+
assertThat(explainIndexMap.get("managed_by_dlm"), is(true));
195+
assertThat(explainIndexMap.get("generation_time"), is(nullValue()));
196+
}
164197
}
165198

166199
public void testChunkCount() {
@@ -186,6 +219,7 @@ private static ExplainIndexDataLifecycle createRandomIndexDLMExplanation(long no
186219
true,
187220
now,
188221
randomBoolean() ? now + TimeValue.timeValueDays(1).getMillis() : null,
222+
randomBoolean() ? TimeValue.timeValueMillis(now) : null,
189223
lifecycle,
190224
randomBoolean() ? new NullPointerException("bad times").getMessage() : null
191225
);

‎server/src/main/java/org/elasticsearch/action/dlm/ExplainIndexDataLifecycle.java

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ public class ExplainIndexDataLifecycle implements Writeable, ToXContentObject {
4646
@Nullable
4747
private final Long rolloverDate;
4848
@Nullable
49+
private final Long generationDateMillis;
50+
@Nullable
4951
private final DataLifecycle lifecycle;
5052
@Nullable
5153
private final String error;
@@ -56,13 +58,15 @@ public ExplainIndexDataLifecycle(
5658
boolean managedByDLM,
5759
@Nullable Long indexCreationDate,
5860
@Nullable Long rolloverDate,
61+
@Nullable TimeValue generationDate,
5962
@Nullable DataLifecycle lifecycle,
6063
@Nullable String error
6164
) {
6265
this.index = index;
6366
this.managedByDLM = managedByDLM;
6467
this.indexCreationDate = indexCreationDate;
6568
this.rolloverDate = rolloverDate;
69+
this.generationDateMillis = generationDate == null ? null : generationDate.millis();
6670
this.lifecycle = lifecycle;
6771
this.error = error;
6872
}
@@ -73,11 +77,13 @@ public ExplainIndexDataLifecycle(StreamInput in) throws IOException {
7377
if (managedByDLM) {
7478
this.indexCreationDate = in.readOptionalLong();
7579
this.rolloverDate = in.readOptionalLong();
80+
this.generationDateMillis = in.readOptionalLong();
7681
this.lifecycle = in.readOptionalWriteable(DataLifecycle::new);
7782
this.error = in.readOptionalString();
7883
} else {
7984
this.indexCreationDate = null;
8085
this.rolloverDate = null;
86+
this.generationDateMillis = null;
8187
this.lifecycle = null;
8288
this.error = null;
8389
}
@@ -108,7 +114,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla
108114
if (rolloverDate != null) {
109115
builder.timeField(ROLLOVER_DATE_MILLIS_FIELD.getPreferredName(), ROLLOVER_DATE_FIELD.getPreferredName(), rolloverDate);
110116
builder.field(TIME_SINCE_ROLLOVER_FIELD.getPreferredName(), getTimeSinceRollover(nowSupplier).toHumanReadableString(2));
111-
// if the index has been rolled over we'll start reporting the generation time
117+
}
118+
if (generationDateMillis != null) {
112119
builder.field(GENERATION_TIME.getPreferredName(), getGenerationTime(nowSupplier).toHumanReadableString(2));
113120
}
114121
if (this.lifecycle != null) {
@@ -130,24 +137,24 @@ public void writeTo(StreamOutput out) throws IOException {
130137
if (managedByDLM) {
131138
out.writeOptionalLong(indexCreationDate);
132139
out.writeOptionalLong(rolloverDate);
140+
out.writeOptionalLong(generationDateMillis);
133141
out.writeOptionalWriteable(lifecycle);
134142
out.writeOptionalString(error);
135143
}
136144
}
137145

138146
/**
139147
* Calculates the time since this index started progressing towards the remaining of its lifecycle past rollover.
140-
* Every index will have to wait to be rolled over before progressing towards its retention part of its lifecycle.
141-
* If the index has not been rolled over this will return null.
142-
* In the future, this will also consider the origination date of the index (however, it'll again only be displayed
143-
* after the index is rolled over).
148+
* Every index will either have to wait to be rolled over before progressing towards its retention part of its lifecycle,
149+
* or be added to the datastream manually.
150+
* If the index is the write index this will return null.
144151
*/
145152
@Nullable
146153
public TimeValue getGenerationTime(Supplier<Long> now) {
147-
if (rolloverDate == null) {
154+
if (generationDateMillis == null) {
148155
return null;
149156
}
150-
return TimeValue.timeValueMillis(Math.max(0L, now.get() - rolloverDate));
157+
return TimeValue.timeValueMillis(Math.max(0L, now.get() - generationDateMillis));
151158
}
152159

153160
/**

0 commit comments

Comments
 (0)