Skip to content

Commit 430dbd4

Browse files
szybiaszybia
committed
Component Templates: Add {created,modified}_date 
1 parent 929f65b commit 430dbd4

File tree

18 files changed

+371
-48
lines changed

18 files changed

+371
-48
lines changed

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
3535
import org.elasticsearch.test.ESSingleNodeTestCase;
3636

37+
import java.time.Instant;
3738
import java.util.ArrayList;
3839
import java.util.Collections;
3940
import java.util.List;
@@ -216,7 +217,8 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() {
216217
xContentRegistry(),
217218
EmptySystemIndices.INSTANCE,
218219
indexSettingProviders,
219-
DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings())
220+
DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()),
221+
Instant::now
220222
);
221223
}
222224

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -644,6 +644,8 @@ private static ProjectMetadata getProjectWithDataStreamWithSettings(
644644
Template.builder().settings(componentTemplateSettings).build(),
645645
null,
646646
null,
647+
null,
648+
null,
647649
null
648650
);
649651
builder.componentTemplates(Map.of("component_template_1", componentTemplate));

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
setup:
2+
- requires:
3+
test_runner_features: capabilities
4+
capabilities:
5+
- method: PUT
6+
path: /_component_template/{id}
7+
capabilities: [ component_template_tracking_info ]
8+
reason: "Templates have tracking info: modified_date and created_date"
9+
10+
---
11+
"Test PUT setting created_date":
12+
- do:
13+
catch: bad_request
14+
cluster.put_component_template:
15+
name: test_tracking
16+
body:
17+
template:
18+
settings:
19+
number_of_shards: 1
20+
created_date: "2025-07-04T12:50:48.415Z"
21+
- match: { status: 400 }
22+
- match: { error.reason: "Validation Failed: 1: Provided a pipeline property which is managed by the system: created_date.;" }
23+
24+
---
25+
"Test PUT setting modified_date":
26+
- do:
27+
catch: bad_request
28+
cluster.put_component_template:
29+
name: test_tracking
30+
body:
31+
template:
32+
settings:
33+
number_of_shards: 1
34+
modified_date: "2025-07-04T12:50:48.415Z"
35+
- match: { status: 400 }
36+
- match: { error.reason: "Validation Failed: 1: Provided a pipeline property which is managed by the system: modified_date.;" }
37+
38+
---
39+
"Test update preserves created_date but updates modified_date":
40+
- do:
41+
cluster.put_component_template:
42+
name: test_tracking
43+
body:
44+
template:
45+
settings:
46+
number_of_shards: 1
47+
- match: { acknowledged: true }
48+
49+
- do:
50+
cluster.get_component_template:
51+
name: test_tracking
52+
- set: { component_templates.0.component_template.created_date: first_created }
53+
- set: { component_templates.0.component_template.modified_date: first_modified }
54+
- match: { $first_created: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" }
55+
- match: { $first_modified: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" }
56+
57+
- do:
58+
cluster.put_component_template:
59+
name: test_tracking
60+
body:
61+
template:
62+
settings:
63+
number_of_shards: 2
64+
65+
- do:
66+
cluster.get_component_template:
67+
name: test_tracking
68+
- set: { component_templates.0.component_template.created_date: second_created }
69+
- set: { component_templates.0.component_template.modified_date: second_modified }
70+
- match: { $second_created: $first_created }
71+
- gt: { $second_modified: $first_modified }
72+

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,7 @@ static TransportVersion def(int id) {
342342
public static final TransportVersion NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO = def(9_121_0_00);
343343
public static final TransportVersion ESQL_CATEGORIZE_OPTIONS = def(9_122_0_00);
344344
public static final TransportVersion ML_INFERENCE_AZURE_AI_STUDIO_RERANK_ADDED = def(9_123_0_00);
345+
public static final TransportVersion COMPONENT_TEMPLATE_TRACKING_INFO = def(9_124_0_00);
345346

346347
/*
347348
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,18 @@ public ActionRequestValidationException validate() {
7979
if (componentTemplate == null) {
8080
validationException = addValidationError("a component template is required", validationException);
8181
}
82+
if (componentTemplate.createdDateMillis().isPresent()) {
83+
validationException = addValidationError(
84+
"Provided a pipeline property which is managed by the system: created_date.",
85+
validationException
86+
);
87+
}
88+
if (componentTemplate.modifiedDateMillis().isPresent()) {
89+
validationException = addValidationError(
90+
"Provided a pipeline property which is managed by the system: modified_date.",
91+
validationException
92+
);
93+
}
8294
return validationException;
8395
}
8496

‎server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,9 @@ public static ComponentTemplate normalizeComponentTemplate(
8585
template,
8686
componentTemplate.version(),
8787
componentTemplate.metadata(),
88-
componentTemplate.deprecated()
88+
componentTemplate.deprecated(),
89+
componentTemplate.createdDateMillis().orElse(null),
90+
componentTemplate.modifiedDateMillis().orElse(null)
8991
);
9092
}
9193

‎server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java‎

Lines changed: 69 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,13 @@
2424
import org.elasticsearch.xcontent.XContentParser;
2525

2626
import java.io.IOException;
27+
import java.time.Instant;
28+
import java.time.format.DateTimeFormatter;
29+
import java.time.format.DateTimeFormatterBuilder;
30+
import java.util.Locale;
2731
import java.util.Map;
2832
import java.util.Objects;
33+
import java.util.Optional;
2934

3035
/**
3136
* A component template is a re-usable {@link Template} as well as metadata about the template. Each
@@ -34,23 +39,43 @@
3439
* "foo" field. These component templates make up the individual pieces composing an index template.
3540
*/
3641
public class ComponentTemplate implements SimpleDiffable<ComponentTemplate>, ToXContentObject {
42+
43+
// always output millis even if instantSource returns millis == 0
44+
private static final DateTimeFormatter ISO8601_WITH_MILLIS_FORMATTER = new DateTimeFormatterBuilder().appendInstant(3)
45+
.toFormatter(Locale.ROOT);
46+
3747
private static final ParseField TEMPLATE = new ParseField("template");
3848
private static final ParseField VERSION = new ParseField("version");
3949
private static final ParseField METADATA = new ParseField("_meta");
4050
private static final ParseField DEPRECATED = new ParseField("deprecated");
51+
private static final ParseField CREATED_DATE = new ParseField("created_date");
52+
private static final ParseField MODIFIED_DATE = new ParseField("modified_date");
4153

4254
@SuppressWarnings("unchecked")
4355
public static final ConstructingObjectParser<ComponentTemplate, Void> PARSER = new ConstructingObjectParser<>(
4456
"component_template",
4557
false,
46-
a -> new ComponentTemplate((Template) a[0], (Long) a[1], (Map<String, Object>) a[2], (Boolean) a[3])
58+
a -> {
59+
final String createdDate = (String) a[4];
60+
final String modifiedDate = (String) a[5];
61+
return new ComponentTemplate(
62+
(Template) a[0],
63+
(Long) a[1],
64+
(Map<String, Object>) a[2],
65+
(Boolean) a[3],
66+
createdDate == null ? null : Instant.parse(createdDate).toEpochMilli(),
67+
modifiedDate == null ? null : Instant.parse(modifiedDate).toEpochMilli()
68+
);
69+
}
4770
);
4871

4972
static {
5073
PARSER.declareObject(ConstructingObjectParser.constructorArg(), Template.PARSER, TEMPLATE);
5174
PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION);
5275
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA);
5376
PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), DEPRECATED);
77+
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), CREATED_DATE);
78+
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), MODIFIED_DATE);
5479
}
5580

5681
private final Template template;
@@ -60,6 +85,10 @@ public class ComponentTemplate implements SimpleDiffable<ComponentTemplate>, ToX
6085
private final Map<String, Object> metadata;
6186
@Nullable
6287
private final Boolean deprecated;
88+
@Nullable
89+
private final Long createdDateMillis;
90+
@Nullable
91+
private final Long modifiedDateMillis;
6392

6493
static Diff<ComponentTemplate> readComponentTemplateDiffFrom(StreamInput in) throws IOException {
6594
return SimpleDiffable.readDiffFrom(ComponentTemplate::new, in);
@@ -70,19 +99,23 @@ public static ComponentTemplate parse(XContentParser parser) {
7099
}
71100

72101
public ComponentTemplate(Template template, @Nullable Long version, @Nullable Map<String, Object> metadata) {
73-
this(template, version, metadata, null);
102+
this(template, version, metadata, null, null, null);
74103
}
75104

76105
public ComponentTemplate(
77106
Template template,
78107
@Nullable Long version,
79108
@Nullable Map<String, Object> metadata,
80-
@Nullable Boolean deprecated
109+
@Nullable Boolean deprecated,
110+
@Nullable Long createdDateMillis,
111+
@Nullable Long modifiedDateMillis
81112
) {
82113
this.template = template;
83114
this.version = version;
84115
this.metadata = metadata;
85116
this.deprecated = deprecated;
117+
this.createdDateMillis = createdDateMillis;
118+
this.modifiedDateMillis = modifiedDateMillis;
86119
}
87120

88121
public ComponentTemplate(StreamInput in) throws IOException {
@@ -98,6 +131,13 @@ public ComponentTemplate(StreamInput in) throws IOException {
98131
} else {
99132
deprecated = null;
100133
}
134+
if (in.getTransportVersion().onOrAfter(TransportVersions.COMPONENT_TEMPLATE_TRACKING_INFO)) {
135+
this.createdDateMillis = in.readOptionalLong();
136+
this.modifiedDateMillis = in.readOptionalLong();
137+
} else {
138+
this.createdDateMillis = null;
139+
this.modifiedDateMillis = null;
140+
}
101141
}
102142

103143
public Template template() {
@@ -122,6 +162,14 @@ public boolean isDeprecated() {
122162
return Boolean.TRUE.equals(deprecated);
123163
}
124164

165+
public Optional<Long> createdDateMillis() {
166+
return Optional.ofNullable(createdDateMillis);
167+
}
168+
169+
public Optional<Long> modifiedDateMillis() {
170+
return Optional.ofNullable(modifiedDateMillis);
171+
}
172+
125173
@Override
126174
public void writeTo(StreamOutput out) throws IOException {
127175
this.template.writeTo(out);
@@ -135,11 +183,15 @@ public void writeTo(StreamOutput out) throws IOException {
135183
if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) {
136184
out.writeOptionalBoolean(this.deprecated);
137185
}
186+
if (out.getTransportVersion().onOrAfter(TransportVersions.COMPONENT_TEMPLATE_TRACKING_INFO)) {
187+
out.writeOptionalLong(this.createdDateMillis);
188+
out.writeOptionalLong(this.modifiedDateMillis);
189+
}
138190
}
139191

140192
@Override
141193
public int hashCode() {
142-
return Objects.hash(template, version, metadata, deprecated);
194+
return Objects.hash(template, version, metadata, deprecated, createdDateMillis, modifiedDateMillis);
143195
}
144196

145197
@Override
@@ -154,7 +206,9 @@ public boolean equals(Object obj) {
154206
return Objects.equals(template, other.template)
155207
&& Objects.equals(version, other.version)
156208
&& Objects.equals(metadata, other.metadata)
157-
&& Objects.equals(deprecated, other.deprecated);
209+
&& Objects.equals(deprecated, other.deprecated)
210+
&& Objects.equals(createdDateMillis, other.createdDateMillis)
211+
&& Objects.equals(modifiedDateMillis, other.modifiedDateMillis);
158212
}
159213

160214
@Override
@@ -184,7 +238,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla
184238
if (this.deprecated != null) {
185239
builder.field(DEPRECATED.getPreferredName(), this.deprecated);
186240
}
241+
if (this.createdDateMillis != null) {
242+
builder.field(CREATED_DATE.getPreferredName(), formatDate(this.createdDateMillis));
243+
}
244+
if (this.modifiedDateMillis != null) {
245+
builder.field(MODIFIED_DATE.getPreferredName(), formatDate(this.modifiedDateMillis));
246+
}
187247
builder.endObject();
188248
return builder;
189249
}
250+
251+
private static String formatDate(long epochMillis) {
252+
return ISO8601_WITH_MILLIS_FORMATTER.format(Instant.ofEpochMilli(epochMillis));
253+
}
190254
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java‎

Lines changed: 37 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
import org.elasticsearch.xcontent.NamedXContentRegistry;
5858

5959
import java.io.IOException;
60-
import java.time.Instant;
60+
import java.time.InstantSource;
6161
import java.util.ArrayList;
6262
import java.util.Arrays;
6363
import java.util.Collection;
@@ -140,6 +140,7 @@ public class MetadataIndexTemplateService {
140140
private final SystemIndices systemIndices;
141141
private final Set<IndexSettingProvider> indexSettingProviders;
142142
private final DataStreamGlobalRetentionSettings globalRetentionSettings;
143+
private final InstantSource instantSource;
143144

144145
/**
145146
* This is the cluster state task executor for all template-based actions.
@@ -193,7 +194,8 @@ public MetadataIndexTemplateService(
193194
NamedXContentRegistry xContentRegistry,
194195
SystemIndices systemIndices,
195196
IndexSettingProviders indexSettingProviders,
196-
DataStreamGlobalRetentionSettings globalRetentionSettings
197+
DataStreamGlobalRetentionSettings globalRetentionSettings,
198+
InstantSource instantSource
197199
) {
198200
this.clusterService = clusterService;
199201
this.taskQueue = clusterService.createTaskQueue("index-templates", Priority.URGENT, TEMPLATE_TASK_EXECUTOR);
@@ -204,6 +206,7 @@ public MetadataIndexTemplateService(
204206
this.systemIndices = systemIndices;
205207
this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders();
206208
this.globalRetentionSettings = globalRetentionSettings;
209+
this.instantSource = instantSource;
207210
}
208211

209212
public void removeTemplates(
@@ -323,15 +326,37 @@ public ProjectMetadata addComponentTemplate(
323326
}
324327

325328
final Template finalTemplate = Template.builder(template.template()).settings(finalSettings).mappings(wrappedMappings).build();
326-
final ComponentTemplate finalComponentTemplate = new ComponentTemplate(
327-
finalTemplate,
328-
template.version(),
329-
template.metadata(),
330-
template.deprecated()
331-
);
332-
333-
if (finalComponentTemplate.equals(existing)) {
334-
return project;
329+
final long now = instantSource.instant().toEpochMilli();
330+
final ComponentTemplate finalComponentTemplate;
331+
if (existing == null) {
332+
finalComponentTemplate = new ComponentTemplate(
333+
finalTemplate,
334+
template.version(),
335+
template.metadata(),
336+
template.deprecated(),
337+
now,
338+
now
339+
);
340+
} else {
341+
final ComponentTemplate templateToCompareToExisting = new ComponentTemplate(
342+
finalTemplate,
343+
template.version(),
344+
template.metadata(),
345+
template.deprecated(),
346+
existing.createdDateMillis().orElse(null),
347+
existing.modifiedDateMillis().orElse(null)
348+
);
349+
if (templateToCompareToExisting.equals(existing)) {
350+
return project;
351+
}
352+
finalComponentTemplate = new ComponentTemplate(
353+
finalTemplate,
354+
template.version(),
355+
template.metadata(),
356+
template.deprecated(),
357+
existing.createdDateMillis().orElse(null),
358+
now
359+
);
335360
}
336361

337362
validateTemplate(finalSettings, wrappedMappings, indicesService);
@@ -715,7 +740,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo
715740
// Workaround for the fact that start_time and end_time are injected by the MetadataCreateDataStreamService upon creation,
716741
// but when validating templates that create data streams the MetadataCreateDataStreamService isn't used.
717742
var finalTemplate = indexTemplate.template();
718-
final var now = Instant.now();
743+
final var now = instantSource.instant();
719744

720745
final var combinedMappings = collectMappings(indexTemplate, projectMetadata.componentTemplates(), "tmp_idx");
721746
final var combinedSettings = resolveSettings(indexTemplate, projectMetadata.componentTemplates());

0 commit comments

Comments
 (0)