From 430dbd4b725428699b667e668520a5f993b80614 Mon Sep 17 00:00:00 2001 From: Szymon Bialkowski Date: Fri, 18 Jul 2025 15:29:12 +0100 Subject: [PATCH 1/6] Component Templates: Add `{created,modified}_date` --- .../MetadataIndexTemplateServiceTests.java | 4 +- .../TransportGetDataStreamsActionTests.java | 2 + .../20_tracking.yml | 72 ++++++++++++ .../org/elasticsearch/TransportVersions.java | 1 + .../put/PutComponentTemplateAction.java | 12 ++ .../TransportPutComponentTemplateAction.java | 4 +- .../cluster/metadata/ComponentTemplate.java | 74 ++++++++++++- .../MetadataIndexTemplateService.java | 49 +++++++-- .../elasticsearch/node/NodeConstruction.java | 6 +- .../RestPutComponentTemplateAction.java | 9 +- .../GetComponentTemplateResponseTests.java | 4 +- ...vedComposableIndexTemplateActionTests.java | 7 +- .../metadata/ComponentTemplateTests.java | 48 ++++++-- ...amLifecycleWithRetentionWarningsTests.java | 14 ++- .../cluster/metadata/DataStreamTests.java | 2 +- .../MetadataIndexTemplateServiceTests.java | 103 ++++++++++++++++-- .../TransportDeprecationInfoAction.java | 4 +- ...adataMigrateToDataTiersRoutingService.java | 4 +- 18 files changed, 371 insertions(+), 48 deletions(-) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index 8a6d510ad1bd8..bda28c5d74b37 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.ESSingleNodeTestCase; +import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -216,7 +217,8 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { xContentRegistry(), EmptySystemIndices.INSTANCE, indexSettingProviders, - DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()) + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()), + Instant::now ); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index d23edf5e91045..6e6251b7d7fd1 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -644,6 +644,8 @@ private static ProjectMetadata getProjectWithDataStreamWithSettings( Template.builder().settings(componentTemplateSettings).build(), null, null, + null, + null, null ); builder.componentTemplates(Map.of("component_template_1", componentTemplate)); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml new file mode 100644 index 0000000000000..94e1919328f9d --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml @@ -0,0 +1,72 @@ +setup: + - requires: + test_runner_features: capabilities + capabilities: + - method: PUT + path: /_component_template/{id} + capabilities: [ component_template_tracking_info ] + reason: "Templates have tracking info: modified_date and created_date" + +--- +"Test PUT setting created_date": + - do: + catch: bad_request + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 1 + created_date: "2025-07-04T12:50:48.415Z" + - match: { status: 400 } + - match: { error.reason: "Validation Failed: 1: Provided a pipeline property which is managed by the system: created_date.;" } + +--- +"Test PUT setting modified_date": + - do: + catch: bad_request + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 1 + modified_date: "2025-07-04T12:50:48.415Z" + - match: { status: 400 } + - match: { error.reason: "Validation Failed: 1: Provided a pipeline property which is managed by the system: modified_date.;" } + +--- +"Test update preserves created_date but updates modified_date": + - do: + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 1 + - match: { acknowledged: true } + + - do: + cluster.get_component_template: + name: test_tracking + - set: { component_templates.0.component_template.created_date: first_created } + - set: { component_templates.0.component_template.modified_date: first_modified } + - match: { $first_created: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" } + - match: { $first_modified: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" } + + - do: + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 2 + + - do: + cluster.get_component_template: + name: test_tracking + - set: { component_templates.0.component_template.created_date: second_created } + - set: { component_templates.0.component_template.modified_date: second_modified } + - match: { $second_created: $first_created } + - gt: { $second_modified: $first_modified } + diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 11a0103cd22e0..5e8563f790e45 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -342,6 +342,7 @@ static TransportVersion def(int id) { public static final TransportVersion NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO = def(9_121_0_00); public static final TransportVersion ESQL_CATEGORIZE_OPTIONS = def(9_122_0_00); public static final TransportVersion ML_INFERENCE_AZURE_AI_STUDIO_RERANK_ADDED = def(9_123_0_00); + public static final TransportVersion COMPONENT_TEMPLATE_TRACKING_INFO = def(9_124_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java index 2c34359dabbad..ec42e4e2429d3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java @@ -79,6 +79,18 @@ public ActionRequestValidationException validate() { if (componentTemplate == null) { validationException = addValidationError("a component template is required", validationException); } + if (componentTemplate.createdDateMillis().isPresent()) { + validationException = addValidationError( + "Provided a pipeline property which is managed by the system: created_date.", + validationException + ); + } + if (componentTemplate.modifiedDateMillis().isPresent()) { + validationException = addValidationError( + "Provided a pipeline property which is managed by the system: modified_date.", + validationException + ); + } return validationException; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index 05d055dbf979c..79a8bffa1e1b6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -85,7 +85,9 @@ public static ComponentTemplate normalizeComponentTemplate( template, componentTemplate.version(), componentTemplate.metadata(), - componentTemplate.deprecated() + componentTemplate.deprecated(), + componentTemplate.createdDateMillis().orElse(null), + componentTemplate.modifiedDateMillis().orElse(null) ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java index 016fec60f06be..e526e3328f019 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java @@ -24,8 +24,13 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * A component template is a re-usable {@link Template} as well as metadata about the template. Each @@ -34,16 +39,34 @@ * "foo" field. These component templates make up the individual pieces composing an index template. */ public class ComponentTemplate implements SimpleDiffable, ToXContentObject { + + // always output millis even if instantSource returns millis == 0 + private static final DateTimeFormatter ISO8601_WITH_MILLIS_FORMATTER = new DateTimeFormatterBuilder().appendInstant(3) + .toFormatter(Locale.ROOT); + private static final ParseField TEMPLATE = new ParseField("template"); private static final ParseField VERSION = new ParseField("version"); private static final ParseField METADATA = new ParseField("_meta"); private static final ParseField DEPRECATED = new ParseField("deprecated"); + private static final ParseField CREATED_DATE = new ParseField("created_date"); + private static final ParseField MODIFIED_DATE = new ParseField("modified_date"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "component_template", false, - a -> new ComponentTemplate((Template) a[0], (Long) a[1], (Map) a[2], (Boolean) a[3]) + a -> { + final String createdDate = (String) a[4]; + final String modifiedDate = (String) a[5]; + return new ComponentTemplate( + (Template) a[0], + (Long) a[1], + (Map) a[2], + (Boolean) a[3], + createdDate == null ? null : Instant.parse(createdDate).toEpochMilli(), + modifiedDate == null ? null : Instant.parse(modifiedDate).toEpochMilli() + ); + } ); static { @@ -51,6 +74,8 @@ public class ComponentTemplate implements SimpleDiffable, ToX PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), DEPRECATED); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), CREATED_DATE); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), MODIFIED_DATE); } private final Template template; @@ -60,6 +85,10 @@ public class ComponentTemplate implements SimpleDiffable, ToX private final Map metadata; @Nullable private final Boolean deprecated; + @Nullable + private final Long createdDateMillis; + @Nullable + private final Long modifiedDateMillis; static Diff readComponentTemplateDiffFrom(StreamInput in) throws IOException { return SimpleDiffable.readDiffFrom(ComponentTemplate::new, in); @@ -70,19 +99,23 @@ public static ComponentTemplate parse(XContentParser parser) { } public ComponentTemplate(Template template, @Nullable Long version, @Nullable Map metadata) { - this(template, version, metadata, null); + this(template, version, metadata, null, null, null); } public ComponentTemplate( Template template, @Nullable Long version, @Nullable Map metadata, - @Nullable Boolean deprecated + @Nullable Boolean deprecated, + @Nullable Long createdDateMillis, + @Nullable Long modifiedDateMillis ) { this.template = template; this.version = version; this.metadata = metadata; this.deprecated = deprecated; + this.createdDateMillis = createdDateMillis; + this.modifiedDateMillis = modifiedDateMillis; } public ComponentTemplate(StreamInput in) throws IOException { @@ -98,6 +131,13 @@ public ComponentTemplate(StreamInput in) throws IOException { } else { deprecated = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.COMPONENT_TEMPLATE_TRACKING_INFO)) { + this.createdDateMillis = in.readOptionalLong(); + this.modifiedDateMillis = in.readOptionalLong(); + } else { + this.createdDateMillis = null; + this.modifiedDateMillis = null; + } } public Template template() { @@ -122,6 +162,14 @@ public boolean isDeprecated() { return Boolean.TRUE.equals(deprecated); } + public Optional createdDateMillis() { + return Optional.ofNullable(createdDateMillis); + } + + public Optional modifiedDateMillis() { + return Optional.ofNullable(modifiedDateMillis); + } + @Override public void writeTo(StreamOutput out) throws IOException { this.template.writeTo(out); @@ -135,11 +183,15 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalBoolean(this.deprecated); } + if (out.getTransportVersion().onOrAfter(TransportVersions.COMPONENT_TEMPLATE_TRACKING_INFO)) { + out.writeOptionalLong(this.createdDateMillis); + out.writeOptionalLong(this.modifiedDateMillis); + } } @Override public int hashCode() { - return Objects.hash(template, version, metadata, deprecated); + return Objects.hash(template, version, metadata, deprecated, createdDateMillis, modifiedDateMillis); } @Override @@ -154,7 +206,9 @@ public boolean equals(Object obj) { return Objects.equals(template, other.template) && Objects.equals(version, other.version) && Objects.equals(metadata, other.metadata) - && Objects.equals(deprecated, other.deprecated); + && Objects.equals(deprecated, other.deprecated) + && Objects.equals(createdDateMillis, other.createdDateMillis) + && Objects.equals(modifiedDateMillis, other.modifiedDateMillis); } @Override @@ -184,7 +238,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla if (this.deprecated != null) { builder.field(DEPRECATED.getPreferredName(), this.deprecated); } + if (this.createdDateMillis != null) { + builder.field(CREATED_DATE.getPreferredName(), formatDate(this.createdDateMillis)); + } + if (this.modifiedDateMillis != null) { + builder.field(MODIFIED_DATE.getPreferredName(), formatDate(this.modifiedDateMillis)); + } builder.endObject(); return builder; } + + private static String formatDate(long epochMillis) { + return ISO8601_WITH_MILLIS_FORMATTER.format(Instant.ofEpochMilli(epochMillis)); + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 69dd4359fd764..ed26339bccd5c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -57,7 +57,7 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; -import java.time.Instant; +import java.time.InstantSource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -140,6 +140,7 @@ public class MetadataIndexTemplateService { private final SystemIndices systemIndices; private final Set indexSettingProviders; private final DataStreamGlobalRetentionSettings globalRetentionSettings; + private final InstantSource instantSource; /** * This is the cluster state task executor for all template-based actions. @@ -193,7 +194,8 @@ public MetadataIndexTemplateService( NamedXContentRegistry xContentRegistry, SystemIndices systemIndices, IndexSettingProviders indexSettingProviders, - DataStreamGlobalRetentionSettings globalRetentionSettings + DataStreamGlobalRetentionSettings globalRetentionSettings, + InstantSource instantSource ) { this.clusterService = clusterService; this.taskQueue = clusterService.createTaskQueue("index-templates", Priority.URGENT, TEMPLATE_TASK_EXECUTOR); @@ -204,6 +206,7 @@ public MetadataIndexTemplateService( this.systemIndices = systemIndices; this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders(); this.globalRetentionSettings = globalRetentionSettings; + this.instantSource = instantSource; } public void removeTemplates( @@ -323,15 +326,37 @@ public ProjectMetadata addComponentTemplate( } final Template finalTemplate = Template.builder(template.template()).settings(finalSettings).mappings(wrappedMappings).build(); - final ComponentTemplate finalComponentTemplate = new ComponentTemplate( - finalTemplate, - template.version(), - template.metadata(), - template.deprecated() - ); - - if (finalComponentTemplate.equals(existing)) { - return project; + final long now = instantSource.instant().toEpochMilli(); + final ComponentTemplate finalComponentTemplate; + if (existing == null) { + finalComponentTemplate = new ComponentTemplate( + finalTemplate, + template.version(), + template.metadata(), + template.deprecated(), + now, + now + ); + } else { + final ComponentTemplate templateToCompareToExisting = new ComponentTemplate( + finalTemplate, + template.version(), + template.metadata(), + template.deprecated(), + existing.createdDateMillis().orElse(null), + existing.modifiedDateMillis().orElse(null) + ); + if (templateToCompareToExisting.equals(existing)) { + return project; + } + finalComponentTemplate = new ComponentTemplate( + finalTemplate, + template.version(), + template.metadata(), + template.deprecated(), + existing.createdDateMillis().orElse(null), + now + ); } validateTemplate(finalSettings, wrappedMappings, indicesService); @@ -715,7 +740,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo // Workaround for the fact that start_time and end_time are injected by the MetadataCreateDataStreamService upon creation, // but when validating templates that create data streams the MetadataCreateDataStreamService isn't used. var finalTemplate = indexTemplate.template(); - final var now = Instant.now(); + final var now = instantSource.instant(); final var combinedMappings = collectMappings(indexTemplate, projectMetadata.componentTemplates(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, projectMetadata.componentTemplates()); diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index bb28ed4a8aff5..f96d9bfb72d05 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -228,6 +228,8 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Instant; +import java.time.InstantSource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1300,6 +1302,7 @@ public Map queryFields() { b.bind(ShutdownPrepareService.class).toInstance(shutdownPrepareService); b.bind(OnlinePrewarmingService.class).toInstance(onlinePrewarmingService); b.bind(MergeMetrics.class).toInstance(mergeMetrics); + b.bind(InstantSource.class).toInstance(Instant::now); }); if (ReadinessService.enabled(environment)) { @@ -1655,7 +1658,8 @@ private List> buildReservedProjectStateHandlers( xContentRegistry, systemIndices, indexSettingProviders, - globalRetentionSettings + globalRetentionSettings, + Instant::now ); reservedStateHandlers.add(new ReservedComposableIndexTemplateAction(templateService, settingsModule.getIndexScopedSettings())); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java index 8e3112510642a..aff4287ecbc58 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java @@ -31,7 +31,12 @@ public class RestPutComponentTemplateAction extends BaseRestHandler { public static final String SUPPORTS_FAILURE_STORE_LIFECYCLE = "data_stream_options.failure_store.lifecycle"; public static final String SUPPORTS_FAILURE_STORE = "data_stream_options.failure_store"; - private static final Set capabilities = Set.of(SUPPORTS_FAILURE_STORE, SUPPORTS_FAILURE_STORE_LIFECYCLE); + private static final String COMPONENT_TEMPLATE_TRACKING_INFO = "component_template_tracking_info"; + private static final Set CAPABILITIES = Set.of( + SUPPORTS_FAILURE_STORE, + SUPPORTS_FAILURE_STORE_LIFECYCLE, + COMPONENT_TEMPLATE_TRACKING_INFO + ); @Override public List routes() { @@ -59,6 +64,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC @Override public Set supportedCapabilities() { - return capabilities; + return CAPABILITIES; } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java index 289e2795edf11..ac7063d8d4e36 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java @@ -58,7 +58,9 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws new Template(settings, mappings, aliases, lifecycle, dataStreamOptions), randomBoolean() ? null : randomNonNegativeLong(), null, - false + false, + null, + null ); var rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions(); var response = new GetComponentTemplateAction.Response(Map.of(randomAlphaOfLength(10), template), rolloverConfiguration); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index cb9fa23aaefbc..5cbb2879840cc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -54,6 +54,7 @@ import org.junit.Before; import java.io.IOException; +import java.time.Instant; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -105,7 +106,8 @@ public void setup() throws IOException { mock(NamedXContentRegistry.class), mock(SystemIndices.class), new IndexSettingProviders(Set.of()), - globalRetentionSettings + globalRetentionSettings, + Instant::now ); } @@ -896,7 +898,8 @@ public void testTemplatesWithReservedPrefix() throws Exception { mock(NamedXContentRegistry.class), mock(SystemIndices.class), new IndexSettingProviders(Set.of()), - globalRetentionSettings + globalRetentionSettings, + Instant::now ); ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java index 2a7eb712da344..14dbe3079a842 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java @@ -101,7 +101,21 @@ public static ComponentTemplate randomInstance(boolean supportsDataStreams, Bool if (randomBoolean()) { meta = randomMeta(); } - return new ComponentTemplate(template, randomBoolean() ? null : randomNonNegativeLong(), meta, deprecated); + final Long createdDate = randomBoolean() ? randomNonNegativeLong() : null; + final Long modifiedDate; + if (randomBoolean()) { + modifiedDate = createdDate == null ? randomNonNegativeLong() : randomLongBetween(createdDate, Long.MAX_VALUE); + } else { + modifiedDate = null; + } + return new ComponentTemplate( + template, + randomBoolean() ? null : randomNonNegativeLong(), + meta, + deprecated, + createdDate, + modifiedDate + ); } public static ResettableValue randomDataStreamOptionsTemplate() { @@ -166,19 +180,25 @@ yield switch (randomIntBetween(0, 4)) { Template.builder(ot).settings(randomValueOtherThan(ot.settings(), ComponentTemplateTests::randomSettings)).build(), orig.version(), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 1 -> new ComponentTemplate( Template.builder(ot).mappings(randomValueOtherThan(ot.mappings(), ComponentTemplateTests::randomMappings)).build(), orig.version(), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 2 -> new ComponentTemplate( Template.builder(ot).aliases(randomValueOtherThan(ot.aliases(), ComponentTemplateTests::randomAliases)).build(), orig.version(), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 3 -> new ComponentTemplate( Template.builder(ot) @@ -186,7 +206,9 @@ yield switch (randomIntBetween(0, 4)) { .build(), orig.version(), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 4 -> new ComponentTemplate( Template.builder(ot) @@ -196,7 +218,9 @@ yield switch (randomIntBetween(0, 4)) { .build(), orig.version(), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); default -> throw new IllegalStateException("illegal randomization branch"); }; @@ -205,19 +229,25 @@ yield switch (randomIntBetween(0, 4)) { orig.template(), randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 2 -> new ComponentTemplate( orig.template(), orig.version(), randomValueOtherThan(orig.metadata(), ComponentTemplateTests::randomMeta), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 3 -> new ComponentTemplate( orig.template(), orig.version(), orig.metadata(), - orig.isDeprecated() ? randomFrom(false, null) : true + orig.isDeprecated() ? randomFrom(false, null) : true, + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); default -> throw new IllegalStateException("illegal randomization branch"); }; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java index a041dadb005f3..fe11099d2d60f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java @@ -25,11 +25,14 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESTestCase; +import java.time.Instant; +import java.time.InstantSource; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.cluster.metadata.DataStreamLifecycleTests.randomDownsampling; import static org.elasticsearch.common.settings.Settings.builder; @@ -265,6 +268,8 @@ public void testValidateLifecycleInComponentTemplate() throws Exception { .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey(), defaultRetention) .build(); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + AtomicInteger instantSourceInvocationCounter = new AtomicInteger(); + InstantSource instantSource = () -> Instant.ofEpochMilli(instantSourceInvocationCounter.getAndIncrement()); MetadataIndexTemplateService metadataIndexTemplateService = new MetadataIndexTemplateService( clusterService, createIndexService, @@ -273,7 +278,8 @@ public void testValidateLifecycleInComponentTemplate() throws Exception { xContentRegistry(), EmptySystemIndices.INSTANCE, new IndexSettingProviders(Set.of()), - DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings(settingsWithDefaultRetention)) + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings(settingsWithDefaultRetention)), + instantSource ); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); @@ -287,8 +293,10 @@ public void testValidateLifecycleInComponentTemplate() throws Exception { ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>()); project = metadataIndexTemplateService.addComponentTemplate(project, false, "foo", componentTemplate); - assertNotNull(project.componentTemplates().get("foo")); - assertThat(project.componentTemplates().get("foo"), equalTo(componentTemplate)); + ComponentTemplate foo = project.componentTemplates().get("foo"); + ComponentTemplate expectedFoo = new ComponentTemplate(template, 1L, Map.of(), null, 0L, 0L); + assertThat(foo, equalTo(expectedFoo)); + Map> responseHeaders = threadContext.getResponseHeaders(); assertThat(responseHeaders.size(), is(1)); assertThat( diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 9888fdcbb8157..63ebe1a855211 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -2497,7 +2497,7 @@ public void testGetEffectiveSettingsComponentTemplateSettingsOnly() { .build(); Settings componentSettings = randomSettings(); Template.Builder componentTemplateBuilder = Template.builder().settings(componentSettings); - ComponentTemplate componentTemplate1 = new ComponentTemplate(componentTemplateBuilder.build(), null, null, null); + ComponentTemplate componentTemplate1 = new ComponentTemplate(componentTemplateBuilder.build(), null, null); ProjectMetadata.Builder projectMetadataBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()) .indexTemplates(Map.of(dataStream.getName(), indexTemplate)) .componentTemplates(Map.of("component-template-1", componentTemplate1)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 9a37cd6dd85b8..65217423e25f4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -43,6 +43,8 @@ import org.elasticsearch.xcontent.XContentParseException; import java.io.IOException; +import java.time.Instant; +import java.time.InstantSource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -54,6 +56,7 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.singletonList; @@ -424,8 +427,9 @@ public void testAddComponentTemplate() throws Exception { ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>()); project = metadataIndexTemplateService.addComponentTemplate(project, false, "foo", componentTemplate); - assertNotNull(project.componentTemplates().get("foo")); - assertThat(project.componentTemplates().get("foo"), equalTo(componentTemplate)); + ComponentTemplate foo = project.componentTemplates().get("foo"); + ComponentTemplate expectedFoo = new ComponentTemplate(template, 1L, Map.of(), null, 0L, 0L); + assertThat(foo, equalTo(expectedFoo)); ProjectMetadata throwState = ProjectMetadata.builder(project).build(); IllegalArgumentException e = expectThrows( @@ -1927,12 +1931,16 @@ public void testRemoveComponentTemplate() throws Exception { ProjectMetadata projectMetadata = service.addComponentTemplate(temp, false, "baz", baz); ProjectMetadata result = innerRemoveComponentTemplate(projectMetadata, "foo"); + // created_date and modified_date come from monotic increasing clock + ComponentTemplate expectedBar = new ComponentTemplate(bar.template(), bar.version(), bar.metadata(), bar.deprecated(), 1L, 1L); + ComponentTemplate expectedBaz = new ComponentTemplate(baz.template(), baz.version(), baz.metadata(), baz.deprecated(), 2L, 2L); assertThat(result.componentTemplates().get("foo"), nullValue()); - assertThat(result.componentTemplates().get("bar"), equalTo(bar)); - assertThat(result.componentTemplates().get("baz"), equalTo(baz)); + assertThat(result.componentTemplates().get("bar"), equalTo(expectedBar)); + assertThat(result.componentTemplates().get("baz"), equalTo(expectedBaz)); result = innerRemoveComponentTemplate(projectMetadata, "bar", "baz"); - assertThat(result.componentTemplates().get("foo"), equalTo(foo)); + ComponentTemplate expectedFoo = new ComponentTemplate(foo.template(), foo.version(), foo.metadata(), foo.deprecated(), 0L, 0L); + assertThat(result.componentTemplates().get("foo"), equalTo(expectedFoo)); assertThat(result.componentTemplates().get("bar"), nullValue()); assertThat(result.componentTemplates().get("baz"), nullValue()); @@ -1946,7 +1954,7 @@ public void testRemoveComponentTemplate() throws Exception { result = innerRemoveComponentTemplate(projectMetadata, "b*"); assertThat(result.componentTemplates().size(), equalTo(1)); - assertThat(result.componentTemplates().get("foo"), equalTo(foo)); + assertThat(result.componentTemplates().get("foo"), equalTo(expectedFoo)); e = expectThrows(ResourceNotFoundException.class, () -> innerRemoveComponentTemplate(projectMetadata, "foo", "b*")); assertThat(e.getMessage(), equalTo("b*")); @@ -2743,6 +2751,81 @@ public void testAddIndexTemplateWithDeprecatedComponentTemplate() throws Excepti assertWarnings("index template [foo] uses deprecated component template [ct]"); } + public void testComponentTemplateNoUpdateWhenNoChange() throws Exception { + final String name = "test-template"; + final ProjectId projectId = randomProjectIdOrDefault(); + + final Template template = new Template(Settings.builder().put("index.number_of_shards", 1).build(), null, null); + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + + // create template + final ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, null); + final ProjectMetadata initialMetadata = ProjectMetadata.builder(projectId).build(); + final ProjectMetadata updatedMetadata = service.addComponentTemplate(initialMetadata, false, name, componentTemplate); + final ComponentTemplate addedTemplate = updatedMetadata.componentTemplates().get(name); + assertThat(addedTemplate.createdDateMillis().orElseThrow(), is(0L)); + assertThat(addedTemplate.modifiedDateMillis().orElseThrow(), is(0L)); + + // update template which should result in NOP + final ProjectMetadata sameMetadata = service.addComponentTemplate(updatedMetadata, false, name, componentTemplate); + assertThat(sameMetadata, sameInstance(updatedMetadata)); + final ComponentTemplate unchangedTemplate = sameMetadata.componentTemplates().get(name); + assertThat(unchangedTemplate.createdDateMillis().orElseThrow(), is(0L)); + assertThat(unchangedTemplate.modifiedDateMillis().orElseThrow(), is(0L)); + } + + public void testComponentTemplateUpdateWithoutExistingTracking() throws Exception { + final String name = "test-template"; + final ProjectId projectId = randomProjectIdOrDefault(); + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + final ComponentTemplate initialTemplate = new ComponentTemplate( + new Template(Settings.builder().put("index.number_of_shards", 1).build(), null, null), + 1L, + null + ); + final ProjectMetadata initialMetadata = ProjectMetadata.builder(projectId) + .componentTemplates(Map.of(name, initialTemplate)) + .build(); + + final ComponentTemplate updateTemplate = new ComponentTemplate( + new Template(Settings.builder().put("index.number_of_shards", 2).build(), null, null), + 1L, + null + ); + final ProjectMetadata afterCreateMetadata = service.addComponentTemplate(initialMetadata, false, name, updateTemplate); + + final ComponentTemplate newTemplate = afterCreateMetadata.componentTemplates().get(name); + assertTrue(newTemplate.createdDateMillis().isEmpty()); + assertThat(newTemplate.modifiedDateMillis().orElseThrow(), is(0L)); + } + + public void testComponentTemplateUpdateChangesModifiedDate() throws Exception { + final String name = "test-template"; + final ProjectId projectId = randomProjectIdOrDefault(); + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + final ProjectMetadata initialMetadata = ProjectMetadata.builder(projectId).build(); + final Template template = new Template(Settings.builder().put("index.number_of_shards", 1).build(), null, null); + + // create template + final ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, null); + final ProjectMetadata afterCreateMetadata = service.addComponentTemplate(initialMetadata, false, name, componentTemplate); + final ComponentTemplate addedTemplate = afterCreateMetadata.componentTemplates().get(name); + assertThat(addedTemplate.createdDateMillis().orElseThrow(), is(0L)); + assertThat(addedTemplate.modifiedDateMillis().orElseThrow(), is(0L)); + + // update template + final ComponentTemplate updatedComponentTemplate = new ComponentTemplate(template, 2L, null); + final ProjectMetadata afterUpdateMetadata = service.addComponentTemplate( + afterCreateMetadata, + false, + name, + updatedComponentTemplate + ); + final ComponentTemplate newTemplate = afterUpdateMetadata.componentTemplates().get(name); + assertThat(newTemplate.createdDateMillis().orElseThrow(), is(0L)); + assertThat(newTemplate.modifiedDateMillis().orElseThrow(), is(1L)); + } + private static List putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { ThreadPool testThreadPool = mock(ThreadPool.class); when(testThreadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); @@ -2769,7 +2852,8 @@ private static List putTemplate(NamedXContentRegistry xContentRegistr xContentRegistry, EmptySystemIndices.INSTANCE, new IndexSettingProviders(Set.of()), - DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()) + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()), + Instant::now ); final List throwables = new ArrayList<>(); @@ -2825,6 +2909,8 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { true, new IndexSettingProviders(Set.of()) ); + AtomicInteger instantSourceInvocationCounter = new AtomicInteger(); + InstantSource instantSource = () -> Instant.ofEpochMilli(instantSourceInvocationCounter.getAndIncrement()); return new MetadataIndexTemplateService( clusterService, createIndexService, @@ -2833,7 +2919,8 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { xContentRegistry(), EmptySystemIndices.INSTANCE, new IndexSettingProviders(Set.of()), - DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()) + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()), + instantSource ); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 316a1ea75b7a3..d14f8a95c68ef 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -309,7 +309,9 @@ private static ProjectMetadata removeSkippedSettings( .build(), componentTemplate.version(), componentTemplate.metadata(), - componentTemplate.deprecated() + componentTemplate.deprecated(), + componentTemplate.createdDateMillis().orElse(null), + componentTemplate.modifiedDateMillis().orElse(null) ) ); }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java index 94bb82fadd7ca..809f2e6178044 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java @@ -796,7 +796,9 @@ static List migrateComponentTemplates( migratedInnerTemplate, componentTemplate.version(), componentTemplate.metadata(), - componentTemplate.deprecated() + componentTemplate.deprecated(), + componentTemplate.createdDateMillis().orElse(null), + componentTemplate.modifiedDateMillis().orElse(null) ); projectMetadataBuilder.put(componentEntry.getKey(), migratedComponentTemplate); From 692c356674a7eabfdf435c7b5dfa0a96de44ab68 Mon Sep 17 00:00:00 2001 From: Szymon Bialkowski Date: Mon, 21 Jul 2025 15:55:09 +0100 Subject: [PATCH 2/6] self-review --- .../MetadataIndexTemplateServiceTests.java | 4 +-- .../20_tracking.yml | 7 +++--- .../put/PutComponentTemplateAction.java | 4 +-- .../MetadataIndexTemplateService.java | 25 +++++++++++++++++++ .../elasticsearch/node/NodeConstruction.java | 6 +---- ...vedComposableIndexTemplateActionTests.java | 7 ++---- .../MetadataIndexTemplateServiceTests.java | 3 +-- 7 files changed, 35 insertions(+), 21 deletions(-) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java index bda28c5d74b37..8a6d510ad1bd8 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.ESSingleNodeTestCase; -import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -217,8 +216,7 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { xContentRegistry(), EmptySystemIndices.INSTANCE, indexSettingProviders, - DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()), - Instant::now + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()) ); } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml index 94e1919328f9d..17ca6069405ec 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml @@ -19,7 +19,7 @@ setup: number_of_shards: 1 created_date: "2025-07-04T12:50:48.415Z" - match: { status: 400 } - - match: { error.reason: "Validation Failed: 1: Provided a pipeline property which is managed by the system: created_date.;" } + - match: { error.reason: "Validation Failed: 1: Provided a template property which is managed by the system: created_date;" } --- "Test PUT setting modified_date": @@ -33,7 +33,7 @@ setup: number_of_shards: 1 modified_date: "2025-07-04T12:50:48.415Z" - match: { status: 400 } - - match: { error.reason: "Validation Failed: 1: Provided a pipeline property which is managed by the system: modified_date.;" } + - match: { error.reason: "Validation Failed: 1: Provided a template property which is managed by the system: modified_date;" } --- "Test update preserves created_date but updates modified_date": @@ -53,6 +53,7 @@ setup: - set: { component_templates.0.component_template.modified_date: first_modified } - match: { $first_created: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" } - match: { $first_modified: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" } + - match: { $first_created: $first_modified } - do: cluster.put_component_template: @@ -66,7 +67,5 @@ setup: cluster.get_component_template: name: test_tracking - set: { component_templates.0.component_template.created_date: second_created } - - set: { component_templates.0.component_template.modified_date: second_modified } - match: { $second_created: $first_created } - - gt: { $second_modified: $first_modified } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java index ec42e4e2429d3..b8667900261b2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java @@ -81,13 +81,13 @@ public ActionRequestValidationException validate() { } if (componentTemplate.createdDateMillis().isPresent()) { validationException = addValidationError( - "Provided a pipeline property which is managed by the system: created_date.", + "Provided a template property which is managed by the system: created_date", validationException ); } if (componentTemplate.modifiedDateMillis().isPresent()) { validationException = addValidationError( - "Provided a pipeline property which is managed by the system: modified_date.", + "Provided a template property which is managed by the system: modified_date", validationException ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index ed26339bccd5c..c5dbbb9641247 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -57,6 +57,7 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.IOException; +import java.time.Instant; import java.time.InstantSource; import java.util.ArrayList; import java.util.Arrays; @@ -187,6 +188,30 @@ public void onFailure(Exception e) { @Inject public MetadataIndexTemplateService( + ClusterService clusterService, + MetadataCreateIndexService metadataCreateIndexService, + IndicesService indicesService, + IndexScopedSettings indexScopedSettings, + NamedXContentRegistry xContentRegistry, + SystemIndices systemIndices, + IndexSettingProviders indexSettingProviders, + DataStreamGlobalRetentionSettings globalRetentionSettings + ) { + this( + clusterService, + metadataCreateIndexService, + indicesService, + indexScopedSettings, + xContentRegistry, + systemIndices, + indexSettingProviders, + globalRetentionSettings, + Instant::now + ); + } + + // constructor allowing for injection of InstantSource/time for testing + MetadataIndexTemplateService( ClusterService clusterService, MetadataCreateIndexService metadataCreateIndexService, IndicesService indicesService, diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index f96d9bfb72d05..bb28ed4a8aff5 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -228,8 +228,6 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; -import java.time.Instant; -import java.time.InstantSource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1302,7 +1300,6 @@ public Map queryFields() { b.bind(ShutdownPrepareService.class).toInstance(shutdownPrepareService); b.bind(OnlinePrewarmingService.class).toInstance(onlinePrewarmingService); b.bind(MergeMetrics.class).toInstance(mergeMetrics); - b.bind(InstantSource.class).toInstance(Instant::now); }); if (ReadinessService.enabled(environment)) { @@ -1658,8 +1655,7 @@ private List> buildReservedProjectStateHandlers( xContentRegistry, systemIndices, indexSettingProviders, - globalRetentionSettings, - Instant::now + globalRetentionSettings ); reservedStateHandlers.add(new ReservedComposableIndexTemplateAction(templateService, settingsModule.getIndexScopedSettings())); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java index 5cbb2879840cc..cb9fa23aaefbc 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/reservedstate/ReservedComposableIndexTemplateActionTests.java @@ -54,7 +54,6 @@ import org.junit.Before; import java.io.IOException; -import java.time.Instant; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -106,8 +105,7 @@ public void setup() throws IOException { mock(NamedXContentRegistry.class), mock(SystemIndices.class), new IndexSettingProviders(Set.of()), - globalRetentionSettings, - Instant::now + globalRetentionSettings ); } @@ -898,8 +896,7 @@ public void testTemplatesWithReservedPrefix() throws Exception { mock(NamedXContentRegistry.class), mock(SystemIndices.class), new IndexSettingProviders(Set.of()), - globalRetentionSettings, - Instant::now + globalRetentionSettings ); ClusterState state = ClusterState.builder(new ClusterName("elasticsearch")) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 65217423e25f4..319e80893fa20 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -2852,8 +2852,7 @@ private static List putTemplate(NamedXContentRegistry xContentRegistr xContentRegistry, EmptySystemIndices.INSTANCE, new IndexSettingProviders(Set.of()), - DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()), - Instant::now + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()) ); final List throwables = new ArrayList<>(); From f52390f5143b82d9deb087838088dc98f869c765 Mon Sep 17 00:00:00 2001 From: Szymon Bialkowski Date: Tue, 22 Jul 2025 11:40:16 +0100 Subject: [PATCH 3/6] spotless --- .../cluster/metadata/MetadataIndexTemplateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index c5dbbb9641247..d4dcf8f302b69 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -210,7 +210,7 @@ public MetadataIndexTemplateService( ); } - // constructor allowing for injection of InstantSource/time for testing + // constructor allowing for injection of InstantSource/time for testing MetadataIndexTemplateService( ClusterService clusterService, MetadataCreateIndexService metadataCreateIndexService, From 6a060ab9558dcc1be907eb7e04a78ae072e1704a Mon Sep 17 00:00:00 2001 From: Szymon Bialkowski Date: Tue, 22 Jul 2025 13:20:09 +0100 Subject: [PATCH 4/6] Update docs/changelog/131536.yaml --- docs/changelog/131536.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/131536.yaml diff --git a/docs/changelog/131536.yaml b/docs/changelog/131536.yaml new file mode 100644 index 0000000000000..63284be220690 --- /dev/null +++ b/docs/changelog/131536.yaml @@ -0,0 +1,5 @@ +pr: 131536 +summary: "Component Templates: Add `{created,modified}_date`" +area: Ingest Node +type: enhancement +issues: [] From 146a5ee27516f88b9038ffb333276d985b1931b3 Mon Sep 17 00:00:00 2001 From: Szymon Bialkowski Date: Fri, 25 Jul 2025 18:52:37 +0100 Subject: [PATCH 5/6] comments and learnings from previous PR --- .../20_tracking.yml | 42 +++++++++++++++++- .../put/PutComponentTemplateAction.java | 4 +- .../cluster/metadata/ComponentTemplate.java | 44 +++++++------------ .../MetadataIndexTemplateServiceTests.java | 2 +- 4 files changed, 58 insertions(+), 34 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml index 17ca6069405ec..d6fad3d5b714a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml @@ -6,6 +6,8 @@ setup: path: /_component_template/{id} capabilities: [ component_template_tracking_info ] reason: "Templates have tracking info: modified_date and created_date" + - requires: + test_runner_features: contains --- "Test PUT setting created_date": @@ -19,7 +21,21 @@ setup: number_of_shards: 1 created_date: "2025-07-04T12:50:48.415Z" - match: { status: 400 } - - match: { error.reason: "Validation Failed: 1: Provided a template property which is managed by the system: created_date;" } + - contains: { error.reason: "[component_template] unknown field [created_date] did you mean [created_date_millis]?" } + +--- +"Test PUT setting created_date_millis": + - do: + catch: bad_request + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 1 + created_date_millis: 1 + - match: { status: 400 } + - match: { error.reason: "Validation Failed: 1: Provided a template property which is managed by the system: created_date_millis;" } --- "Test PUT setting modified_date": @@ -33,7 +49,21 @@ setup: number_of_shards: 1 modified_date: "2025-07-04T12:50:48.415Z" - match: { status: 400 } - - match: { error.reason: "Validation Failed: 1: Provided a template property which is managed by the system: modified_date;" } + - contains: { error.reason: "[component_template] unknown field [modified_date] did you mean [modified_date_millis]?" } + +--- +"Test PUT setting modified_date_millis": + - do: + catch: bad_request + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 1 + modified_date_millis: 1 + - match: { status: 400 } + - match: { error.reason: "Validation Failed: 1: Provided a template property which is managed by the system: modified_date_millis;" } --- "Test update preserves created_date but updates modified_date": @@ -48,12 +78,17 @@ setup: - do: cluster.get_component_template: + human: true name: test_tracking - set: { component_templates.0.component_template.created_date: first_created } + - set: { component_templates.0.component_template.created_date_millis: first_created_millis } - set: { component_templates.0.component_template.modified_date: first_modified } + - set: { component_templates.0.component_template.modified_date_millis: first_modified_millis } - match: { $first_created: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" } - match: { $first_modified: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" } - match: { $first_created: $first_modified } + - match: { $first_created_millis: $first_modified_millis } + - gte: { $first_modified_millis: 0 } - do: cluster.put_component_template: @@ -65,7 +100,10 @@ setup: - do: cluster.get_component_template: + human: true name: test_tracking - set: { component_templates.0.component_template.created_date: second_created } + - set: { component_templates.0.component_template.created_date_millis: second_created_millis } - match: { $second_created: $first_created } + - match: { $second_created_millis: $first_created_millis } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java index b8667900261b2..b286055c4549a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java @@ -81,13 +81,13 @@ public ActionRequestValidationException validate() { } if (componentTemplate.createdDateMillis().isPresent()) { validationException = addValidationError( - "Provided a template property which is managed by the system: created_date", + "Provided a template property which is managed by the system: created_date_millis", validationException ); } if (componentTemplate.modifiedDateMillis().isPresent()) { validationException = addValidationError( - "Provided a template property which is managed by the system: modified_date", + "Provided a template property which is managed by the system: modified_date_millis", validationException ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java index e526e3328f019..3b8bdc96307c3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java @@ -24,10 +24,6 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; -import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -39,34 +35,20 @@ * "foo" field. These component templates make up the individual pieces composing an index template. */ public class ComponentTemplate implements SimpleDiffable, ToXContentObject { - - // always output millis even if instantSource returns millis == 0 - private static final DateTimeFormatter ISO8601_WITH_MILLIS_FORMATTER = new DateTimeFormatterBuilder().appendInstant(3) - .toFormatter(Locale.ROOT); - private static final ParseField TEMPLATE = new ParseField("template"); private static final ParseField VERSION = new ParseField("version"); private static final ParseField METADATA = new ParseField("_meta"); private static final ParseField DEPRECATED = new ParseField("deprecated"); private static final ParseField CREATED_DATE = new ParseField("created_date"); + private static final ParseField CREATED_DATE_MILLIS = new ParseField("created_date_millis"); private static final ParseField MODIFIED_DATE = new ParseField("modified_date"); + private static final ParseField MODIFIED_DATE_MILLIS = new ParseField("modified_date_millis"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "component_template", false, - a -> { - final String createdDate = (String) a[4]; - final String modifiedDate = (String) a[5]; - return new ComponentTemplate( - (Template) a[0], - (Long) a[1], - (Map) a[2], - (Boolean) a[3], - createdDate == null ? null : Instant.parse(createdDate).toEpochMilli(), - modifiedDate == null ? null : Instant.parse(modifiedDate).toEpochMilli() - ); - } + a -> new ComponentTemplate((Template) a[0], (Long) a[1], (Map) a[2], (Boolean) a[3], (Long) a[4], (Long) a[5]) ); static { @@ -74,8 +56,8 @@ public class ComponentTemplate implements SimpleDiffable, ToX PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), DEPRECATED); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), CREATED_DATE); - PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), MODIFIED_DATE); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), CREATED_DATE_MILLIS); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MODIFIED_DATE_MILLIS); } private final Template template; @@ -239,16 +221,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla builder.field(DEPRECATED.getPreferredName(), this.deprecated); } if (this.createdDateMillis != null) { - builder.field(CREATED_DATE.getPreferredName(), formatDate(this.createdDateMillis)); + builder.timestampFieldsFromUnixEpochMillis( + CREATED_DATE_MILLIS.getPreferredName(), + CREATED_DATE.getPreferredName(), + this.createdDateMillis + ); } if (this.modifiedDateMillis != null) { - builder.field(MODIFIED_DATE.getPreferredName(), formatDate(this.modifiedDateMillis)); + builder.timestampFieldsFromUnixEpochMillis( + MODIFIED_DATE_MILLIS.getPreferredName(), + MODIFIED_DATE.getPreferredName(), + this.modifiedDateMillis + ); } builder.endObject(); return builder; } - - private static String formatDate(long epochMillis) { - return ISO8601_WITH_MILLIS_FORMATTER.format(Instant.ofEpochMilli(epochMillis)); - } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 319e80893fa20..8b5d9577a8dc7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1931,7 +1931,7 @@ public void testRemoveComponentTemplate() throws Exception { ProjectMetadata projectMetadata = service.addComponentTemplate(temp, false, "baz", baz); ProjectMetadata result = innerRemoveComponentTemplate(projectMetadata, "foo"); - // created_date and modified_date come from monotic increasing clock + // created_date and modified_date come from monotonically increasing clock ComponentTemplate expectedBar = new ComponentTemplate(bar.template(), bar.version(), bar.metadata(), bar.deprecated(), 1L, 1L); ComponentTemplate expectedBaz = new ComponentTemplate(baz.template(), baz.version(), baz.metadata(), baz.deprecated(), 2L, 2L); assertThat(result.componentTemplates().get("foo"), nullValue()); From 86c4db43eaa8a0f715888ca4b302c1fd07177c2c Mon Sep 17 00:00:00 2001 From: Szymon Bialkowski Date: Fri, 25 Jul 2025 19:11:02 +0100 Subject: [PATCH 6/6] lee's comment --- .../java/org/elasticsearch/ingest/PipelineConfiguration.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index ca8170bccf403..9770afc4653ed 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -251,6 +251,7 @@ PipelineConfiguration maybeUpgradeProcessors(String type, IngestMetadata.Process } } + /** Remove system properties from config if they aren't supported by the transport version */ private Map configForTransport(final TransportVersion transportVersion) { final boolean transportSupportsNewProperties = transportVersion.onOrAfter(TransportVersions.PIPELINE_TRACKING_INFO); final boolean noNewProperties = config.containsKey(Pipeline.CREATED_DATE_MILLIS) == false