diff --git a/docs/changelog/131536.yaml b/docs/changelog/131536.yaml new file mode 100644 index 0000000000000..63284be220690 --- /dev/null +++ b/docs/changelog/131536.yaml @@ -0,0 +1,5 @@ +pr: 131536 +summary: "Component Templates: Add `{created,modified}_date`" +area: Ingest Node +type: enhancement +issues: [] diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index d23edf5e91045..6e6251b7d7fd1 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -644,6 +644,8 @@ private static ProjectMetadata getProjectWithDataStreamWithSettings( Template.builder().settings(componentTemplateSettings).build(), null, null, + null, + null, null ); builder.componentTemplates(Map.of("component_template_1", componentTemplate)); diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml new file mode 100644 index 0000000000000..d6fad3d5b714a --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/20_tracking.yml @@ -0,0 +1,109 @@ +setup: + - requires: + test_runner_features: capabilities + capabilities: + - method: PUT + path: /_component_template/{id} + capabilities: [ component_template_tracking_info ] + reason: "Templates have tracking info: modified_date and created_date" + - requires: + test_runner_features: contains + +--- +"Test PUT setting created_date": + - do: + catch: bad_request + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 1 + created_date: "2025-07-04T12:50:48.415Z" + - match: { status: 400 } + - contains: { error.reason: "[component_template] unknown field [created_date] did you mean [created_date_millis]?" } + +--- +"Test PUT setting created_date_millis": + - do: + catch: bad_request + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 1 + created_date_millis: 1 + - match: { status: 400 } + - match: { error.reason: "Validation Failed: 1: Provided a template property which is managed by the system: created_date_millis;" } + +--- +"Test PUT setting modified_date": + - do: + catch: bad_request + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 1 + modified_date: "2025-07-04T12:50:48.415Z" + - match: { status: 400 } + - contains: { error.reason: "[component_template] unknown field [modified_date] did you mean [modified_date_millis]?" } + +--- +"Test PUT setting modified_date_millis": + - do: + catch: bad_request + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 1 + modified_date_millis: 1 + - match: { status: 400 } + - match: { error.reason: "Validation Failed: 1: Provided a template property which is managed by the system: modified_date_millis;" } + +--- +"Test update preserves created_date but updates modified_date": + - do: + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 1 + - match: { acknowledged: true } + + - do: + cluster.get_component_template: + human: true + name: test_tracking + - set: { component_templates.0.component_template.created_date: first_created } + - set: { component_templates.0.component_template.created_date_millis: first_created_millis } + - set: { component_templates.0.component_template.modified_date: first_modified } + - set: { component_templates.0.component_template.modified_date_millis: first_modified_millis } + - match: { $first_created: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" } + - match: { $first_modified: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" } + - match: { $first_created: $first_modified } + - match: { $first_created_millis: $first_modified_millis } + - gte: { $first_modified_millis: 0 } + + - do: + cluster.put_component_template: + name: test_tracking + body: + template: + settings: + number_of_shards: 2 + + - do: + cluster.get_component_template: + human: true + name: test_tracking + - set: { component_templates.0.component_template.created_date: second_created } + - set: { component_templates.0.component_template.created_date_millis: second_created_millis } + - match: { $second_created: $first_created } + - match: { $second_created_millis: $first_created_millis } + diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index afd8bb89b4539..2722340858446 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -353,6 +353,7 @@ static TransportVersion def(int id) { public static final TransportVersion NODE_WEIGHTS_ADDED_TO_NODE_BALANCE_STATS = def(9_129_0_00); public static final TransportVersion RERANK_SNIPPETS = def(9_130_0_00); public static final TransportVersion PIPELINE_TRACKING_INFO = def(9_131_0_00); + public static final TransportVersion COMPONENT_TEMPLATE_TRACKING_INFO = def(9_132_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java index 2c34359dabbad..b286055c4549a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutComponentTemplateAction.java @@ -79,6 +79,18 @@ public ActionRequestValidationException validate() { if (componentTemplate == null) { validationException = addValidationError("a component template is required", validationException); } + if (componentTemplate.createdDateMillis().isPresent()) { + validationException = addValidationError( + "Provided a template property which is managed by the system: created_date_millis", + validationException + ); + } + if (componentTemplate.modifiedDateMillis().isPresent()) { + validationException = addValidationError( + "Provided a template property which is managed by the system: modified_date_millis", + validationException + ); + } return validationException; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index 05d055dbf979c..79a8bffa1e1b6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -85,7 +85,9 @@ public static ComponentTemplate normalizeComponentTemplate( template, componentTemplate.version(), componentTemplate.metadata(), - componentTemplate.deprecated() + componentTemplate.deprecated(), + componentTemplate.createdDateMillis().orElse(null), + componentTemplate.modifiedDateMillis().orElse(null) ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java index 016fec60f06be..3b8bdc96307c3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplate.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * A component template is a re-usable {@link Template} as well as metadata about the template. Each @@ -38,12 +39,16 @@ public class ComponentTemplate implements SimpleDiffable, ToX private static final ParseField VERSION = new ParseField("version"); private static final ParseField METADATA = new ParseField("_meta"); private static final ParseField DEPRECATED = new ParseField("deprecated"); + private static final ParseField CREATED_DATE = new ParseField("created_date"); + private static final ParseField CREATED_DATE_MILLIS = new ParseField("created_date_millis"); + private static final ParseField MODIFIED_DATE = new ParseField("modified_date"); + private static final ParseField MODIFIED_DATE_MILLIS = new ParseField("modified_date_millis"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "component_template", false, - a -> new ComponentTemplate((Template) a[0], (Long) a[1], (Map) a[2], (Boolean) a[3]) + a -> new ComponentTemplate((Template) a[0], (Long) a[1], (Map) a[2], (Boolean) a[3], (Long) a[4], (Long) a[5]) ); static { @@ -51,6 +56,8 @@ public class ComponentTemplate implements SimpleDiffable, ToX PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), VERSION); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), DEPRECATED); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), CREATED_DATE_MILLIS); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MODIFIED_DATE_MILLIS); } private final Template template; @@ -60,6 +67,10 @@ public class ComponentTemplate implements SimpleDiffable, ToX private final Map metadata; @Nullable private final Boolean deprecated; + @Nullable + private final Long createdDateMillis; + @Nullable + private final Long modifiedDateMillis; static Diff readComponentTemplateDiffFrom(StreamInput in) throws IOException { return SimpleDiffable.readDiffFrom(ComponentTemplate::new, in); @@ -70,19 +81,23 @@ public static ComponentTemplate parse(XContentParser parser) { } public ComponentTemplate(Template template, @Nullable Long version, @Nullable Map metadata) { - this(template, version, metadata, null); + this(template, version, metadata, null, null, null); } public ComponentTemplate( Template template, @Nullable Long version, @Nullable Map metadata, - @Nullable Boolean deprecated + @Nullable Boolean deprecated, + @Nullable Long createdDateMillis, + @Nullable Long modifiedDateMillis ) { this.template = template; this.version = version; this.metadata = metadata; this.deprecated = deprecated; + this.createdDateMillis = createdDateMillis; + this.modifiedDateMillis = modifiedDateMillis; } public ComponentTemplate(StreamInput in) throws IOException { @@ -98,6 +113,13 @@ public ComponentTemplate(StreamInput in) throws IOException { } else { deprecated = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.COMPONENT_TEMPLATE_TRACKING_INFO)) { + this.createdDateMillis = in.readOptionalLong(); + this.modifiedDateMillis = in.readOptionalLong(); + } else { + this.createdDateMillis = null; + this.modifiedDateMillis = null; + } } public Template template() { @@ -122,6 +144,14 @@ public boolean isDeprecated() { return Boolean.TRUE.equals(deprecated); } + public Optional createdDateMillis() { + return Optional.ofNullable(createdDateMillis); + } + + public Optional modifiedDateMillis() { + return Optional.ofNullable(modifiedDateMillis); + } + @Override public void writeTo(StreamOutput out) throws IOException { this.template.writeTo(out); @@ -135,11 +165,15 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalBoolean(this.deprecated); } + if (out.getTransportVersion().onOrAfter(TransportVersions.COMPONENT_TEMPLATE_TRACKING_INFO)) { + out.writeOptionalLong(this.createdDateMillis); + out.writeOptionalLong(this.modifiedDateMillis); + } } @Override public int hashCode() { - return Objects.hash(template, version, metadata, deprecated); + return Objects.hash(template, version, metadata, deprecated, createdDateMillis, modifiedDateMillis); } @Override @@ -154,7 +188,9 @@ public boolean equals(Object obj) { return Objects.equals(template, other.template) && Objects.equals(version, other.version) && Objects.equals(metadata, other.metadata) - && Objects.equals(deprecated, other.deprecated); + && Objects.equals(deprecated, other.deprecated) + && Objects.equals(createdDateMillis, other.createdDateMillis) + && Objects.equals(modifiedDateMillis, other.modifiedDateMillis); } @Override @@ -184,6 +220,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params, @Nulla if (this.deprecated != null) { builder.field(DEPRECATED.getPreferredName(), this.deprecated); } + if (this.createdDateMillis != null) { + builder.timestampFieldsFromUnixEpochMillis( + CREATED_DATE_MILLIS.getPreferredName(), + CREATED_DATE.getPreferredName(), + this.createdDateMillis + ); + } + if (this.modifiedDateMillis != null) { + builder.timestampFieldsFromUnixEpochMillis( + MODIFIED_DATE_MILLIS.getPreferredName(), + MODIFIED_DATE.getPreferredName(), + this.modifiedDateMillis + ); + } builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 69dd4359fd764..d4dcf8f302b69 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -58,6 +58,7 @@ import java.io.IOException; import java.time.Instant; +import java.time.InstantSource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -140,6 +141,7 @@ public class MetadataIndexTemplateService { private final SystemIndices systemIndices; private final Set indexSettingProviders; private final DataStreamGlobalRetentionSettings globalRetentionSettings; + private final InstantSource instantSource; /** * This is the cluster state task executor for all template-based actions. @@ -194,6 +196,31 @@ public MetadataIndexTemplateService( SystemIndices systemIndices, IndexSettingProviders indexSettingProviders, DataStreamGlobalRetentionSettings globalRetentionSettings + ) { + this( + clusterService, + metadataCreateIndexService, + indicesService, + indexScopedSettings, + xContentRegistry, + systemIndices, + indexSettingProviders, + globalRetentionSettings, + Instant::now + ); + } + + // constructor allowing for injection of InstantSource/time for testing + MetadataIndexTemplateService( + ClusterService clusterService, + MetadataCreateIndexService metadataCreateIndexService, + IndicesService indicesService, + IndexScopedSettings indexScopedSettings, + NamedXContentRegistry xContentRegistry, + SystemIndices systemIndices, + IndexSettingProviders indexSettingProviders, + DataStreamGlobalRetentionSettings globalRetentionSettings, + InstantSource instantSource ) { this.clusterService = clusterService; this.taskQueue = clusterService.createTaskQueue("index-templates", Priority.URGENT, TEMPLATE_TASK_EXECUTOR); @@ -204,6 +231,7 @@ public MetadataIndexTemplateService( this.systemIndices = systemIndices; this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders(); this.globalRetentionSettings = globalRetentionSettings; + this.instantSource = instantSource; } public void removeTemplates( @@ -323,15 +351,37 @@ public ProjectMetadata addComponentTemplate( } final Template finalTemplate = Template.builder(template.template()).settings(finalSettings).mappings(wrappedMappings).build(); - final ComponentTemplate finalComponentTemplate = new ComponentTemplate( - finalTemplate, - template.version(), - template.metadata(), - template.deprecated() - ); - - if (finalComponentTemplate.equals(existing)) { - return project; + final long now = instantSource.instant().toEpochMilli(); + final ComponentTemplate finalComponentTemplate; + if (existing == null) { + finalComponentTemplate = new ComponentTemplate( + finalTemplate, + template.version(), + template.metadata(), + template.deprecated(), + now, + now + ); + } else { + final ComponentTemplate templateToCompareToExisting = new ComponentTemplate( + finalTemplate, + template.version(), + template.metadata(), + template.deprecated(), + existing.createdDateMillis().orElse(null), + existing.modifiedDateMillis().orElse(null) + ); + if (templateToCompareToExisting.equals(existing)) { + return project; + } + finalComponentTemplate = new ComponentTemplate( + finalTemplate, + template.version(), + template.metadata(), + template.deprecated(), + existing.createdDateMillis().orElse(null), + now + ); } validateTemplate(finalSettings, wrappedMappings, indicesService); @@ -715,7 +765,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo // Workaround for the fact that start_time and end_time are injected by the MetadataCreateDataStreamService upon creation, // but when validating templates that create data streams the MetadataCreateDataStreamService isn't used. var finalTemplate = indexTemplate.template(); - final var now = Instant.now(); + final var now = instantSource.instant(); final var combinedMappings = collectMappings(indexTemplate, projectMetadata.componentTemplates(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, projectMetadata.componentTemplates()); diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java index ca8170bccf403..9770afc4653ed 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java @@ -251,6 +251,7 @@ PipelineConfiguration maybeUpgradeProcessors(String type, IngestMetadata.Process } } + /** Remove system properties from config if they aren't supported by the transport version */ private Map configForTransport(final TransportVersion transportVersion) { final boolean transportSupportsNewProperties = transportVersion.onOrAfter(TransportVersions.PIPELINE_TRACKING_INFO); final boolean noNewProperties = config.containsKey(Pipeline.CREATED_DATE_MILLIS) == false diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java index 8e3112510642a..aff4287ecbc58 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutComponentTemplateAction.java @@ -31,7 +31,12 @@ public class RestPutComponentTemplateAction extends BaseRestHandler { public static final String SUPPORTS_FAILURE_STORE_LIFECYCLE = "data_stream_options.failure_store.lifecycle"; public static final String SUPPORTS_FAILURE_STORE = "data_stream_options.failure_store"; - private static final Set capabilities = Set.of(SUPPORTS_FAILURE_STORE, SUPPORTS_FAILURE_STORE_LIFECYCLE); + private static final String COMPONENT_TEMPLATE_TRACKING_INFO = "component_template_tracking_info"; + private static final Set CAPABILITIES = Set.of( + SUPPORTS_FAILURE_STORE, + SUPPORTS_FAILURE_STORE_LIFECYCLE, + COMPONENT_TEMPLATE_TRACKING_INFO + ); @Override public List routes() { @@ -59,6 +64,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC @Override public Set supportedCapabilities() { - return capabilities; + return CAPABILITIES; } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java index 289e2795edf11..ac7063d8d4e36 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java @@ -58,7 +58,9 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws new Template(settings, mappings, aliases, lifecycle, dataStreamOptions), randomBoolean() ? null : randomNonNegativeLong(), null, - false + false, + null, + null ); var rolloverConfiguration = RolloverConfigurationTests.randomRolloverConditions(); var response = new GetComponentTemplateAction.Response(Map.of(randomAlphaOfLength(10), template), rolloverConfiguration); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java index 2a7eb712da344..14dbe3079a842 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComponentTemplateTests.java @@ -101,7 +101,21 @@ public static ComponentTemplate randomInstance(boolean supportsDataStreams, Bool if (randomBoolean()) { meta = randomMeta(); } - return new ComponentTemplate(template, randomBoolean() ? null : randomNonNegativeLong(), meta, deprecated); + final Long createdDate = randomBoolean() ? randomNonNegativeLong() : null; + final Long modifiedDate; + if (randomBoolean()) { + modifiedDate = createdDate == null ? randomNonNegativeLong() : randomLongBetween(createdDate, Long.MAX_VALUE); + } else { + modifiedDate = null; + } + return new ComponentTemplate( + template, + randomBoolean() ? null : randomNonNegativeLong(), + meta, + deprecated, + createdDate, + modifiedDate + ); } public static ResettableValue randomDataStreamOptionsTemplate() { @@ -166,19 +180,25 @@ yield switch (randomIntBetween(0, 4)) { Template.builder(ot).settings(randomValueOtherThan(ot.settings(), ComponentTemplateTests::randomSettings)).build(), orig.version(), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 1 -> new ComponentTemplate( Template.builder(ot).mappings(randomValueOtherThan(ot.mappings(), ComponentTemplateTests::randomMappings)).build(), orig.version(), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 2 -> new ComponentTemplate( Template.builder(ot).aliases(randomValueOtherThan(ot.aliases(), ComponentTemplateTests::randomAliases)).build(), orig.version(), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 3 -> new ComponentTemplate( Template.builder(ot) @@ -186,7 +206,9 @@ yield switch (randomIntBetween(0, 4)) { .build(), orig.version(), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 4 -> new ComponentTemplate( Template.builder(ot) @@ -196,7 +218,9 @@ yield switch (randomIntBetween(0, 4)) { .build(), orig.version(), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); default -> throw new IllegalStateException("illegal randomization branch"); }; @@ -205,19 +229,25 @@ yield switch (randomIntBetween(0, 4)) { orig.template(), randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong), orig.metadata(), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 2 -> new ComponentTemplate( orig.template(), orig.version(), randomValueOtherThan(orig.metadata(), ComponentTemplateTests::randomMeta), - orig.deprecated() + orig.deprecated(), + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); case 3 -> new ComponentTemplate( orig.template(), orig.version(), orig.metadata(), - orig.isDeprecated() ? randomFrom(false, null) : true + orig.isDeprecated() ? randomFrom(false, null) : true, + orig.createdDateMillis().orElse(null), + orig.modifiedDateMillis().orElse(null) ); default -> throw new IllegalStateException("illegal randomization branch"); }; diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java index a041dadb005f3..fe11099d2d60f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java @@ -25,11 +25,14 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESTestCase; +import java.time.Instant; +import java.time.InstantSource; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.cluster.metadata.DataStreamLifecycleTests.randomDownsampling; import static org.elasticsearch.common.settings.Settings.builder; @@ -265,6 +268,8 @@ public void testValidateLifecycleInComponentTemplate() throws Exception { .put(DataStreamGlobalRetentionSettings.DATA_STREAMS_DEFAULT_RETENTION_SETTING.getKey(), defaultRetention) .build(); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).build(); + AtomicInteger instantSourceInvocationCounter = new AtomicInteger(); + InstantSource instantSource = () -> Instant.ofEpochMilli(instantSourceInvocationCounter.getAndIncrement()); MetadataIndexTemplateService metadataIndexTemplateService = new MetadataIndexTemplateService( clusterService, createIndexService, @@ -273,7 +278,8 @@ public void testValidateLifecycleInComponentTemplate() throws Exception { xContentRegistry(), EmptySystemIndices.INSTANCE, new IndexSettingProviders(Set.of()), - DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings(settingsWithDefaultRetention)) + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings(settingsWithDefaultRetention)), + instantSource ); ThreadContext threadContext = new ThreadContext(Settings.EMPTY); @@ -287,8 +293,10 @@ public void testValidateLifecycleInComponentTemplate() throws Exception { ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>()); project = metadataIndexTemplateService.addComponentTemplate(project, false, "foo", componentTemplate); - assertNotNull(project.componentTemplates().get("foo")); - assertThat(project.componentTemplates().get("foo"), equalTo(componentTemplate)); + ComponentTemplate foo = project.componentTemplates().get("foo"); + ComponentTemplate expectedFoo = new ComponentTemplate(template, 1L, Map.of(), null, 0L, 0L); + assertThat(foo, equalTo(expectedFoo)); + Map> responseHeaders = threadContext.getResponseHeaders(); assertThat(responseHeaders.size(), is(1)); assertThat( diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 9888fdcbb8157..63ebe1a855211 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -2497,7 +2497,7 @@ public void testGetEffectiveSettingsComponentTemplateSettingsOnly() { .build(); Settings componentSettings = randomSettings(); Template.Builder componentTemplateBuilder = Template.builder().settings(componentSettings); - ComponentTemplate componentTemplate1 = new ComponentTemplate(componentTemplateBuilder.build(), null, null, null); + ComponentTemplate componentTemplate1 = new ComponentTemplate(componentTemplateBuilder.build(), null, null); ProjectMetadata.Builder projectMetadataBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()) .indexTemplates(Map.of(dataStream.getName(), indexTemplate)) .componentTemplates(Map.of("component-template-1", componentTemplate1)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 9a37cd6dd85b8..8b5d9577a8dc7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -43,6 +43,8 @@ import org.elasticsearch.xcontent.XContentParseException; import java.io.IOException; +import java.time.Instant; +import java.time.InstantSource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -54,6 +56,7 @@ import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import static java.util.Collections.singletonList; @@ -424,8 +427,9 @@ public void testAddComponentTemplate() throws Exception { ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>()); project = metadataIndexTemplateService.addComponentTemplate(project, false, "foo", componentTemplate); - assertNotNull(project.componentTemplates().get("foo")); - assertThat(project.componentTemplates().get("foo"), equalTo(componentTemplate)); + ComponentTemplate foo = project.componentTemplates().get("foo"); + ComponentTemplate expectedFoo = new ComponentTemplate(template, 1L, Map.of(), null, 0L, 0L); + assertThat(foo, equalTo(expectedFoo)); ProjectMetadata throwState = ProjectMetadata.builder(project).build(); IllegalArgumentException e = expectThrows( @@ -1927,12 +1931,16 @@ public void testRemoveComponentTemplate() throws Exception { ProjectMetadata projectMetadata = service.addComponentTemplate(temp, false, "baz", baz); ProjectMetadata result = innerRemoveComponentTemplate(projectMetadata, "foo"); + // created_date and modified_date come from monotonically increasing clock + ComponentTemplate expectedBar = new ComponentTemplate(bar.template(), bar.version(), bar.metadata(), bar.deprecated(), 1L, 1L); + ComponentTemplate expectedBaz = new ComponentTemplate(baz.template(), baz.version(), baz.metadata(), baz.deprecated(), 2L, 2L); assertThat(result.componentTemplates().get("foo"), nullValue()); - assertThat(result.componentTemplates().get("bar"), equalTo(bar)); - assertThat(result.componentTemplates().get("baz"), equalTo(baz)); + assertThat(result.componentTemplates().get("bar"), equalTo(expectedBar)); + assertThat(result.componentTemplates().get("baz"), equalTo(expectedBaz)); result = innerRemoveComponentTemplate(projectMetadata, "bar", "baz"); - assertThat(result.componentTemplates().get("foo"), equalTo(foo)); + ComponentTemplate expectedFoo = new ComponentTemplate(foo.template(), foo.version(), foo.metadata(), foo.deprecated(), 0L, 0L); + assertThat(result.componentTemplates().get("foo"), equalTo(expectedFoo)); assertThat(result.componentTemplates().get("bar"), nullValue()); assertThat(result.componentTemplates().get("baz"), nullValue()); @@ -1946,7 +1954,7 @@ public void testRemoveComponentTemplate() throws Exception { result = innerRemoveComponentTemplate(projectMetadata, "b*"); assertThat(result.componentTemplates().size(), equalTo(1)); - assertThat(result.componentTemplates().get("foo"), equalTo(foo)); + assertThat(result.componentTemplates().get("foo"), equalTo(expectedFoo)); e = expectThrows(ResourceNotFoundException.class, () -> innerRemoveComponentTemplate(projectMetadata, "foo", "b*")); assertThat(e.getMessage(), equalTo("b*")); @@ -2743,6 +2751,81 @@ public void testAddIndexTemplateWithDeprecatedComponentTemplate() throws Excepti assertWarnings("index template [foo] uses deprecated component template [ct]"); } + public void testComponentTemplateNoUpdateWhenNoChange() throws Exception { + final String name = "test-template"; + final ProjectId projectId = randomProjectIdOrDefault(); + + final Template template = new Template(Settings.builder().put("index.number_of_shards", 1).build(), null, null); + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + + // create template + final ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, null); + final ProjectMetadata initialMetadata = ProjectMetadata.builder(projectId).build(); + final ProjectMetadata updatedMetadata = service.addComponentTemplate(initialMetadata, false, name, componentTemplate); + final ComponentTemplate addedTemplate = updatedMetadata.componentTemplates().get(name); + assertThat(addedTemplate.createdDateMillis().orElseThrow(), is(0L)); + assertThat(addedTemplate.modifiedDateMillis().orElseThrow(), is(0L)); + + // update template which should result in NOP + final ProjectMetadata sameMetadata = service.addComponentTemplate(updatedMetadata, false, name, componentTemplate); + assertThat(sameMetadata, sameInstance(updatedMetadata)); + final ComponentTemplate unchangedTemplate = sameMetadata.componentTemplates().get(name); + assertThat(unchangedTemplate.createdDateMillis().orElseThrow(), is(0L)); + assertThat(unchangedTemplate.modifiedDateMillis().orElseThrow(), is(0L)); + } + + public void testComponentTemplateUpdateWithoutExistingTracking() throws Exception { + final String name = "test-template"; + final ProjectId projectId = randomProjectIdOrDefault(); + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + final ComponentTemplate initialTemplate = new ComponentTemplate( + new Template(Settings.builder().put("index.number_of_shards", 1).build(), null, null), + 1L, + null + ); + final ProjectMetadata initialMetadata = ProjectMetadata.builder(projectId) + .componentTemplates(Map.of(name, initialTemplate)) + .build(); + + final ComponentTemplate updateTemplate = new ComponentTemplate( + new Template(Settings.builder().put("index.number_of_shards", 2).build(), null, null), + 1L, + null + ); + final ProjectMetadata afterCreateMetadata = service.addComponentTemplate(initialMetadata, false, name, updateTemplate); + + final ComponentTemplate newTemplate = afterCreateMetadata.componentTemplates().get(name); + assertTrue(newTemplate.createdDateMillis().isEmpty()); + assertThat(newTemplate.modifiedDateMillis().orElseThrow(), is(0L)); + } + + public void testComponentTemplateUpdateChangesModifiedDate() throws Exception { + final String name = "test-template"; + final ProjectId projectId = randomProjectIdOrDefault(); + final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + final ProjectMetadata initialMetadata = ProjectMetadata.builder(projectId).build(); + final Template template = new Template(Settings.builder().put("index.number_of_shards", 1).build(), null, null); + + // create template + final ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, null); + final ProjectMetadata afterCreateMetadata = service.addComponentTemplate(initialMetadata, false, name, componentTemplate); + final ComponentTemplate addedTemplate = afterCreateMetadata.componentTemplates().get(name); + assertThat(addedTemplate.createdDateMillis().orElseThrow(), is(0L)); + assertThat(addedTemplate.modifiedDateMillis().orElseThrow(), is(0L)); + + // update template + final ComponentTemplate updatedComponentTemplate = new ComponentTemplate(template, 2L, null); + final ProjectMetadata afterUpdateMetadata = service.addComponentTemplate( + afterCreateMetadata, + false, + name, + updatedComponentTemplate + ); + final ComponentTemplate newTemplate = afterUpdateMetadata.componentTemplates().get(name); + assertThat(newTemplate.createdDateMillis().orElseThrow(), is(0L)); + assertThat(newTemplate.modifiedDateMillis().orElseThrow(), is(1L)); + } + private static List putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { ThreadPool testThreadPool = mock(ThreadPool.class); when(testThreadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); @@ -2825,6 +2908,8 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { true, new IndexSettingProviders(Set.of()) ); + AtomicInteger instantSourceInvocationCounter = new AtomicInteger(); + InstantSource instantSource = () -> Instant.ofEpochMilli(instantSourceInvocationCounter.getAndIncrement()); return new MetadataIndexTemplateService( clusterService, createIndexService, @@ -2833,7 +2918,8 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { xContentRegistry(), EmptySystemIndices.INSTANCE, new IndexSettingProviders(Set.of()), - DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()) + DataStreamGlobalRetentionSettings.create(ClusterSettings.createBuiltInClusterSettings()), + instantSource ); } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java index 316a1ea75b7a3..d14f8a95c68ef 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java @@ -309,7 +309,9 @@ private static ProjectMetadata removeSkippedSettings( .build(), componentTemplate.version(), componentTemplate.metadata(), - componentTemplate.deprecated() + componentTemplate.deprecated(), + componentTemplate.createdDateMillis().orElse(null), + componentTemplate.modifiedDateMillis().orElse(null) ) ); }).collect(Collectors.toMap(Tuple::v1, Tuple::v2))); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java index 94bb82fadd7ca..809f2e6178044 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/cluster/metadata/MetadataMigrateToDataTiersRoutingService.java @@ -796,7 +796,9 @@ static List migrateComponentTemplates( migratedInnerTemplate, componentTemplate.version(), componentTemplate.metadata(), - componentTemplate.deprecated() + componentTemplate.deprecated(), + componentTemplate.createdDateMillis().orElse(null), + componentTemplate.modifiedDateMillis().orElse(null) ); projectMetadataBuilder.put(componentEntry.getKey(), migratedComponentTemplate);