From d3d9714ec667c63cb23e6628ad135109926dda43 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 25 Mar 2025 11:27:50 -0700 Subject: [PATCH 1/5] Rename deprecation index template The dataset name for the deprecation logs index was previously renamed from `deprecation.elasticsearch` to `elasticsearch.deprecation` in order to follow the pattern of `product.group`. The deprecation index template, however, was not updated. This causes indexing errors once upgraded to 9.0 due to the dataset name having changed on a constant_keyword field. In order to avoid that mismatch, this commit renames the deprecation indexing datastream to match the dataset name. The old template is kept in place, but marked as deprecated, so that any deprecation logs written during upgrading to 9.x will continue to be indexed into the old datastream. closes #125445 --- .../deprecation-indexing-template.json | 3 ++- .../deprecation/DeprecationTestUtils.java | 2 +- .../logging/DeprecationIndexingAppender.java | 2 +- .../logging/DeprecationIndexingComponent.java | 2 +- .../DeprecationIndexingTemplateRegistry.java | 26 ++++++++++++++++--- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/deprecation/deprecation-indexing-template.json b/x-pack/plugin/core/template-resources/src/main/resources/deprecation/deprecation-indexing-template.json index 3dc97ba50a7b8..fa7a0ba972059 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/deprecation/deprecation-indexing-template.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/deprecation/deprecation-indexing-template.json @@ -1,5 +1,5 @@ { - "index_patterns": [".logs-deprecation.*"], + "index_patterns": ["${xpack.deprecation.indexing.template.pattern}"], "priority": 1000, "data_stream": { "hidden": true @@ -13,5 +13,6 @@ "description": "default template for Stack deprecation logs index template installed by x-pack", "managed": true }, + "deprecated": ${xpack.deprecation.indexing.template.deprecated}, "version": ${xpack.deprecation.indexing.template.version} } diff --git a/x-pack/plugin/deprecation/qa/common/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationTestUtils.java b/x-pack/plugin/deprecation/qa/common/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationTestUtils.java index a9018c9f80a87..74c3f5b56a0b1 100644 --- a/x-pack/plugin/deprecation/qa/common/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationTestUtils.java +++ b/x-pack/plugin/deprecation/qa/common/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationTestUtils.java @@ -23,7 +23,7 @@ public class DeprecationTestUtils { /** * Same as DeprecationIndexingAppender#DEPRECATION_MESSAGES_DATA_STREAM, but that class isn't visible from here. */ - public static final String DATA_STREAM_NAME = ".logs-deprecation.elasticsearch-default"; + public static final String DATA_STREAM_NAME = ".logs-elasticsearch.deprecation-default"; static List> getIndexedDeprecations(RestClient client) throws IOException { return getIndexedDeprecations(client, null); diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java index 22637b1640b51..4607a1b6baf7a 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java @@ -32,7 +32,7 @@ @Plugin(name = "DeprecationIndexingAppender", category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE) public class DeprecationIndexingAppender extends AbstractAppender { private static final Logger logger = LogManager.getLogger(DeprecationIndexingAppender.class); - public static final String DEPRECATION_MESSAGES_DATA_STREAM = ".logs-deprecation.elasticsearch-default"; + public static final String DEPRECATION_MESSAGES_DATA_STREAM = ".logs-elasticsearch.deprecation-default"; private final Consumer requestConsumer; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java index 7fee57c724354..ad6518eccca4c 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java @@ -137,7 +137,7 @@ public void clusterChanged(ClusterChangedEvent event) { } final IndexLifecycleMetadata indexLifecycleMetadata = event.state().metadata().getProject().custom(IndexLifecycleMetadata.TYPE); - if (event.state().getMetadata().getProject().templatesV2().containsKey(".deprecation-indexing-template") + if (event.state().getMetadata().getProject().templatesV2().containsKey(".deprecation-indexing-template-9") && indexLifecycleMetadata != null && indexLifecycleMetadata.getPolicies().containsKey(".deprecation-indexing-ilm-policy")) { flushEnabled.set(true); diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java index 065053f117de0..4e5f75461316d 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java @@ -33,13 +33,15 @@ public class DeprecationIndexingTemplateRegistry extends IndexTemplateRegistry { // history (please add a comment why you increased the version here) // version 1: initial - public static final int INDEX_TEMPLATE_VERSION = 1; + // version 2: deprecated old name and renamed index pattern + public static final int INDEX_TEMPLATE_VERSION = 2; public static final String DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE = "xpack.deprecation.indexing.template.version"; public static final String DEPRECATION_INDEXING_MAPPINGS_NAME = ".deprecation-indexing-mappings"; public static final String DEPRECATION_INDEXING_SETTINGS_NAME = ".deprecation-indexing-settings"; - public static final String DEPRECATION_INDEXING_TEMPLATE_NAME = ".deprecation-indexing-template"; + public static final String DEPRECATION_INDEXING_TEMPLATE_NAME_OLD = ".deprecation-indexing-template"; + public static final String DEPRECATION_INDEXING_TEMPLATE_NAME = ".deprecation-indexing-template-9"; public static final String DEPRECATION_INDEXING_POLICY_NAME = ".deprecation-indexing-ilm-policy"; public DeprecationIndexingTemplateRegistry( @@ -89,7 +91,25 @@ protected Map getComponentTemplateConfigs() { DEPRECATION_INDEXING_TEMPLATE_NAME, "/deprecation/deprecation-indexing-template.json", INDEX_TEMPLATE_VERSION, - DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE + DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE, + Map.of( + "xpack.deprecation.indexing.template.deprecated", + "false", + "xpack.deprecation.indexing.template.pattern", + ".logs-elasticsearch.deprecation-*" + ) + ), + new IndexTemplateConfig( + DEPRECATION_INDEXING_TEMPLATE_NAME_OLD, + "/deprecation/deprecation-indexing-template.json", + INDEX_TEMPLATE_VERSION, + DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE, + Map.of( + "xpack.deprecation.indexing.template.deprecated", + "true", + "xpack.deprecation.indexing.template.pattern", + ".logs-deprecation.*" + ) ) ); From 8b92759b5a93261c000a4cbe4b48b6223aa08259 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 25 Mar 2025 11:35:40 -0700 Subject: [PATCH 2/5] Update docs/changelog/125606.yaml --- docs/changelog/125606.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/changelog/125606.yaml diff --git a/docs/changelog/125606.yaml b/docs/changelog/125606.yaml new file mode 100644 index 0000000000000..771d1af573683 --- /dev/null +++ b/docs/changelog/125606.yaml @@ -0,0 +1,13 @@ +pr: 125606 +summary: Rename deprecation index template +area: Infra/Core +type: breaking +issues: + - 125445 +breaking: + title: Rename deprecation index template + area: Infra/Core + details: Please describe the details of this change for the release notes. You can + use asciidoc. + impact: Please describe the impact of this change to users + notable: false From b0d7e7a48b16b00d937337df4c82e4750b9cec47 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 25 Mar 2025 11:41:34 -0700 Subject: [PATCH 3/5] update changelog --- docs/changelog/125606.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/changelog/125606.yaml b/docs/changelog/125606.yaml index 771d1af573683..b9b4a2c6a4b3c 100644 --- a/docs/changelog/125606.yaml +++ b/docs/changelog/125606.yaml @@ -7,7 +7,11 @@ issues: breaking: title: Rename deprecation index template area: Infra/Core - details: Please describe the details of this change for the release notes. You can - use asciidoc. - impact: Please describe the impact of this change to users + details: The deprecation datastream contains log entries for deprecations that + occured while Elasticsearch is running. The deprecation log entries had a + mismatch in their dataset name. In order to avoid changing the dataset + name in the existing datastream, a new datastream now exists. + impact: If querying for deprecations previously using the + `.logs-deprecation.elasticsearch-default` datastream, you should now use the + `.logs-elasticsearch.deprecation-default` datastream. notable: false From 041d8a32d1068bd7b51a86e749ab1aba9e8b7916 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 25 Mar 2025 11:43:48 -0700 Subject: [PATCH 4/5] iter --- docs/changelog/125606.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog/125606.yaml b/docs/changelog/125606.yaml index b9b4a2c6a4b3c..559ad4089b66d 100644 --- a/docs/changelog/125606.yaml +++ b/docs/changelog/125606.yaml @@ -1,12 +1,12 @@ pr: 125606 summary: Rename deprecation index template -area: Infra/Core +area: Logging type: breaking issues: - 125445 breaking: title: Rename deprecation index template - area: Infra/Core + area: Logging details: The deprecation datastream contains log entries for deprecations that occured while Elasticsearch is running. The deprecation log entries had a mismatch in their dataset name. In order to avoid changing the dataset From 1fcd40888d9d4dd1a6f132eae0cdc2a654e1e557 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 25 Mar 2025 11:51:02 -0700 Subject: [PATCH 5/5] iter --- docs/changelog/125606.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/125606.yaml b/docs/changelog/125606.yaml index 559ad4089b66d..d647464892d92 100644 --- a/docs/changelog/125606.yaml +++ b/docs/changelog/125606.yaml @@ -1,6 +1,6 @@ pr: 125606 summary: Rename deprecation index template -area: Logging +area: Infra/Logging type: breaking issues: - 125445