Rename deprecation index template (#125606) (#125637)

The dataset name for the deprecation logs index was previously renamed
from `deprecation.elasticsearch` to `elasticsearch.deprecation` in
order to follow the pattern of `product.group`. The deprecation index
template, however, was not updated. This causes indexing errors once
upgraded to 9.0 due to the dataset name having changed on a
constant_keyword field. In order to avoid that mismatch, this commit
renames the deprecation indexing datastream to match the dataset name.
The old template is kept in place, but marked as deprecated, so that any
deprecation logs written during upgrading to 9.x will continue to be
indexed into the old datastream.

closes #125445

Author: rjernst

docs/changelog/125606.yaml

@@ -0,0 +1,17 @@
+pr: 125606
+summary: Rename deprecation index template
+area: Infra/Logging
+type: breaking
+issues:
+ - 125445
+breaking:
+  title: Rename deprecation index template
+  area: Logging
+  details: The deprecation datastream contains log entries for deprecations that
+    occured while Elasticsearch is running. The deprecation log entries had a
+    mismatch in their dataset name. In order to avoid changing the dataset
+    name in the existing datastream, a new datastream now exists.
+  impact: If querying for deprecations previously using the
+    `.logs-deprecation.elasticsearch-default` datastream, you should now use the
+    `.logs-elasticsearch.deprecation-default` datastream.
+  notable: false

x-pack/plugin/core/template-resources/src/main/resources/deprecation/deprecation-indexing-template.json

@@ -1,5 +1,5 @@
 {
-  "index_patterns": [".logs-deprecation.*"],
+  "index_patterns": ["${xpack.deprecation.indexing.template.pattern}"],
   "priority": 1000,
   "data_stream": {
     "hidden": true
@@ -13,5 +13,6 @@
     "description": "default template for Stack deprecation logs index template installed by x-pack",
     "managed": true
   },
+  "deprecated": ${xpack.deprecation.indexing.template.deprecated},
   "version": ${xpack.deprecation.indexing.template.version}
 }

x-pack/plugin/deprecation/qa/common/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationTestUtils.java

@@ -23,7 +23,7 @@ public class DeprecationTestUtils {
     /**
      * Same as <code>DeprecationIndexingAppender#DEPRECATION_MESSAGES_DATA_STREAM</code>, but that class isn't visible from here.
      */
-    public static final String DATA_STREAM_NAME = ".logs-deprecation.elasticsearch-default";
+    public static final String DATA_STREAM_NAME = ".logs-elasticsearch.deprecation-default";
 
     static List<Map<String, Object>> getIndexedDeprecations(RestClient client) throws IOException {
         return getIndexedDeprecations(client, null);

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingAppender.java

@@ -32,7 +32,7 @@
 @Plugin(name = "DeprecationIndexingAppender", category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE)
 public class DeprecationIndexingAppender extends AbstractAppender {
     private static final Logger logger = LogManager.getLogger(DeprecationIndexingAppender.class);
-    public static final String DEPRECATION_MESSAGES_DATA_STREAM = ".logs-deprecation.elasticsearch-default";
+    public static final String DEPRECATION_MESSAGES_DATA_STREAM = ".logs-elasticsearch.deprecation-default";
 
     private final Consumer<IndexRequest> requestConsumer;
 

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingComponent.java

@@ -137,7 +137,7 @@ public void clusterChanged(ClusterChangedEvent event) {
         }
         final IndexLifecycleMetadata indexLifecycleMetadata = event.state().metadata().custom(IndexLifecycleMetadata.TYPE);
 
-        if (event.state().getMetadata().templatesV2().containsKey(".deprecation-indexing-template")
+        if (event.state().getMetadata().templatesV2().containsKey(".deprecation-indexing-template-9")
             && indexLifecycleMetadata != null
             && indexLifecycleMetadata.getPolicies().containsKey(".deprecation-indexing-ilm-policy")) {
             flushEnabled.set(true);

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/logging/DeprecationIndexingTemplateRegistry.java

@@ -33,13 +33,15 @@
 public class DeprecationIndexingTemplateRegistry extends IndexTemplateRegistry {
     // history (please add a comment why you increased the version here)
     // version 1: initial
-    public static final int INDEX_TEMPLATE_VERSION = 1;
+    // version 2: deprecated old name and renamed index pattern
+    public static final int INDEX_TEMPLATE_VERSION = 2;
 
     public static final String DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE = "xpack.deprecation.indexing.template.version";
 
     public static final String DEPRECATION_INDEXING_MAPPINGS_NAME = ".deprecation-indexing-mappings";
     public static final String DEPRECATION_INDEXING_SETTINGS_NAME = ".deprecation-indexing-settings";
-    public static final String DEPRECATION_INDEXING_TEMPLATE_NAME = ".deprecation-indexing-template";
+    public static final String DEPRECATION_INDEXING_TEMPLATE_NAME_OLD = ".deprecation-indexing-template";
+    public static final String DEPRECATION_INDEXING_TEMPLATE_NAME = ".deprecation-indexing-template-9";
     public static final String DEPRECATION_INDEXING_POLICY_NAME = ".deprecation-indexing-ilm-policy";
 
     public DeprecationIndexingTemplateRegistry(
@@ -89,7 +91,25 @@ protected Map<String, ComponentTemplate> getComponentTemplateConfigs() {
             DEPRECATION_INDEXING_TEMPLATE_NAME,
             "/deprecation/deprecation-indexing-template.json",
             INDEX_TEMPLATE_VERSION,
-            DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE
+            DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE,
+            Map.of(
+                "xpack.deprecation.indexing.template.deprecated",
+                "false",
+                "xpack.deprecation.indexing.template.pattern",
+                ".logs-elasticsearch.deprecation-*"
+            )
+        ),
+        new IndexTemplateConfig(
+            DEPRECATION_INDEXING_TEMPLATE_NAME_OLD,
+            "/deprecation/deprecation-indexing-template.json",
+            INDEX_TEMPLATE_VERSION,
+            DEPRECATION_INDEXING_TEMPLATE_VERSION_VARIABLE,
+            Map.of(
+                "xpack.deprecation.indexing.template.deprecated",
+                "true",
+                "xpack.deprecation.indexing.template.pattern",
+                ".logs-deprecation.*"
+            )
         )
     );