Add rollover to watcher history ILM policy (#82603)

In `7.x` watcher history was kept for 7 days with daily index pattern
and ILM policy that had only delete phase. In `8.x` we've moved to data
stream but policy was not updated so watcher history index would grow
forever. This change fixes that by adding rollover. 3 days rollover and
4 days delete mean that data will be kept for 7 days as before.

Author: probakowski

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -571,6 +571,7 @@ protected Set<String> preserveILMPolicyIds() {
             "ilm-history-ilm-policy",
             "slm-history-ilm-policy",
             "watch-history-ilm-policy",
+            "watch-history-ilm-policy-16",
             "ml-size-based-ilm-policy",
             "logs",
             "metrics",

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java

@@ -21,8 +21,9 @@ public final class WatcherIndexTemplateRegistryField {
     // version 13: add `allow_auto_create` setting
     // version 14: move watch history to data stream
     // version 15: remove watches and triggered watches, these are now system indices
+    // version 16: change watch history ILM policy
     // Note: if you change this, also inform the kibana team around the watcher-ui
-    public static final int INDEX_TEMPLATE_VERSION = 14;
+    public static final int INDEX_TEMPLATE_VERSION = 16;
     public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
     public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION;
     public static final String[] TEMPLATE_NAMES = new String[] { HISTORY_TEMPLATE_NAME };

x-pack/plugin/core/src/main/resources/watch-history-ilm-policy.json

@@ -1,7 +1,15 @@
 {
   "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_age": "3d",
+          "max_primary_shard_size": "50gb"
+        }
+      }
+    },
     "delete": {
-      "min_age": "7d",
+      "min_age": "4d",
       "actions": {
         "delete": {}
       }

x-pack/plugin/core/src/main/resources/watch-history.json

@@ -9,7 +9,7 @@
       "index.number_of_shards": 1,
       "index.number_of_replicas": 0,
       "index.auto_expand_replicas": "0-1",
-      "index.lifecycle.name": "watch-history-ilm-policy",
+      "index.lifecycle.name": "watch-history-ilm-policy-16",
       "index.hidden": true,
       "index.format": 6
     },

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java

@@ -67,7 +67,7 @@ protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
     }
 
     private static final List<LifecyclePolicy> LIFECYCLE_POLICIES = List.of(
-        new LifecyclePolicyConfig("watch-history-ilm-policy", "/watch-history-ilm-policy.json").load(
+        new LifecyclePolicyConfig("watch-history-ilm-policy-16", "/watch-history-ilm-policy.json").load(
             LifecyclePolicyConfig.DEFAULT_X_CONTENT_REGISTRY
         )
     );

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java

@@ -140,7 +140,7 @@ public void testThatNonExistingTemplatesAreAddedImmediately() {
             .filter(r -> r.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME))
             .findFirst()
             .orElseThrow(() -> new AssertionError("expected the watch history template to be put"));
-        assertThat(req.indexTemplate().template().settings().get("index.lifecycle.name"), equalTo("watch-history-ilm-policy"));
+        assertThat(req.indexTemplate().template().settings().get("index.lifecycle.name"), equalTo("watch-history-ilm-policy-16"));
     }
 
     public void testThatNonExistingTemplatesAreAddedEvenWithILMUsageDisabled() {