Add default ILM policies and switch to ILM for apm-data plugin (#115687)

lahsivjar · web-flow · commit 6325e46231c1 · 2024-11-13T15:01:59.000+01:00
diff --git a/docs/changelog/115687.yaml b/docs/changelog/115687.yaml
@@ -0,0 +1,5 @@
+pr: 115687
+summary: Add default ILM policies and switch to ILM for apm-data plugin
+area: Data streams
+type: feature
+issues: []
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -817,7 +817,26 @@ protected Set<String> preserveILMPolicyIds() {
             ".fleet-file-tohost-meta-ilm-policy",
             ".deprecation-indexing-ilm-policy",
             ".monitoring-8-ilm-policy",
-            "behavioral_analytics-events-default_policy"
+            "behavioral_analytics-events-default_policy",
+            "logs-apm.app_logs-default_policy",
+            "logs-apm.error_logs-default_policy",
+            "metrics-apm.app_metrics-default_policy",
+            "metrics-apm.internal_metrics-default_policy",
+            "metrics-apm.service_destination_10m_metrics-default_policy",
+            "metrics-apm.service_destination_1m_metrics-default_policy",
+            "metrics-apm.service_destination_60m_metrics-default_policy",
+            "metrics-apm.service_summary_10m_metrics-default_policy",
+            "metrics-apm.service_summary_1m_metrics-default_policy",
+            "metrics-apm.service_summary_60m_metrics-default_policy",
+            "metrics-apm.service_transaction_10m_metrics-default_policy",
+            "metrics-apm.service_transaction_1m_metrics-default_policy",
+            "metrics-apm.service_transaction_60m_metrics-default_policy",
+            "metrics-apm.transaction_10m_metrics-default_policy",
+            "metrics-apm.transaction_1m_metrics-default_policy",
+            "metrics-apm.transaction_60m_metrics-default_policy",
+            "traces-apm.rum_traces-default_policy",
+            "traces-apm.sampled_traces-default_policy",
+            "traces-apm.traces-default_policy"
         );
     }
 
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.app-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: logs-apm.app_logs-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: logs-apm.error_logs-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.app-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.app_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.internal-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.internal_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.10m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.service_destination_10m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.1m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.service_destination_1m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination.60m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.service_destination_60m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.10m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.service_summary_10m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.1m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.service_summary_1m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary.60m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.service_summary_60m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.10m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.service_transaction_10m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.1m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.service_transaction_1m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction.60m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.service_transaction_60m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.10m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.transaction_10m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.1m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.transaction_1m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction.60m-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: metrics-apm.transaction_60m_metrics-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: traces-apm.traces-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: traces-apm.rum_traces-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.sampled-fallback@ilm.yaml
@@ -8,4 +8,3 @@ template:
     index:
       lifecycle:
         name: traces-apm.sampled_traces-default_policy
-        prefer_ilm: false
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.app_logs-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.app_logs-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 30d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 10d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.error_logs-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/logs-apm.error_logs-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 30d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 10d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.app_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.app_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 30d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 90d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.internal_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.internal_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 30d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 90d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_10m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_10m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 14d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 180d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_1m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_1m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 7d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 90d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_60m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_destination_60m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 30d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 390d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_10m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_10m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 14d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 180d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_1m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_1m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 7d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 90d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_60m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_summary_60m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 30d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 390d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_10m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_10m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 14d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 180d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_1m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_1m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 7d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 90d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_60m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.service_transaction_60m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 30d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 390d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_10m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_10m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 14d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 180d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_1m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_1m_metrics-default_policy.yaml
@@ -0,0 +1,16 @@
+---
+_meta:
+  description: Default ILM policy for APM managed datastreams
+  managed: true
+phases:
+  hot:
+    actions:
+      rollover:
+        max_age: 7d
+        max_primary_shard_size: 50gb
+      set_priority:
+        priority: 100
+  delete:
+    min_age: 90d
+    actions:
+      delete: {}
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_60m_metrics-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/metrics-apm.transaction_60m_metrics-default_policy.yaml
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.rum_traces-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.rum_traces-default_policy.yaml
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.sampled_traces-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.sampled_traces-default_policy.yaml
diff --git a/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.traces-default_policy.yaml b/x-pack/plugin/apm-data/src/main/resources/lifecycle-policies/traces-apm.traces-default_policy.yaml
diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml
diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecyclePolicyUtils.java
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/YamlTemplateRegistry.java
