Make .kibana-reporting a system data stream

modules/kibana/src/main/java/module-info.java

@@ -9,4 +9,5 @@
 
 module org.elasticsearch.kibana {
     requires org.elasticsearch.server;
+    requires org.elasticsearch.xcontent;
 }

modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java

@@ -9,14 +9,26 @@
 
 package org.elasticsearch.kibana;
 
+import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
+import org.elasticsearch.cluster.metadata.Template;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.SystemDataStreamDescriptor;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.indices.SystemIndexDescriptor.Type;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SystemIndexPlugin;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.indices.ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS;
 
 public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
 
@@ -31,6 +43,61 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
         .setAllowsTemplates()
         .build();
 
+    public static final SystemDataStreamDescriptor KIBANA_REPORTING_DS_DESCRIPTOR;
+
+    private static String loadTemplateSource() throws IOException {
+        try (InputStream is = KibanaPlugin.class.getResourceAsStream("/kibana-reporting-template.json")) {
+            if (is == null) {
+                throw new IOException(
+                    "Kibana reporting template [/kibana-reporting-template.json] not found in kibana template resources."
+                );
+            }
+            return new String(is.readAllBytes(), StandardCharsets.UTF_8);
+        }
+    }
+
+    static {
+        try {
+            final String source = loadTemplateSource();
+            KIBANA_REPORTING_DS_DESCRIPTOR = new SystemDataStreamDescriptor(
+                ".kibana-reporting",
+                "system data stream for reporting",
+                SystemDataStreamDescriptor.Type.EXTERNAL,
+                ComposableIndexTemplate.builder()
+                    .indexPatterns(List.of(".kibana-reporting"))
+                    .priority(200L)
+                    .version(15L)
+                    .allowAutoCreate(true)
+                    .deprecated(false)
+                    .ignoreMissingComponentTemplates(List.of("kibana-reporting@custom"))
+                    .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(true, false))
+                    .metadata(Map.of("managed", "true", "description", "default kibana reporting template installed by elasticsearch"))
+                    .componentTemplates(List.of("kibana-reporting@settings", "kibana-reporting@custom"))
+                    .template(
+                        Template.builder()
+                            .mappings(CompressedXContent.fromJSON(source))
+                            .lifecycle(DataStreamLifecycle.dataLifecycleBuilder().enabled(true))
+                    )
+                    .build(),
+                Map.of(
+                    "kibana-reporting@settings",
+                    new ComponentTemplate(
+                        Template.builder()
+                            .settings(Settings.builder().put("index.number_of_shards", 1).put("index.auto_expand_replicas", "0-1"))
+                            .build(),
+                        null,
+                        null
+                    )
+                ),
+                KIBANA_PRODUCT_ORIGIN,
+                KIBANA_PRODUCT_ORIGIN.getFirst(),
+                DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
+            );
+        } catch (IOException e) {
+            throw new RuntimeException("unable to read kibana reporting template JSON", e);
+        }
+    }
+
     public static final SystemIndexDescriptor REPORTING_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
         .setIndexPattern(".reporting-*")
         .setDescription("system index for reporting")
@@ -60,6 +127,11 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
         .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN)
         .build();
 
+    @Override
+    public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
+        return List.of(KIBANA_REPORTING_DS_DESCRIPTOR);
+    }
+
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
         return List.of(

modules/kibana/src/main/resources/kibana-reporting-template.json

@@ -0,0 +1,164 @@
+{
+  "properties": {
+    "meta": {
+      "properties": {
+        "objectType": {
+          "type": "text",
+          "fields": {
+            "keyword": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
+        },
+        "layout": {
+          "type": "text",
+          "fields": {
+            "keyword": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
+        },
+        "isDeprecated": {
+          "type": "boolean"
+        }
+      }
+    },
+    "migration_version": {
+      "type": "keyword"
+    },
+    "jobtype": {
+      "type": "keyword"
+    },
+    "scheduled_report_id": {
+      "type": "keyword"
+    },
+    "payload": {
+      "type": "object",
+      "enabled": false
+    },
+    "priority": {
+      "type": "byte"
+    },
+    "timeout": {
+      "type": "long"
+    },
+    "process_expiration": {
+      "type": "date"
+    },
+    "created_by": {
+      "type": "keyword"
+    },
+    "created_at": {
+      "type": "date"
+    },
+    "started_at": {
+      "type": "date"
+    },
+    "completed_at": {
+      "type": "date"
+    },
+    "attempts": {
+      "type": "short"
+    },
+    "max_attempts": {
+      "type": "short"
+    },
+    "kibana_name": {
+      "type": "keyword"
+    },
+    "kibana_id": {
+      "type": "keyword"
+    },
+    "status": {
+      "type": "keyword"
+    },
+    "space_id": {
+      "type": "keyword"
+    },
+    "parent_id": {
+      "type": "keyword"
+    },
+    "output": {
+      "type": "object",
+      "properties": {
+        "chunk": {
+          "type": "long"
+        },
+        "content": {
+          "type": "object",
+          "enabled": false
+        },
+        "content_type": {
+          "type": "keyword"
+        },
+        "csv_contains_formulas": {
+          "type": "boolean"
+        },
+        "error_code": {
+          "type": "keyword"
+        },
+        "max_size_reached": {
+          "type": "boolean"
+        },
+        "size": {
+          "type": "long"
+        },
+        "warnings": {
+          "type": "text"
+        }
+      }
+    },
+    "metrics": {
+      "type": "object",
+      "properties": {
+        "csv": {
+          "type": "object",
+          "properties": {
+            "rows": {
+              "type": "long"
+            }
+          }
+        },
+        "pdf": {
+          "type": "object",
+          "properties": {
+            "pages": {
+              "type": "long"
+            },
+            "cpu": {
+              "type": "double"
+            },
+            "cpuInPercentage": {
+              "type": "double"
+            },
+            "memory": {
+              "type": "long"
+            },
+            "memoryInMegabytes": {
+              "type": "double"
+            }
+          }
+        },
+        "png": {
+          "type": "object",
+          "properties": {
+            "cpu": {
+              "type": "double"
+            },
+            "cpuInPercentage": {
+              "type": "double"
+            },
+            "memory": {
+              "type": "long"
+            },
+            "memoryInMegabytes": {
+              "type": "double"
+            }
+          }
+        }
+      }
+    }
+  }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java

@@ -195,6 +195,27 @@ public class InternalUsers {
                         )
                     )
                     .allowRestrictedIndices(true)
+                    .build(),
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(
+                        // System data stream for Kibana reporting (see KibanaPlugin#KIBANA_REPORTING_DS_DESCRIPTOR)
+                        ".kibana-reporting"
+                    )
+                    .privileges(
+                        filterNonNull(
+                            // needed to rollover failure store
+                            "manage_failure_store",
+                            "delete_index",
+                            RolloverAction.NAME,
+                            ForceMergeAction.NAME + "*",
+                            // indices stats is used by rollover, so we need to grant it here
+                            IndicesStatsAction.NAME + "*",
+                            TransportUpdateSettingsAction.TYPE.name(),
+                            DownsampleAction.NAME,
+                            TransportAddIndexBlockAction.TYPE.name()
+                        )
+                    )
+                    .allowRestrictedIndices(true)
                     .build() },
             null,
             null,

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java

@@ -63,6 +63,7 @@
 import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.SECURITY_MAIN_ALIAS;
 import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.SECURITY_TOKENS_ALIAS;
 import static org.elasticsearch.xpack.core.security.user.UsernamesField.REINDEX_DATA_STREAM_NAME;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
@@ -244,10 +245,17 @@ public void testDataStreamLifecycleUser() {
         assertThat(role.application(), is(ApplicationPermission.NONE));
         assertThat(role.remoteIndices(), is(RemoteIndicesPermission.NONE));
 
-        final List<String> allowedSystemDataStreams = Arrays.asList(".fleet-actions-results", ".fleet-fileds*");
+        final List<String> allowedFleetSystemDataStreams = Arrays.asList(".fleet-actions-results", ".fleet-fileds*");
+        final List<String> allowedKibanaSystemDataStreams = List.of(".kibana-reporting");
         for (var group : role.indices().groups()) {
             if (group.allowRestrictedIndices()) {
-                assertThat(group.indices(), arrayContaining(allowedSystemDataStreams.toArray(new String[0])));
+                assertThat(
+                    group.indices(),
+                    anyOf(
+                        arrayContaining(allowedFleetSystemDataStreams.toArray(new String[0])),
+                        arrayContaining(allowedKibanaSystemDataStreams.toArray(new String[0]))
+                    )
+                );
             }
         }
 
@@ -290,7 +298,25 @@ public void testDataStreamLifecycleUser() {
             true
         );
 
-        allowedSystemDataStreams.forEach(allowedSystemDataStream -> {
+        allowedFleetSystemDataStreams.forEach(allowedSystemDataStream -> {
+            checkIndexAccess(role, randomFrom(sampleSystemDataStreamActions), allowedSystemDataStream, true);
+            checkIndexAccess(
+                role,
+                randomFrom(sampleSystemDataStreamActions),
+                DataStream.BACKING_INDEX_PREFIX + allowedSystemDataStream + randomAlphaOfLengthBetween(4, 8),
+                true
+            );
+
+            checkIndexAccess(role, randomFrom(sampleSystemDataStreamActions), allowedSystemDataStream + "::failures", true);
+            checkIndexAccess(
+                role,
+                randomFrom(sampleSystemDataStreamActions),
+                DataStream.FAILURE_STORE_PREFIX + allowedSystemDataStream + randomAlphaOfLengthBetween(4, 8),
+                true
+            );
+        });
+
+        allowedKibanaSystemDataStreams.forEach(allowedSystemDataStream -> {
             checkIndexAccess(role, randomFrom(sampleSystemDataStreamActions), allowedSystemDataStream, true);
             checkIndexAccess(
                 role,

x-pack/plugin/stack/build.gradle

@@ -48,6 +48,7 @@ tasks.named('javaRestTest') { task ->
 tasks.named("yamlRestCompatTestTransform") { task ->
   task.skipTest("stack/10_basic/Test kibana reporting index auto creation", "warning does not exist for compatibility")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
+  task.skipTest("stack/10_basic/Test stack template installation", ".kibana-reporting templates are no longer installed due to transition to system index")
 }
 
 configurations {