From efe520ae4b196d4f3fd3ca939b6d68cb44fdd42d Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 3 Jan 2025 14:35:19 -0700 Subject: [PATCH 1/6] WIP make kibana reporting a system data stream --- modules/kibana/src/main/java/module-info.java | 1 + .../elasticsearch/kibana/KibanaPlugin.java | 212 ++++++++++++++++++ .../xpack/stack/StackTemplateRegistry.java | 20 -- .../stack/StackTemplateRegistryTests.java | 5 - .../rest-api-spec/test/stack/10_basic.yml | 45 ---- 5 files changed, 213 insertions(+), 70 deletions(-) diff --git a/modules/kibana/src/main/java/module-info.java b/modules/kibana/src/main/java/module-info.java index 6c5692e7e04a1..364febb32180e 100644 --- a/modules/kibana/src/main/java/module-info.java +++ b/modules/kibana/src/main/java/module-info.java @@ -9,4 +9,5 @@ module org.elasticsearch.kibana { requires org.elasticsearch.server; + requires org.elasticsearch.xcontent; } diff --git a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java index 42c92c81ff2fc..eb6454fe7f11d 100644 --- a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java +++ b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java @@ -9,14 +9,24 @@ package org.elasticsearch.kibana; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor.Type; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; +import java.io.IOException; import java.util.Collection; import java.util.List; +import java.util.Map; + +import static org.elasticsearch.indices.ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS; public class KibanaPlugin extends Plugin implements SystemIndexPlugin { @@ -31,6 +41,203 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin { .setAllowsTemplates() .build(); + public static final SystemDataStreamDescriptor KIBANA_REPORTING_DS_DESCRIPTOR; + + static { + try { + KIBANA_REPORTING_DS_DESCRIPTOR = new SystemDataStreamDescriptor( + ".kibana-reporting", + "system data stream for reporting", + SystemDataStreamDescriptor.Type.EXTERNAL, + ComposableIndexTemplate.builder() + .indexPatterns(List.of(".kibana-reporting")) + .priority(200L) + .version(15L) + .allowAutoCreate(true) + .deprecated(false) + .ignoreMissingComponentTemplates(List.of("kibana-reporting@custom")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(true, false)) + .metadata(Map.of("managed", "true", "description", "default kibana reporting template installed by elasticsearch")) + .componentTemplates(List.of("kibana-reporting@settings", "kibana-reporting@custom")) + .template(Template.builder().mappings(CompressedXContent.fromJSON(""" + { + "properties" : { + "kibana_name" : { + "type" : "keyword" + }, + "created_at" : { + "type" : "date" + }, + "priority" : { + "type" : "byte" + }, + "jobtype" : { + "type" : "keyword" + }, + "created_by" : { + "type" : "keyword" + }, + "migration_version" : { + "type" : "keyword" + }, + "timeout" : { + "type" : "long" + }, + "kibana_id" : { + "type" : "keyword" + }, + "output" : { + "type" : "object", + "properties" : { + "content_type" : { + "type" : "keyword" + }, + "size" : { + "type" : "long" + }, + "csv_contains_formulas" : { + "type" : "boolean" + }, + "warnings" : { + "type" : "text" + }, + "chunk" : { + "type" : "long" + }, + "error_code" : { + "type" : "keyword" + }, + "max_size_reached" : { + "type" : "boolean" + }, + "content" : { + "type" : "object", + "enabled" : false + } + } + }, + "process_expiration" : { + "type" : "date" + }, + "completed_at" : { + "type" : "date" + }, + "payload" : { + "type" : "object", + "enabled" : false + }, + "meta" : { + "properties" : { + "layout" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "isDeprecated" : { + "type" : "boolean" + }, + "objectType" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + } + } + }, + "parent_id" : { + "type" : "keyword" + }, + "max_attempts" : { + "type" : "short" + }, + "started_at" : { + "type" : "date" + }, + "metrics" : { + "type" : "object", + "properties" : { + "pdf" : { + "type" : "object", + "properties" : { + "pages" : { + "type" : "long" + }, + "memory" : { + "type" : "long" + }, + "cpuInPercentage" : { + "type" : "double" + }, + "cpu" : { + "type" : "double" + }, + "memoryInMegabytes" : { + "type" : "double" + } + } + }, + "csv" : { + "type" : "object", + "properties" : { + "rows" : { + "type" : "long" + } + } + }, + "png" : { + "type" : "object", + "properties" : { + "memory" : { + "type" : "long" + }, + "cpuInPercentage" : { + "type" : "double" + }, + "cpu" : { + "type" : "double" + }, + "memoryInMegabytes" : { + "type" : "double" + } + } + } + } + }, + "attempts" : { + "type" : "short" + }, + "status" : { + "type" : "keyword" + } + } + }""")).lifecycle(DataStreamLifecycle.dataLifecycleBuilder().enabled(true))) + .build(), + Map.of( + "kibana-reporting@settings", + new ComponentTemplate( + Template.builder() + .settings(Settings.builder().put("index.number_of_shards", 1).put("index.auto_expand_replicas", "0-1")) + .build(), + null, + null + ) + ), + KIBANA_PRODUCT_ORIGIN, + KIBANA_PRODUCT_ORIGIN.getFirst(), + DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS + ); + } catch (IOException e) { + throw new RuntimeException("unable to read kibana reporting template JSON", e); + } + } + public static final SystemIndexDescriptor REPORTING_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder() .setIndexPattern(".reporting-*") .setDescription("system index for reporting") @@ -60,6 +267,11 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin { .setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN) .build(); + @Override + public Collection getSystemDataStreamDescriptors() { + return List.of(KIBANA_REPORTING_DS_DESCRIPTOR); + } + @Override public Collection getSystemIndexDescriptors(Settings settings) { return List.of( diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index a2f8dd91176e2..c198b4efb13d6 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -107,12 +107,6 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { public static final String AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME = "agentless@settings"; public static final String AGENTLESS_INDEX_TEMPLATE_NAME = "agentless"; - /////////////////////////////////// - // Kibana reporting template - /////////////////////////////////// - public static final String KIBANA_REPORTING_INDEX_TEMPLATE_NAME = ".kibana-reporting"; - public static final String KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME = "kibana-reporting@settings"; - public StackTemplateRegistry( Settings nodeSettings, ClusterService clusterService, @@ -219,13 +213,6 @@ private Map loadComponentTemplateConfigs() { REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, - "/kibana-reporting@settings.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES ) )) { try { @@ -314,13 +301,6 @@ protected Map getComponentTemplateConfigs() { REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, ADDITIONAL_TEMPLATE_VARIABLES - ), - new IndexTemplateConfig( - KIBANA_REPORTING_INDEX_TEMPLATE_NAME, - "/kibana-reporting@template.json", - REGISTRY_VERSION, - TEMPLATE_VERSION_VARIABLE, - ADDITIONAL_TEMPLATE_VARIABLES ) ); diff --git a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java index 970ba67e25667..6646b2a97800d 100644 --- a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java +++ b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java @@ -418,7 +418,6 @@ public void testSameOrHigherVersionTemplateNotUpgraded() { versions.put(StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); - versions.put(StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes); @@ -484,10 +483,6 @@ public void testSameOrHigherVersionTemplateNotUpgraded() { StackTemplateRegistry.AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) ); - versions.put( - StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, - StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) - ); versions.put( StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) diff --git a/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml index 3b8bfa9dfb7c8..2843b3e8e8e73 100644 --- a/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml +++ b/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml @@ -74,10 +74,6 @@ setup: indices.get_index_template: name: synthetics - - do: - indices.get_index_template: - name: .kibana-reporting - --- "Test logs index auto creation": - do: @@ -221,47 +217,6 @@ setup: indices.delete_data_stream: name: synthetics-foo-bar ---- -"Test kibana reporting index auto creation": - - requires: - test_runner_features: ["headers"] - - - do: - headers: { X-elastic-product-origin: kibana } - index: - index: .kibana-reporting-foo - body: - "@timestamp": "2020-01-01" - jobtype: "thing" - - - do: - indices.get_data_stream: - name: .kibana-reporting-foo - - - match: { data_streams.0.name: .kibana-reporting-foo } - - match: { data_streams.0.hidden: true } - - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 1 } - - length: { data_streams.0.indices: 1 } - - match: { data_streams.0.lifecycle.enabled: true } - - match: { data_streams.0.indices.0.index_name: '/\.ds-.kibana-reporting-foo-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - - set: { data_streams.0.indices.0.index_name: idx0name } - - - do: - indices.get: - index: $idx0name - - - is_true: .$idx0name.settings - - is_true: .$idx0name.mappings - - match: { .$idx0name.mappings.properties.meta.properties.objectType.type: "text" } - - match: { .$idx0name.mappings.properties.meta.properties.layout.type: "text" } - - match: { .$idx0name.data_stream: ".kibana-reporting-foo" } - - - do: - indices.delete_data_stream: - name: .kibana-reporting-foo - --- "Test wrong data_stream type": From d19c99511d6263e6adae6ae3dfc6bb21e4a2b60b Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Mon, 23 Jun 2025 16:04:52 -0600 Subject: [PATCH 2/6] Add skip for kibana reporting test that cannot run --- x-pack/plugin/stack/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/stack/build.gradle b/x-pack/plugin/stack/build.gradle index f0af1e618cac6..4f8a6247a4d17 100644 --- a/x-pack/plugin/stack/build.gradle +++ b/x-pack/plugin/stack/build.gradle @@ -48,6 +48,7 @@ tasks.named('javaRestTest') { task -> tasks.named("yamlRestCompatTestTransform") { task -> task.skipTest("stack/10_basic/Test kibana reporting index auto creation", "warning does not exist for compatibility") task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0") + task.skipTest("stack/10_basic/Test stack template installation", ".kibana-reporting templates are no longer installed due to transition to system index") } configurations { From 2828e18cd9c23847fdc3bc9ef8d2ef76a32dbaad Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 25 Jun 2025 17:09:24 -0600 Subject: [PATCH 3/6] Read the mappings from a file --- .../elasticsearch/kibana/KibanaPlugin.java | 178 ++---------------- .../resources/kibana-reporting-template.json | 158 ++++++++++++++++ 2 files changed, 177 insertions(+), 159 deletions(-) create mode 100644 modules/kibana/src/main/resources/kibana-reporting-template.json diff --git a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java index eb6454fe7f11d..4179f762a4108 100644 --- a/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java +++ b/modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java @@ -22,6 +22,8 @@ import org.elasticsearch.plugins.SystemIndexPlugin; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.List; import java.util.Map; @@ -43,8 +45,20 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin { public static final SystemDataStreamDescriptor KIBANA_REPORTING_DS_DESCRIPTOR; + private static String loadTemplateSource() throws IOException { + try (InputStream is = KibanaPlugin.class.getResourceAsStream("/kibana-reporting-template.json")) { + if (is == null) { + throw new IOException( + "Kibana reporting template [/kibana-reporting-template.json] not found in kibana template resources." + ); + } + return new String(is.readAllBytes(), StandardCharsets.UTF_8); + } + } + static { try { + final String source = loadTemplateSource(); KIBANA_REPORTING_DS_DESCRIPTOR = new SystemDataStreamDescriptor( ".kibana-reporting", "system data stream for reporting", @@ -59,165 +73,11 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin { .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(true, false)) .metadata(Map.of("managed", "true", "description", "default kibana reporting template installed by elasticsearch")) .componentTemplates(List.of("kibana-reporting@settings", "kibana-reporting@custom")) - .template(Template.builder().mappings(CompressedXContent.fromJSON(""" - { - "properties" : { - "kibana_name" : { - "type" : "keyword" - }, - "created_at" : { - "type" : "date" - }, - "priority" : { - "type" : "byte" - }, - "jobtype" : { - "type" : "keyword" - }, - "created_by" : { - "type" : "keyword" - }, - "migration_version" : { - "type" : "keyword" - }, - "timeout" : { - "type" : "long" - }, - "kibana_id" : { - "type" : "keyword" - }, - "output" : { - "type" : "object", - "properties" : { - "content_type" : { - "type" : "keyword" - }, - "size" : { - "type" : "long" - }, - "csv_contains_formulas" : { - "type" : "boolean" - }, - "warnings" : { - "type" : "text" - }, - "chunk" : { - "type" : "long" - }, - "error_code" : { - "type" : "keyword" - }, - "max_size_reached" : { - "type" : "boolean" - }, - "content" : { - "type" : "object", - "enabled" : false - } - } - }, - "process_expiration" : { - "type" : "date" - }, - "completed_at" : { - "type" : "date" - }, - "payload" : { - "type" : "object", - "enabled" : false - }, - "meta" : { - "properties" : { - "layout" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - }, - "isDeprecated" : { - "type" : "boolean" - }, - "objectType" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - } - } - }, - "parent_id" : { - "type" : "keyword" - }, - "max_attempts" : { - "type" : "short" - }, - "started_at" : { - "type" : "date" - }, - "metrics" : { - "type" : "object", - "properties" : { - "pdf" : { - "type" : "object", - "properties" : { - "pages" : { - "type" : "long" - }, - "memory" : { - "type" : "long" - }, - "cpuInPercentage" : { - "type" : "double" - }, - "cpu" : { - "type" : "double" - }, - "memoryInMegabytes" : { - "type" : "double" - } - } - }, - "csv" : { - "type" : "object", - "properties" : { - "rows" : { - "type" : "long" - } - } - }, - "png" : { - "type" : "object", - "properties" : { - "memory" : { - "type" : "long" - }, - "cpuInPercentage" : { - "type" : "double" - }, - "cpu" : { - "type" : "double" - }, - "memoryInMegabytes" : { - "type" : "double" - } - } - } - } - }, - "attempts" : { - "type" : "short" - }, - "status" : { - "type" : "keyword" - } - } - }""")).lifecycle(DataStreamLifecycle.dataLifecycleBuilder().enabled(true))) + .template( + Template.builder() + .mappings(CompressedXContent.fromJSON(source)) + .lifecycle(DataStreamLifecycle.dataLifecycleBuilder().enabled(true)) + ) .build(), Map.of( "kibana-reporting@settings", diff --git a/modules/kibana/src/main/resources/kibana-reporting-template.json b/modules/kibana/src/main/resources/kibana-reporting-template.json new file mode 100644 index 0000000000000..81707fd4b3ba1 --- /dev/null +++ b/modules/kibana/src/main/resources/kibana-reporting-template.json @@ -0,0 +1,158 @@ +{ + "properties" : { + "kibana_name" : { + "type" : "keyword" + }, + "created_at" : { + "type" : "date" + }, + "priority" : { + "type" : "byte" + }, + "jobtype" : { + "type" : "keyword" + }, + "created_by" : { + "type" : "keyword" + }, + "migration_version" : { + "type" : "keyword" + }, + "timeout" : { + "type" : "long" + }, + "kibana_id" : { + "type" : "keyword" + }, + "output" : { + "type" : "object", + "properties" : { + "content_type" : { + "type" : "keyword" + }, + "size" : { + "type" : "long" + }, + "csv_contains_formulas" : { + "type" : "boolean" + }, + "warnings" : { + "type" : "text" + }, + "chunk" : { + "type" : "long" + }, + "error_code" : { + "type" : "keyword" + }, + "max_size_reached" : { + "type" : "boolean" + }, + "content" : { + "type" : "object", + "enabled" : false + } + } + }, + "process_expiration" : { + "type" : "date" + }, + "completed_at" : { + "type" : "date" + }, + "payload" : { + "type" : "object", + "enabled" : false + }, + "meta" : { + "properties" : { + "layout" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + }, + "isDeprecated" : { + "type" : "boolean" + }, + "objectType" : { + "type" : "text", + "fields" : { + "keyword" : { + "ignore_above" : 256, + "type" : "keyword" + } + } + } + } + }, + "parent_id" : { + "type" : "keyword" + }, + "max_attempts" : { + "type" : "short" + }, + "started_at" : { + "type" : "date" + }, + "metrics" : { + "type" : "object", + "properties" : { + "pdf" : { + "type" : "object", + "properties" : { + "pages" : { + "type" : "long" + }, + "memory" : { + "type" : "long" + }, + "cpuInPercentage" : { + "type" : "double" + }, + "cpu" : { + "type" : "double" + }, + "memoryInMegabytes" : { + "type" : "double" + } + } + }, + "csv" : { + "type" : "object", + "properties" : { + "rows" : { + "type" : "long" + } + } + }, + "png" : { + "type" : "object", + "properties" : { + "memory" : { + "type" : "long" + }, + "cpuInPercentage" : { + "type" : "double" + }, + "cpu" : { + "type" : "double" + }, + "memoryInMegabytes" : { + "type" : "double" + } + } + } + } + }, + "attempts" : { + "type" : "short" + }, + "status" : { + "type" : "keyword" + } + } +} From 9f916e6e502352fdaeab9a57fbaf28270ffa7fb6 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 27 Jun 2025 15:02:33 -0600 Subject: [PATCH 4/6] Update to the latest verison of the mappings --- .../resources/kibana-reporting-template.json | 244 +++++++++--------- 1 file changed, 125 insertions(+), 119 deletions(-) diff --git a/modules/kibana/src/main/resources/kibana-reporting-template.json b/modules/kibana/src/main/resources/kibana-reporting-template.json index 81707fd4b3ba1..6f1272f172f92 100644 --- a/modules/kibana/src/main/resources/kibana-reporting-template.json +++ b/modules/kibana/src/main/resources/kibana-reporting-template.json @@ -1,158 +1,164 @@ { - "properties" : { - "kibana_name" : { - "type" : "keyword" + "properties": { + "meta": { + "properties": { + "objectType": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "layout": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "isDeprecated": { + "type": "boolean" + } + } + }, + "migration_version": { + "type": "keyword" + }, + "jobtype": { + "type": "keyword" + }, + "scheduled_report_id": { + "type": "keyword" + }, + "payload": { + "type": "object", + "enabled": false + }, + "priority": { + "type": "byte" + }, + "timeout": { + "type": "long" + }, + "process_expiration": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, + "created_at": { + "type": "date" + }, + "started_at": { + "type": "date" }, - "created_at" : { - "type" : "date" + "completed_at": { + "type": "date" }, - "priority" : { - "type" : "byte" + "attempts": { + "type": "short" }, - "jobtype" : { - "type" : "keyword" + "max_attempts": { + "type": "short" }, - "created_by" : { - "type" : "keyword" + "kibana_name": { + "type": "keyword" }, - "migration_version" : { - "type" : "keyword" + "kibana_id": { + "type": "keyword" }, - "timeout" : { - "type" : "long" + "status": { + "type": "keyword" }, - "kibana_id" : { - "type" : "keyword" + "space_id": { + "type": "keyword" }, - "output" : { - "type" : "object", - "properties" : { - "content_type" : { - "type" : "keyword" + "parent_id": { + "type": "keyword" + }, + "output": { + "type": "object", + "properties": { + "chunk": { + "type": "long" }, - "size" : { - "type" : "long" + "content": { + "type": "object", + "enabled": false }, - "csv_contains_formulas" : { - "type" : "boolean" + "content_type": { + "type": "keyword" }, - "warnings" : { - "type" : "text" + "csv_contains_formulas": { + "type": "boolean" }, - "chunk" : { - "type" : "long" + "error_code": { + "type": "keyword" }, - "error_code" : { - "type" : "keyword" + "max_size_reached": { + "type": "boolean" }, - "max_size_reached" : { - "type" : "boolean" + "size": { + "type": "long" }, - "content" : { - "type" : "object", - "enabled" : false + "warnings": { + "type": "text" } } }, - "process_expiration" : { - "type" : "date" - }, - "completed_at" : { - "type" : "date" - }, - "payload" : { - "type" : "object", - "enabled" : false - }, - "meta" : { - "properties" : { - "layout" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" + "metrics": { + "type": "object", + "properties": { + "csv": { + "type": "object", + "properties": { + "rows": { + "type": "long" } } }, - "isDeprecated" : { - "type" : "boolean" - }, - "objectType" : { - "type" : "text", - "fields" : { - "keyword" : { - "ignore_above" : 256, - "type" : "keyword" - } - } - } - } - }, - "parent_id" : { - "type" : "keyword" - }, - "max_attempts" : { - "type" : "short" - }, - "started_at" : { - "type" : "date" - }, - "metrics" : { - "type" : "object", - "properties" : { - "pdf" : { - "type" : "object", - "properties" : { - "pages" : { - "type" : "long" + "pdf": { + "type": "object", + "properties": { + "pages": { + "type": "long" }, - "memory" : { - "type" : "long" + "cpu": { + "type": "double" }, - "cpuInPercentage" : { - "type" : "double" + "cpuInPercentage": { + "type": "double" }, - "cpu" : { - "type" : "double" + "memory": { + "type": "long" }, - "memoryInMegabytes" : { - "type" : "double" + "memoryInMegabytes": { + "type": "double" } } }, - "csv" : { - "type" : "object", - "properties" : { - "rows" : { - "type" : "long" - } - } - }, - "png" : { - "type" : "object", - "properties" : { - "memory" : { - "type" : "long" + "png": { + "type": "object", + "properties": { + "cpu": { + "type": "double" }, - "cpuInPercentage" : { - "type" : "double" + "cpuInPercentage": { + "type": "double" }, - "cpu" : { - "type" : "double" + "memory": { + "type": "long" }, - "memoryInMegabytes" : { - "type" : "double" + "memoryInMegabytes": { + "type": "double" } } } } - }, - "attempts" : { - "type" : "short" - }, - "status" : { - "type" : "keyword" } } } From cdd3935af409ed5891dbc264142e633464b67246 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 27 Jun 2025 15:07:46 -0600 Subject: [PATCH 5/6] Grant permissions for DLM on .kibana-reporting --- .../core/security/user/InternalUsers.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java index a2610d6392362..2529bba9b3f08 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java @@ -195,6 +195,27 @@ public class InternalUsers { ) ) .allowRestrictedIndices(true) + .build(), + RoleDescriptor.IndicesPrivileges.builder() + .indices( + // System data stream for Kibana reporting (see KibanaPlugin#KIBANA_REPORTING_DS_DESCRIPTOR) + ".kibana-reporting" + ) + .privileges( + filterNonNull( + // needed to rollover failure store + "manage_failure_store", + "delete_index", + RolloverAction.NAME, + ForceMergeAction.NAME + "*", + // indices stats is used by rollover, so we need to grant it here + IndicesStatsAction.NAME + "*", + TransportUpdateSettingsAction.TYPE.name(), + DownsampleAction.NAME, + TransportAddIndexBlockAction.TYPE.name() + ) + ) + .allowRestrictedIndices(true) .build() }, null, null, From 7cbc35f626f46889842c6019a72374dca4c8ae6f Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 27 Jun 2025 16:01:51 -0600 Subject: [PATCH 6/6] Add/fix tests for internal user for .kibana-reporting data stream --- .../security/user/InternalUsersTests.java | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java index 9b8caca209ad0..392d77ce3faf0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java @@ -63,6 +63,7 @@ import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.SECURITY_MAIN_ALIAS; import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.SECURITY_TOKENS_ALIAS; import static org.elasticsearch.xpack.core.security.user.UsernamesField.REINDEX_DATA_STREAM_NAME; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -244,10 +245,17 @@ public void testDataStreamLifecycleUser() { assertThat(role.application(), is(ApplicationPermission.NONE)); assertThat(role.remoteIndices(), is(RemoteIndicesPermission.NONE)); - final List allowedSystemDataStreams = Arrays.asList(".fleet-actions-results", ".fleet-fileds*"); + final List allowedFleetSystemDataStreams = Arrays.asList(".fleet-actions-results", ".fleet-fileds*"); + final List allowedKibanaSystemDataStreams = List.of(".kibana-reporting"); for (var group : role.indices().groups()) { if (group.allowRestrictedIndices()) { - assertThat(group.indices(), arrayContaining(allowedSystemDataStreams.toArray(new String[0]))); + assertThat( + group.indices(), + anyOf( + arrayContaining(allowedFleetSystemDataStreams.toArray(new String[0])), + arrayContaining(allowedKibanaSystemDataStreams.toArray(new String[0])) + ) + ); } } @@ -290,7 +298,25 @@ public void testDataStreamLifecycleUser() { true ); - allowedSystemDataStreams.forEach(allowedSystemDataStream -> { + allowedFleetSystemDataStreams.forEach(allowedSystemDataStream -> { + checkIndexAccess(role, randomFrom(sampleSystemDataStreamActions), allowedSystemDataStream, true); + checkIndexAccess( + role, + randomFrom(sampleSystemDataStreamActions), + DataStream.BACKING_INDEX_PREFIX + allowedSystemDataStream + randomAlphaOfLengthBetween(4, 8), + true + ); + + checkIndexAccess(role, randomFrom(sampleSystemDataStreamActions), allowedSystemDataStream + "::failures", true); + checkIndexAccess( + role, + randomFrom(sampleSystemDataStreamActions), + DataStream.FAILURE_STORE_PREFIX + allowedSystemDataStream + randomAlphaOfLengthBetween(4, 8), + true + ); + }); + + allowedKibanaSystemDataStreams.forEach(allowedSystemDataStream -> { checkIndexAccess(role, randomFrom(sampleSystemDataStreamActions), allowedSystemDataStream, true); checkIndexAccess( role,