Skip to content

Commit 3065898

Browse files
dakronedakrone
committed
WIP make kibana reporting a system data stream
1 parent aeb3718 commit 3065898

File tree

5 files changed

+213
-70
lines changed

5 files changed

+213
-70
lines changed

‎modules/kibana/src/main/java/module-info.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@
99

1010
module org.elasticsearch.kibana {
1111
requires org.elasticsearch.server;
12+
requires org.elasticsearch.xcontent;
1213
}

‎modules/kibana/src/main/java/org/elasticsearch/kibana/KibanaPlugin.java‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,24 @@
99

1010
package org.elasticsearch.kibana;
1111

12+
import org.elasticsearch.cluster.metadata.ComponentTemplate;
13+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
14+
import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
15+
import org.elasticsearch.cluster.metadata.Template;
16+
import org.elasticsearch.common.compress.CompressedXContent;
1217
import org.elasticsearch.common.settings.Settings;
18+
import org.elasticsearch.indices.SystemDataStreamDescriptor;
1319
import org.elasticsearch.indices.SystemIndexDescriptor;
1420
import org.elasticsearch.indices.SystemIndexDescriptor.Type;
1521
import org.elasticsearch.plugins.Plugin;
1622
import org.elasticsearch.plugins.SystemIndexPlugin;
1723

24+
import java.io.IOException;
1825
import java.util.Collection;
1926
import java.util.List;
27+
import java.util.Map;
28+
29+
import static org.elasticsearch.indices.ExecutorNames.DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS;
2030

2131
public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
2232

@@ -31,6 +41,203 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
3141
.setAllowsTemplates()
3242
.build();
3343

44+
public static final SystemDataStreamDescriptor KIBANA_REPORTING_DS_DESCRIPTOR;
45+
46+
static {
47+
try {
48+
KIBANA_REPORTING_DS_DESCRIPTOR = new SystemDataStreamDescriptor(
49+
".kibana-reporting",
50+
"system data stream for reporting",
51+
SystemDataStreamDescriptor.Type.EXTERNAL,
52+
ComposableIndexTemplate.builder()
53+
.indexPatterns(List.of(".kibana-reporting"))
54+
.priority(200L)
55+
.version(15L)
56+
.allowAutoCreate(true)
57+
.deprecated(false)
58+
.ignoreMissingComponentTemplates(List.of("kibana-reporting@custom"))
59+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(true, false))
60+
.metadata(Map.of("managed", "true", "description", "default kibana reporting template installed by elasticsearch"))
61+
.componentTemplates(List.of("kibana-reporting@settings", "kibana-reporting@custom"))
62+
.template(Template.builder().mappings(CompressedXContent.fromJSON("""
63+
{
64+
"properties" : {
65+
"kibana_name" : {
66+
"type" : "keyword"
67+
},
68+
"created_at" : {
69+
"type" : "date"
70+
},
71+
"priority" : {
72+
"type" : "byte"
73+
},
74+
"jobtype" : {
75+
"type" : "keyword"
76+
},
77+
"created_by" : {
78+
"type" : "keyword"
79+
},
80+
"migration_version" : {
81+
"type" : "keyword"
82+
},
83+
"timeout" : {
84+
"type" : "long"
85+
},
86+
"kibana_id" : {
87+
"type" : "keyword"
88+
},
89+
"output" : {
90+
"type" : "object",
91+
"properties" : {
92+
"content_type" : {
93+
"type" : "keyword"
94+
},
95+
"size" : {
96+
"type" : "long"
97+
},
98+
"csv_contains_formulas" : {
99+
"type" : "boolean"
100+
},
101+
"warnings" : {
102+
"type" : "text"
103+
},
104+
"chunk" : {
105+
"type" : "long"
106+
},
107+
"error_code" : {
108+
"type" : "keyword"
109+
},
110+
"max_size_reached" : {
111+
"type" : "boolean"
112+
},
113+
"content" : {
114+
"type" : "object",
115+
"enabled" : false
116+
}
117+
}
118+
},
119+
"process_expiration" : {
120+
"type" : "date"
121+
},
122+
"completed_at" : {
123+
"type" : "date"
124+
},
125+
"payload" : {
126+
"type" : "object",
127+
"enabled" : false
128+
},
129+
"meta" : {
130+
"properties" : {
131+
"layout" : {
132+
"type" : "text",
133+
"fields" : {
134+
"keyword" : {
135+
"ignore_above" : 256,
136+
"type" : "keyword"
137+
}
138+
}
139+
},
140+
"isDeprecated" : {
141+
"type" : "boolean"
142+
},
143+
"objectType" : {
144+
"type" : "text",
145+
"fields" : {
146+
"keyword" : {
147+
"ignore_above" : 256,
148+
"type" : "keyword"
149+
}
150+
}
151+
}
152+
}
153+
},
154+
"parent_id" : {
155+
"type" : "keyword"
156+
},
157+
"max_attempts" : {
158+
"type" : "short"
159+
},
160+
"started_at" : {
161+
"type" : "date"
162+
},
163+
"metrics" : {
164+
"type" : "object",
165+
"properties" : {
166+
"pdf" : {
167+
"type" : "object",
168+
"properties" : {
169+
"pages" : {
170+
"type" : "long"
171+
},
172+
"memory" : {
173+
"type" : "long"
174+
},
175+
"cpuInPercentage" : {
176+
"type" : "double"
177+
},
178+
"cpu" : {
179+
"type" : "double"
180+
},
181+
"memoryInMegabytes" : {
182+
"type" : "double"
183+
}
184+
}
185+
},
186+
"csv" : {
187+
"type" : "object",
188+
"properties" : {
189+
"rows" : {
190+
"type" : "long"
191+
}
192+
}
193+
},
194+
"png" : {
195+
"type" : "object",
196+
"properties" : {
197+
"memory" : {
198+
"type" : "long"
199+
},
200+
"cpuInPercentage" : {
201+
"type" : "double"
202+
},
203+
"cpu" : {
204+
"type" : "double"
205+
},
206+
"memoryInMegabytes" : {
207+
"type" : "double"
208+
}
209+
}
210+
}
211+
}
212+
},
213+
"attempts" : {
214+
"type" : "short"
215+
},
216+
"status" : {
217+
"type" : "keyword"
218+
}
219+
}
220+
}""")).lifecycle(DataStreamLifecycle.dataLifecycleBuilder().enabled(true)))
221+
.build(),
222+
Map.of(
223+
"kibana-reporting@settings",
224+
new ComponentTemplate(
225+
Template.builder()
226+
.settings(Settings.builder().put("index.number_of_shards", 1).put("index.auto_expand_replicas", "0-1"))
227+
.build(),
228+
null,
229+
null
230+
)
231+
),
232+
KIBANA_PRODUCT_ORIGIN,
233+
KIBANA_PRODUCT_ORIGIN.getFirst(),
234+
DEFAULT_SYSTEM_DATA_STREAM_THREAD_POOLS
235+
);
236+
} catch (IOException e) {
237+
throw new RuntimeException("unable to read kibana reporting template JSON", e);
238+
}
239+
}
240+
34241
public static final SystemIndexDescriptor REPORTING_INDEX_DESCRIPTOR = SystemIndexDescriptor.builder()
35242
.setIndexPattern(".reporting-*")
36243
.setDescription("system index for reporting")
@@ -52,6 +259,11 @@ public class KibanaPlugin extends Plugin implements SystemIndexPlugin {
52259
.setAllowedElasticProductOrigins(KIBANA_PRODUCT_ORIGIN)
53260
.build();
54261

262+
@Override
263+
public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
264+
return List.of(KIBANA_REPORTING_DS_DESCRIPTOR);
265+
}
266+
55267
@Override
56268
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
57269
return List.of(

‎x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -107,12 +107,6 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
107107
public static final String AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME = "agentless@settings";
108108
public static final String AGENTLESS_INDEX_TEMPLATE_NAME = "agentless";
109109

110-
///////////////////////////////////
111-
// Kibana reporting template
112-
///////////////////////////////////
113-
public static final String KIBANA_REPORTING_INDEX_TEMPLATE_NAME = ".kibana-reporting";
114-
public static final String KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME = "kibana-reporting@settings";
115-
116110
public StackTemplateRegistry(
117111
Settings nodeSettings,
118112
ClusterService clusterService,
@@ -219,13 +213,6 @@ private Map<String, ComponentTemplate> loadComponentTemplateConfigs() {
219213
REGISTRY_VERSION,
220214
TEMPLATE_VERSION_VARIABLE,
221215
ADDITIONAL_TEMPLATE_VARIABLES
222-
),
223-
new IndexTemplateConfig(
224-
KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME,
225-
"/[email protected]",
226-
REGISTRY_VERSION,
227-
TEMPLATE_VERSION_VARIABLE,
228-
ADDITIONAL_TEMPLATE_VARIABLES
229216
)
230217
)) {
231218
try {
@@ -314,13 +301,6 @@ protected Map<String, ComponentTemplate> getComponentTemplateConfigs() {
314301
REGISTRY_VERSION,
315302
TEMPLATE_VERSION_VARIABLE,
316303
ADDITIONAL_TEMPLATE_VARIABLES
317-
),
318-
new IndexTemplateConfig(
319-
KIBANA_REPORTING_INDEX_TEMPLATE_NAME,
320-
"/[email protected]",
321-
REGISTRY_VERSION,
322-
TEMPLATE_VERSION_VARIABLE,
323-
ADDITIONAL_TEMPLATE_VARIABLES
324304
)
325305
);
326306

‎x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,6 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
417417
versions.put(StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
418418
versions.put(StackTemplateRegistry.AGENTLESS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
419419
versions.put(StackTemplateRegistry.AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
420-
versions.put(StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
421420
versions.put(StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
422421
versions.put(StackTemplateRegistry.TRACES_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
423422
ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes);
@@ -483,10 +482,6 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
483482
StackTemplateRegistry.AGENTLESS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
484483
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
485484
);
486-
versions.put(
487-
StackTemplateRegistry.KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME,
488-
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
489-
);
490485
versions.put(
491486
StackTemplateRegistry.TRACES_MAPPINGS_COMPONENT_TEMPLATE_NAME,
492487
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)

‎x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml‎

Lines changed: 0 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,6 @@ setup:
7474
indices.get_index_template:
7575
name: synthetics
7676

77-
- do:
78-
indices.get_index_template:
79-
name: .kibana-reporting
80-
8177
---
8278
"Test logs index auto creation":
8379
- do:
@@ -221,47 +217,6 @@ setup:
221217
indices.delete_data_stream:
222218
name: synthetics-foo-bar
223219

224-
---
225-
"Test kibana reporting index auto creation":
226-
- requires:
227-
test_runner_features: ["headers"]
228-
229-
- do:
230-
headers: { X-elastic-product-origin: kibana }
231-
index:
232-
index: .kibana-reporting-foo
233-
body:
234-
"@timestamp": "2020-01-01"
235-
jobtype: "thing"
236-
237-
- do:
238-
indices.get_data_stream:
239-
name: .kibana-reporting-foo
240-
241-
- match: { data_streams.0.name: .kibana-reporting-foo }
242-
- match: { data_streams.0.hidden: true }
243-
- match: { data_streams.0.timestamp_field.name: '@timestamp' }
244-
- match: { data_streams.0.generation: 1 }
245-
- length: { data_streams.0.indices: 1 }
246-
- match: { data_streams.0.lifecycle.enabled: true }
247-
- match: { data_streams.0.indices.0.index_name: '/\.ds-.kibana-reporting-foo-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
248-
249-
- set: { data_streams.0.indices.0.index_name: idx0name }
250-
251-
- do:
252-
indices.get:
253-
index: $idx0name
254-
255-
- is_true: .$idx0name.settings
256-
- is_true: .$idx0name.mappings
257-
- match: { .$idx0name.mappings.properties.meta.properties.objectType.type: "text" }
258-
- match: { .$idx0name.mappings.properties.meta.properties.layout.type: "text" }
259-
- match: { .$idx0name.data_stream: ".kibana-reporting-foo" }
260-
261-
- do:
262-
indices.delete_data_stream:
263-
name: .kibana-reporting-foo
264-
265220
---
266221
"Test wrong data_stream type":
267222

0 commit comments

Comments
 (0)