Skip to content

Commit 29b390c

Browse files
tomwilkietomwilkie
authored
Enable (optional) sharding of dashboard folder config maps too. (#237)
* Enable (optional) sharding of dashboard folder config maps too. Signed-off-by: Tom Wilkie <[email protected]> * Only add dashboard folders for mixins with actual dashboards. Signed-off-by: Tom Wilkie <[email protected]> * Dashboard names cannot be the same as folder names. See grafana/grafana#15642 Signed-off-by: Tom Wilkie <[email protected]>
1 parent d461051 commit 29b390c

File tree

5 files changed

+93
-61
lines changed

5 files changed

+93
-61
lines changed

‎consul-mixin/dashboards.libsonnet

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ local panel_settings = {
1313

1414
{
1515
grafanaDashboards+:: {
16-
'consul.json':
17-
g.dashboard('Consul')
16+
'consul-overview.json':
17+
g.dashboard('Consul Overview')
1818
.addTemplate('job', 'consul_up', 'job')
1919
.addMultiTemplate('instance', 'consul_up{job="$job"}', 'instance')
2020
.addRow(

‎memcached-mixin/dashboards.libsonnet

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ local g = (import 'grafana-builder/grafana.libsonnet');
22

33
{
44
grafanaDashboards+: {
5-
'memcached.json':
6-
g.dashboard('Memcached')
5+
'memcached-overview.json':
6+
g.dashboard('Memcached Overview')
77
.addMultiTemplate('cluster', 'memcached_commands_total', 'cluster')
88
.addMultiTemplate('job', 'memcached_commands_total{cluster=~"$cluster"}', 'job')
99
.addMultiTemplate('instance', 'memcached_commands_total{cluster=~"$cluster",job=~"$job"}', 'instance')

‎prometheus-ksonnet/grafana/dashboards.libsonnet

Lines changed: 62 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -52,55 +52,73 @@
5252
grafanaDashboards+:: std.foldr(
5353
function(mixinName, acc)
5454
local mixin = $.mixins[mixinName] + mixinProto;
55-
if !std.objectHas(mixin, 'grafanaDashboardFolder')
55+
if !$.isFolderedMixin(mixin)
5656
then acc + mixin.grafanaDashboards
5757
else acc,
5858
std.objectFields($.mixins),
5959
{}
6060
),
6161

62-
dashboardsByFolder+:: std.foldr(
63-
function(mixinName, acc)
64-
local mixin = $.mixins[mixinName] + mixinProto;
65-
if std.objectHas(mixin, 'grafanaDashboardFolder')
66-
then acc {
67-
[mixin.grafanaDashboardFolder]: mixin.grafanaDashboards,
68-
}
69-
else acc,
70-
std.objectFields($.mixins),
71-
{}
72-
),
62+
// Config map names can't contain special chars, this is a hack but will
63+
// do for now.
64+
folderID(folder)::
65+
local lower = std.asciiLower(folder);
66+
local underscore = std.strReplace(lower, '_', '-');
67+
local space = std.strReplace(underscore, ' ', '-');
68+
space,
7369

74-
local materialise_config_map(config_map_name, dashboards) =
75-
configMap.new(config_map_name) +
76-
configMap.withDataMixin({
77-
[name]: std.toString(dashboards[name])
78-
for name in std.objectFields(dashboards)
79-
}) +
80-
configMap.mixin.metadata.withLabels($._config.grafana_dashboard_labels),
70+
// Helper to decide is a mixin should go in a folder or not.
71+
isFolderedMixin(m)::
72+
local mixin = m + mixinProto;
73+
std.objectHas(mixin, 'grafanaDashboardFolder') &&
74+
std.length(mixin.grafanaDashboards) > 0,
8175

82-
// When sharding is enabled, this is a map of config maps, each map named
83-
// "dashboard-0" ... "dashboard-N" and containing dashboards whose name
84-
// hashes to that shard.
85-
dashboards_config_maps: {
86-
['dashboard-%d' % shard]:
87-
materialise_config_map('dashboards-%d' % shard, {
88-
[name]: $.grafanaDashboards[name]
89-
for name in std.objectFields($.grafanaDashboards)
90-
if std.codepoint(std.md5(name)[1]) % $._config.dashboard_config_maps == shard
91-
})
92-
for shard in std.range(0, $._config.dashboard_config_maps - 1)
76+
// Its super common for a single mixin's worth of dashboards to not even fit
77+
// in a single config map. So we split each mixin's dashboards up over
78+
// multiple config maps, depending on the hash of dashboards name.
79+
local sharded_config_maps(name_prefix, shards, dashboards) = {
80+
['%s-%d' % [name_prefix, shard]]:
81+
configMap.new('%s-%d' % [name_prefix, shard]) +
82+
configMap.withDataMixin({
83+
[name]: std.toString(dashboards[name])
84+
for name in std.objectFields(dashboards)
85+
if std.codepoint(std.md5(name)[1]) % shards == shard
86+
}) +
87+
configMap.mixin.metadata.withLabels($._config.grafana_dashboard_labels)
88+
for shard in std.range(0, shards - 1)
9389
},
9490

95-
// A map of config maps, one per folder, for dashboards in folders.
96-
dashboard_folders_config_maps: {
97-
['dashboard-%s' % std.asciiLower(folder)]:
98-
materialise_config_map(
99-
'dashboards-%s' % std.asciiLower(folder),
100-
$.dashboardsByFolder[folder]
101-
)
102-
for folder in std.objectFields($.dashboardsByFolder)
103-
},
91+
// Map containing all the sharded config maps for dashboards in no folder.
92+
// ie dashboards_config_maps[dashboard name] -> dashboard
93+
dashboards_config_maps:
94+
sharded_config_maps(
95+
'dashboards',
96+
$._config.dashboard_config_maps,
97+
$.grafanaDashboards,
98+
),
99+
100+
// Map containing maps of all the sharded config maps for each folder.
101+
// ie dashboard_folders_config_maps[dashboard folder][dashboard name] -> dashboard
102+
dashboard_folders_config_maps: std.foldr(
103+
function(mixinName, acc)
104+
local mixin = $.mixins[mixinName] + mixinProto;
105+
if !$.isFolderedMixin(mixin)
106+
then acc
107+
else
108+
local config_map_name = 'dashboards-%s' % $.folderID(mixin.grafanaDashboardFolder);
109+
acc {
110+
[config_map_name]:
111+
sharded_config_maps(
112+
config_map_name,
113+
if std.objectHas(mixin, 'grafanaDashboardShards')
114+
then mixin.grafanaDashboardShards
115+
else 1,
116+
mixin.grafanaDashboards
117+
),
118+
},
119+
std.objectFields($.mixins),
120+
{},
121+
),
104122

105123
// Config map containing the dashboard provisioning YAML, telling
106124
// Grafana where to find the dashboard JSONs.
@@ -123,17 +141,18 @@
123141
},
124142
] + [
125143
{
126-
name: 'dashboards-%s' % std.asciiLower(folder),
144+
name: 'dashboards-%s' % $.folderID($.mixins[mixinName].grafanaDashboardFolder),
127145
orgId: 1,
128-
folder: folder,
146+
folder: $.mixins[mixinName].grafanaDashboardFolder,
129147
type: 'file',
130148
disableDeletion: true,
131149
editable: false,
132150
options: {
133-
path: '/grafana/dashboard-folders/%s' % std.asciiLower(folder),
151+
path: '/grafana/dashboards-%s' % $.folderID($.mixins[mixinName].grafanaDashboardFolder),
134152
},
135153
}
136-
for folder in std.objectFields($.dashboardsByFolder)
154+
for mixinName in std.objectFields($.mixins)
155+
if $.isFolderedMixin($.mixins[mixinName])
137156
],
138157
}),
139158
}),

‎prometheus-ksonnet/grafana/deployment.libsonnet

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,18 @@
3535

3636
local deployment = $.apps.v1.deployment,
3737

38+
// Helper to mount a variable number of shareded config maps.
39+
local sharded_config_map_mounts(prefix, shards) =
40+
std.foldr(
41+
function(shard, acc)
42+
$.util.configVolumeMount(
43+
'%s-%d' % [prefix, shard],
44+
'/grafana/%s/%d' % [prefix, shard]
45+
) + acc,
46+
std.range(0, shards - 1),
47+
{}
48+
),
49+
3850
grafana_deployment:
3951
deployment.new('grafana', 1, [$.grafana_container]) +
4052
deployment.mixin.spec.template.spec.securityContext.withRunAsUser(0) +
@@ -44,23 +56,23 @@
4456
$.util.configVolumeMount('grafana-notification-channels', '%(grafana_provisioning_dir)s/notifiers' % $._config) +
4557
(
4658
// Mount _all_ the dashboard config map shards.
47-
std.foldr(
48-
function(m, acc) m + acc,
49-
[
50-
$.util.configVolumeMount('dashboards-%d' % shard, '/grafana/dashboards/%d' % shard)
51-
for shard in std.range(0, $._config.dashboard_config_maps - 1)
52-
],
53-
{}
54-
)
59+
sharded_config_map_mounts('dashboards', $._config.dashboard_config_maps)
5560
) + (
5661
// Add config map mounts for each folder for dashboards.
5762
std.foldr(
58-
function(m, acc) m + acc,
59-
[
60-
$.util.configVolumeMount('dashboards-%s' % std.asciiLower(folder), '/grafana/dashboard-folders/%s' % std.asciiLower(folder))
61-
for folder in std.objectFields($.dashboardsByFolder)
62-
],
63-
{}
63+
function(mixinName, acc)
64+
local mixin = $.mixins[mixinName];
65+
if !$.isFolderedMixin(mixin)
66+
then acc
67+
else
68+
local config_map_name = 'dashboards-%s' % $.folderID(mixin.grafanaDashboardFolder);
69+
local shards =
70+
if std.objectHas(mixin, 'grafanaDashboardShards')
71+
then mixin.grafanaDashboardShards
72+
else 1;
73+
sharded_config_map_mounts(config_map_name, shards) + acc,
74+
std.objectFields($.mixins),
75+
{},
6476
)
6577
) +
6678
$.util.podPriority('critical'),

‎prometheus-ksonnet/mixins.libsonnet

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
kubernetes:
55
(import 'kubernetes-mixin/mixin.libsonnet') {
66
grafanaDashboardFolder: 'Kubernetes',
7+
grafanaDashboardShards: 8,
78

89
_config+:: {
910
cadvisorSelector: 'job="kube-system/cadvisor"',

0 commit comments

Comments
 (0)