Skip to content

Commit 1cd23af

Browse files
schmikeischmikei
authored
Update ClickHouse mixin to better utilize libraries (#1445)
* update clickhouse to latest mixin standards * remove redundant wrapPanels * cleanup * fix variable setting * remove unused variables * wip: minstep adjust clickhouse to use signals and other feedback suggestions * move over to signals for defining the targets * make fmt * apply pr feedback, including removing enableMultiCluster flag * make fmt and re-add description for gauge? * newline at Eof of jsonnetfile.json * regenerate dashboards_out * use k8s_patch * fix lint * group things into rows * make fmt
1 parent 68bb22c commit 1cd23af

26 files changed

+1854
-3582
lines changed

‎clickhouse-mixin/alerts/alerts.libsonnet renamed to ‎clickhouse-mixin/alerts.libsonnet

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,22 @@
11
{
2-
prometheusAlerts+:: {
3-
groups+: [
2+
new(this): {
3+
groups: [
44
{
55
name: 'ClickHouseAlerts',
66
rules: [
77
{
88
alert: 'ClickHouseReplicationQueueBackingUp',
99
expr: |||
1010
ClickHouseAsyncMetrics_ReplicasMaxQueueSize > %(alertsReplicasMaxQueueSize)s
11-
||| % $._config,
11+
||| % this.config,
1212
labels: {
1313
severity: 'warning',
1414
},
1515
annotations: {
1616
summary: 'ClickHouse replica max queue size backing up.',
1717
description: |||
1818
ClickHouse replication tasks are processing slower than expected on {{ $labels.instance }} causing replication queue size to back up at {{ $value }} exceeding the threshold value of %(alertsReplicasMaxQueueSize)s.
19-
||| % $._config,
19+
||| % this.config,
2020
},
2121
'for': '5m',
2222
keep_firing_for: '5m',

‎clickhouse-mixin/config.libsonnet

Lines changed: 28 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,35 @@
11
{
2-
_config+:: {
3-
enableMultiCluster: false,
4-
clickhouseSelector: if self.enableMultiCluster then 'job=~"$job", instance=~"$instance", cluster=~"$cluster"' else 'job=~"$job", instance=~"$instance"',
2+
local this = self,
3+
filteringSelector: 'job="integrations/clickhouse"',
4+
groupLabels: ['job', 'cluster'],
5+
logLabels: ['job', 'cluster', 'instance'],
6+
instanceLabels: ['instance'],
57

6-
dashboardTags: ['clickhouse-mixin'],
7-
dashboardPeriod: 'now-30m',
8-
dashboardTimezone: 'default',
9-
dashboardRefresh: '1m',
10-
logLabels: if self.enableMultiCluster then ['job', 'instance', 'cluster', 'level']
11-
else ['job', 'instance', 'level'],
8+
dashboardTags: [self.uid],
9+
uid: 'clickhouse',
10+
dashboardNamePrefix: 'Clickhouse',
11+
dashboardPeriod: 'now-30m',
12+
dashboardTimezone: 'default',
13+
dashboardRefresh: '1m',
14+
metricsSource: 'prometheus', // metrics source for signals
1215

13-
// for alerts
14-
alertsReplicasMaxQueueSize: '99',
16+
// Logging configuration
17+
enableLokiLogs: true,
18+
extraLogLabels: ['level'], // Required by logs-lib
19+
logsVolumeGroupBy: 'level',
20+
showLogsVolume: true,
1521

16-
filterSelector: 'job=~".*/clickhouse.*"',
22+
// Alerts configuration
23+
alertsReplicasMaxQueueSize: '99',
1724

18-
enableLokiLogs: true,
25+
// Signals configuration
26+
signals+: {
27+
replica: (import './signals/replica.libsonnet')(this),
28+
zookeeper: (import './signals/zookeeper.libsonnet')(this),
29+
queries: (import './signals/queries.libsonnet')(this),
30+
memory: (import './signals/memory.libsonnet')(this),
31+
connections: (import './signals/connections.libsonnet')(this),
32+
network: (import './signals/network.libsonnet')(this),
33+
disk: (import './signals/disk.libsonnet')(this),
1934
},
2035
}

‎clickhouse-mixin/dashboards.libsonnet

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
local g = import './g.libsonnet';
2+
local logslib = import 'logs-lib/logs/main.libsonnet';
3+
{
4+
local root = self,
5+
new(this)::
6+
local prefix = this.config.dashboardNamePrefix;
7+
local links = this.grafana.links;
8+
local tags = this.config.dashboardTags;
9+
local uid = g.util.string.slugify(this.config.uid);
10+
local vars = this.grafana.variables;
11+
local annotations = this.grafana.annotations;
12+
local refresh = this.config.dashboardRefresh;
13+
local period = this.config.dashboardPeriod;
14+
local timezone = this.config.dashboardTimezone;
15+
local rows = this.grafana.rows;
16+
17+
{
18+
'clickhouse-replica.json':
19+
g.dashboard.new(prefix + ' replica')
20+
+ g.dashboard.withPanels(
21+
g.util.panel.resolveCollapsedFlagOnRows(
22+
g.util.grid.wrapPanels(
23+
[rows.replicaOperationsPanels, rows.replicaZookeeperPanels]
24+
)
25+
)
26+
)
27+
+ root.applyCommon(
28+
vars.singleInstance,
29+
uid + '_replica',
30+
tags,
31+
links { clickhouseReplica+:: {} },
32+
annotations,
33+
timezone,
34+
refresh,
35+
period
36+
),
37+
38+
'clickhouse-overview.json':
39+
g.dashboard.new(prefix + ' overview')
40+
+ g.dashboard.withPanels(
41+
g.util.panel.resolveCollapsedFlagOnRows(
42+
g.util.grid.wrapPanels(
43+
[rows.overviewQueriesPanels, rows.overviewMemoryPanels, rows.overviewNetworkPanels]
44+
)
45+
)
46+
)
47+
+ root.applyCommon(
48+
vars.singleInstance,
49+
uid + '_overview',
50+
tags,
51+
links { clickhouseOverview+:: {} },
52+
annotations,
53+
timezone,
54+
refresh,
55+
period
56+
),
57+
58+
'clickhouse-latency.json':
59+
g.dashboard.new(prefix + ' latency')
60+
+ g.dashboard.withPanels(
61+
g.util.panel.resolveCollapsedFlagOnRows(
62+
g.util.grid.wrapPanels(
63+
[rows.latencyDiskPanels, rows.latencyNetworkPanels, rows.latencyZooKeeperPanels]
64+
)
65+
)
66+
)
67+
+ root.applyCommon(
68+
vars.singleInstance,
69+
uid + '_latency',
70+
tags,
71+
links { clickhouseLatency+:: {} },
72+
annotations,
73+
timezone,
74+
refresh,
75+
period
76+
),
77+
78+
}
79+
+
80+
if this.config.enableLokiLogs then
81+
{
82+
'logs.json':
83+
logslib.new(
84+
prefix + ' logs',
85+
datasourceName=this.grafana.variables.datasources.loki.name,
86+
datasourceRegex=this.grafana.variables.datasources.loki.regex,
87+
filterSelector=this.config.filteringSelector,
88+
labels=this.config.logLabels + this.config.extraLogLabels,
89+
formatParser=null,
90+
showLogsVolume=this.config.showLogsVolume,
91+
)
92+
{
93+
dashboards+:
94+
{
95+
logs+:
96+
// reference to self, already generated variables, to keep them, but apply other common data in applyCommon
97+
root.applyCommon(super.logs.templating.list, uid=uid + '-logs', tags=tags, links=links { logs+:: {} }, annotations=annotations, timezone=timezone, refresh=refresh, period=period),
98+
},
99+
panels+:
100+
{
101+
logs+:
102+
g.panel.logs.options.withEnableLogDetails(true)
103+
+ g.panel.logs.options.withShowTime(false)
104+
+ g.panel.logs.options.withWrapLogMessage(false),
105+
},
106+
variables+: {
107+
// add prometheus datasource for annotations processing
108+
toArray+: [
109+
this.grafana.variables.datasources.prometheus { hide: 2 },
110+
],
111+
},
112+
}.dashboards.logs,
113+
}
114+
else {},
115+
116+
applyCommon(vars, uid, tags, links, annotations, timezone, refresh, period):
117+
g.dashboard.withTags(tags)
118+
+ g.dashboard.withUid(uid)
119+
+ g.dashboard.withLinks(std.objectValues(links))
120+
+ g.dashboard.withTimezone(timezone)
121+
+ g.dashboard.withRefresh(refresh)
122+
+ g.dashboard.time.withFrom(period)
123+
+ g.dashboard.withVariables(vars)
124+
+ g.dashboard.withAnnotations(std.objectValues(annotations)),
125+
}

‎clickhouse-mixin/dashboards/addlogs.libsonnet

Lines changed: 0 additions & 36 deletions
This file was deleted.

0 commit comments

Comments
 (0)