Skip to content

Commit 591e1f8

Browse files
pracuccipracucci
authored
Merge pull request grafana#412 from aallawala/aja_bugfix_alerts
Span the annotation.message in alerts as YAML multiline strings.
2 parents 92f3b64 + d6bdd88 commit 591e1f8

File tree

4 files changed

+79
-26
lines changed

4 files changed

+79
-26
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
* [BUGFIX] Upstream recording rule `node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate` renamed. #379
8282
* [BUGFIX] Treat `compactor_blocks_retention_period` type as string rather than int.#395
8383
* [BUGFIX] Fixed writes/reads/alertmanager resources dashboards to use `$._config.job_names.gateway`. #403
84+
* [BUGFIX] Span the annotation.message in alerts as YAML multiline strings. #412
8485

8586
## 1.9.0 / 2021-05-18
8687

‎cortex-mixin/alerts/alerts.libsonnet

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,9 @@
1818
severity: 'critical',
1919
},
2020
annotations: {
21-
message: 'Cortex cluster %(alert_aggregation_variables)s has {{ printf "%%f" $value }} unhealthy ingester(s).' % $._config,
21+
message: |||
22+
Cortex cluster %(alert_aggregation_variables)s has {{ printf "%%f" $value }} unhealthy ingester(s).
23+
||| % $._config,
2224
},
2325
},
2426
{
@@ -199,7 +201,9 @@
199201
severity: 'warning',
200202
},
201203
annotations: {
202-
message: '{{ $labels.job }}/{{ $labels.instance }} has restarted {{ printf "%.2f" $value }} times in the last 30 mins.',
204+
message: |||
205+
{{ $labels.job }}/{{ $labels.instance }} has restarted {{ printf "%.2f" $value }} times in the last 30 mins.
206+
|||,
203207
},
204208
},
205209
{
@@ -212,7 +216,9 @@
212216
severity: 'critical',
213217
},
214218
annotations: {
215-
message: '{{ $labels.job }}/{{ $labels.instance }} transfer failed.',
219+
message: |||
220+
{{ $labels.job }}/{{ $labels.instance }} transfer failed.
221+
|||,
216222
},
217223
},
218224
{
@@ -266,7 +272,9 @@
266272
severity: 'critical',
267273
},
268274
annotations: {
269-
message: '{{ $labels.job }}/{{ $labels.instance }} has a number of mmap-ed areas close to the limit.',
275+
message: |||
276+
{{ $labels.job }}/{{ $labels.instance }} has a number of mmap-ed areas close to the limit.
277+
|||,
270278
},
271279
},
272280
],
@@ -707,7 +715,9 @@
707715
severity: 'warning',
708716
},
709717
annotations: {
710-
message: 'Cortex instance {{ $labels.instance }} in %(alert_aggregation_variables)s sees incorrect number of gossip members.' % $._config,
718+
message: |||
719+
Cortex instance {{ $labels.instance }} in %(alert_aggregation_variables)s sees incorrect number of gossip members.
720+
||| % $._config,
711721
},
712722
},
713723
],

‎cortex-mixin/alerts/blocks.libsonnet

Lines changed: 45 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@
2626
severity: 'critical',
2727
},
2828
annotations: {
29-
message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.' % $._config,
29+
message: |||
30+
Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.
31+
||| % $._config,
3032
},
3133
},
3234
{
@@ -43,7 +45,9 @@
4345
severity: 'critical',
4446
},
4547
annotations: {
46-
message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.' % $._config,
48+
message: |||
49+
Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has not shipped any block in the last 4 hours.
50+
||| % $._config,
4751
},
4852
},
4953
{
@@ -61,7 +65,9 @@
6165
severity: 'critical',
6266
},
6367
annotations: {
64-
message: "Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has compacted a block {{ $value | humanizeDuration }} ago but it hasn't been successfully uploaded to the storage yet." % $._config,
68+
message: |||
69+
Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s has compacted a block {{ $value | humanizeDuration }} ago but it hasn't been successfully uploaded to the storage yet.
70+
||| % $._config,
6571
},
6672
},
6773
{
@@ -77,7 +83,9 @@
7783
severity: 'critical',
7884
},
7985
annotations: {
80-
message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to compact TSDB head.' % $._config,
86+
message: |||
87+
Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to compact TSDB head.
88+
||| % $._config,
8189
},
8290
},
8391
{
@@ -89,7 +97,9 @@
8997
severity: 'critical',
9098
},
9199
annotations: {
92-
message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB head.' % $._config,
100+
message: |||
101+
Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB head.
102+
||| % $._config,
93103
},
94104
},
95105
{
@@ -101,7 +111,9 @@
101111
severity: 'critical',
102112
},
103113
annotations: {
104-
message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to create TSDB checkpoint.' % $._config,
114+
message: |||
115+
Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to create TSDB checkpoint.
116+
||| % $._config,
105117
},
106118
},
107119
{
@@ -113,7 +125,9 @@
113125
severity: 'critical',
114126
},
115127
annotations: {
116-
message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to delete TSDB checkpoint.' % $._config,
128+
message: |||
129+
Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to delete TSDB checkpoint.
130+
||| % $._config,
117131
},
118132
},
119133
{
@@ -125,7 +139,9 @@
125139
severity: 'warning',
126140
},
127141
annotations: {
128-
message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB WAL.' % $._config,
142+
message: |||
143+
Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to truncate TSDB WAL.
144+
||| % $._config,
129145
},
130146
},
131147
{
@@ -137,7 +153,9 @@
137153
severity: 'critical',
138154
},
139155
annotations: {
140-
message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s got a corrupted TSDB WAL.' % $._config,
156+
message: |||
157+
Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s got a corrupted TSDB WAL.
158+
||| % $._config,
141159
},
142160
},
143161
{
@@ -150,7 +168,9 @@
150168
severity: 'critical',
151169
},
152170
annotations: {
153-
message: 'Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to write to TSDB WAL.' % $._config,
171+
message: |||
172+
Cortex Ingester {{ $labels.instance }} in %(alert_aggregation_variables)s is failing to write to TSDB WAL.
173+
||| % $._config,
154174
},
155175
},
156176
{
@@ -166,7 +186,9 @@
166186
severity: 'critical',
167187
},
168188
annotations: {
169-
message: 'Cortex Querier {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully scanned the bucket since {{ $value | humanizeDuration }}.' % $._config,
189+
message: |||
190+
Cortex Querier {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully scanned the bucket since {{ $value | humanizeDuration }}.
191+
||| % $._config,
170192
},
171193
},
172194
{
@@ -190,7 +212,9 @@
190212
severity: 'warning',
191213
},
192214
annotations: {
193-
message: 'Cortex Queries in %(alert_aggregation_variables)s are refetching series from different store-gateways (because of missing blocks) for the {{ printf "%%.0f" $value }}%% of queries.' % $._config,
215+
message: |||
216+
Cortex Queries in %(alert_aggregation_variables)s are refetching series from different store-gateways (because of missing blocks) for the {{ printf "%%.0f" $value }}%% of queries.
217+
||| % $._config,
194218
},
195219
},
196220
{
@@ -206,7 +230,9 @@
206230
severity: 'critical',
207231
},
208232
annotations: {
209-
message: 'Cortex Store Gateway {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully synched the bucket since {{ $value | humanizeDuration }}.' % $._config,
233+
message: |||
234+
Cortex Store Gateway {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully synched the bucket since {{ $value | humanizeDuration }}.
235+
||| % $._config,
210236
},
211237
},
212238
{
@@ -219,7 +245,9 @@
219245
severity: 'critical',
220246
},
221247
annotations: {
222-
message: 'Cortex bucket index for tenant {{ $labels.user }} in %(alert_aggregation_variables)s has not been updated since {{ $value | humanizeDuration }}.' % $._config,
248+
message: |||
249+
Cortex bucket index for tenant {{ $labels.user }} in %(alert_aggregation_variables)s has not been updated since {{ $value | humanizeDuration }}.
250+
||| % $._config,
223251
},
224252
},
225253
{
@@ -233,7 +261,9 @@
233261
severity: 'warning',
234262
},
235263
annotations: {
236-
message: 'Cortex tenant {{ $labels.user }} in %(alert_aggregation_variables)s has {{ $value }} partial blocks.' % $._config,
264+
message: |||
265+
Cortex tenant {{ $labels.user }} in %(alert_aggregation_variables)s has {{ $value }} partial blocks.
266+
||| % $._config,
237267
},
238268
},
239269
],

‎cortex-mixin/alerts/compactor.libsonnet

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@
1414
severity: 'critical',
1515
},
1616
annotations: {
17-
message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully cleaned up blocks in the last 6 hours.' % $._config,
17+
message: |||
18+
Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not successfully cleaned up blocks in the last 6 hours.
19+
||| % $._config,
1820
},
1921
},
2022
{
@@ -30,7 +32,9 @@
3032
severity: 'critical',
3133
},
3234
annotations: {
33-
message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.' % $._config,
35+
message: |||
36+
Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.
37+
||| % $._config,
3438
},
3539
},
3640
{
@@ -44,7 +48,9 @@
4448
severity: 'critical',
4549
},
4650
annotations: {
47-
message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.' % $._config,
51+
message: |||
52+
Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not run compaction in the last 24 hours.
53+
||| % $._config,
4854
},
4955
},
5056
{
@@ -57,7 +63,9 @@
5763
severity: 'critical',
5864
},
5965
annotations: {
60-
message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s failed to run 2 consecutive compactions.' % $._config,
66+
message: |||
67+
Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s failed to run 2 consecutive compactions.
68+
||| % $._config,
6169
},
6270
},
6371
{
@@ -73,7 +81,9 @@
7381
severity: 'critical',
7482
},
7583
annotations: {
76-
message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.' % $._config,
84+
message: |||
85+
Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.
86+
||| % $._config,
7787
},
7888
},
7989
{
@@ -87,7 +97,9 @@
8797
severity: 'critical',
8898
},
8999
annotations: {
90-
message: 'Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.' % $._config,
100+
message: |||
101+
Cortex Compactor {{ $labels.instance }} in %(alert_aggregation_variables)s has not uploaded any block in the last 24 hours.
102+
||| % $._config,
91103
},
92104
},
93105
],

0 commit comments

Comments
 (0)