Skip to content

Commit 77e8b3c

Browse files
ArthurSensArthurSens
authored
Refactor Alertmanager configuration generation (#274)
* Refactor alerting configuration surface Signed-off-by: ArthurSens <[email protected]> * Remove depracted alerting config field Signed-off-by: ArthurSens <[email protected]> * Implement refactored Alertmanager config generation Signed-off-by: ArthurSens <[email protected]> Signed-off-by: ArthurSens <[email protected]>
1 parent 5621792 commit 77e8b3c

File tree

5 files changed

+260
-58
lines changed

5 files changed

+260
-58
lines changed

‎installer/examples/default-config.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
alerting:
2-
config:
3-
templates: null
2+
genericSlackChannel: ""
3+
slackOAuthToken: ""
44
certmanager:
55
installServiceMonitors: false
66
gitpod:

‎installer/examples/full-config.yaml

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,12 @@
11
alerting:
2-
config:
3-
global:
4-
resolve_timeout: 5m
5-
pagerduty_url: https://events.pagerduty.com/v2/enqueue
6-
route:
7-
receiver: Default
8-
group_by:
9-
- namespace
10-
continue: false
11-
routes:
12-
- receiver: Watchdog
13-
matchers:
14-
- alertname="Watchdog"
15-
continue: false
16-
group_wait: 30s
17-
group_interval: 5m
18-
repeat_interval: 12h
19-
receivers:
20-
- name: Default
21-
- name: Watchdog
22-
templates: []
2+
genericSlackChannel: "#generic_alerts"
3+
slackOAuthToken: "fake-token"
4+
pagerdutyRoutingKey: "fake-pd-key"
5+
teamRoutes:
6+
- teamLabel: platform
7+
slackChannel: "#team_platform_alerts"
8+
- teamLabel: newTeam
9+
slackChannel: "#newTeam_channel"
2310
certmanager:
2411
installServiceMonitors: false
2512
gitpod:

‎installer/pkg/components/alertmanager/configsecret.go

Lines changed: 210 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,43 @@
11
package alertmanager
22

33
import (
4+
"fmt"
5+
6+
"github.com/prometheus/common/model"
47
corev1 "k8s.io/api/core/v1"
58
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
69
"k8s.io/apimachinery/pkg/runtime"
710

811
"github.com/gitpod-io/observability/installer/pkg/common"
12+
"github.com/gitpod-io/observability/installer/pkg/config"
913
)
1014

15+
const queryString = `{{ reReplaceAll "%22" "%5C%22" (index .Alerts 0).GeneratorURL | reReplaceAll ".*expr=" "https://grafana.gitpod.io/explore?orgId=1&left=%7B%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D,%22datasource%22:%22VictoriaMetrics%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22" | reReplaceAll "&g0.tab=1" "%22%7D%5D%7D" | reReplaceAll ` + `\+` + ` "%20" | reReplaceAll "%0A" "" | reReplaceAll "%28" "(" | reReplaceAll "%29" ")" }}`
16+
1117
func configSecret(ctx *common.RenderContext) ([]runtime.Object, error) {
18+
var receivers []*config.Receiver
19+
20+
receivers = append(receivers, criticalReceiver(ctx))
21+
receivers = append(receivers, defaultReceivers(ctx)...)
22+
receivers = append(receivers, teamSlackReceivers(ctx)...)
23+
resolveTimeout, _ := model.ParseDuration("5m")
24+
25+
alertingConfig := config.AlertmanagerConfig{
26+
Global: &config.GlobalConfig{
27+
ResolveTimeout: &resolveTimeout,
28+
},
29+
Route: &config.Route{
30+
Receiver: "Black_Hole",
31+
GroupByStr: []string{"..."},
32+
GroupWait: "30s",
33+
GroupInterval: "5m",
34+
RepeatInterval: "6h",
35+
Routes: routes(ctx),
36+
},
37+
InhibitRules: inhibitRules(),
38+
Receivers: receivers,
39+
}
40+
1241
return []runtime.Object{
1342
&corev1.Secret{
1443
TypeMeta: metav1.TypeMeta{
@@ -21,8 +50,188 @@ func configSecret(ctx *common.RenderContext) ([]runtime.Object, error) {
2150
Labels: common.Labels(Name, Component, App, Version),
2251
},
2352
StringData: map[string]string{
24-
"alertmanager.yaml": ctx.Config.Alerting.Config.String(),
53+
"alertmanager.yaml": alertingConfig.String(),
2554
},
2655
},
2756
}, nil
2857
}
58+
59+
func routes(ctx *common.RenderContext) []*config.Route {
60+
var routes []*config.Route
61+
62+
routes = append(routes, &config.Route{
63+
Receiver: "criticalReceiver",
64+
Match: map[string]string{
65+
"severity": "critical",
66+
},
67+
Continue: false,
68+
})
69+
70+
for _, tRoute := range ctx.Config.Alerting.TeamRoutes {
71+
routes = append(routes, &config.Route{
72+
Receiver: fmt.Sprintf("%s-slackReceiver", tRoute.TeamLabel),
73+
Match: map[string]string{
74+
"team": tRoute.TeamLabel,
75+
},
76+
Continue: false,
77+
})
78+
}
79+
80+
routes = append(routes, &config.Route{
81+
Receiver: "genericReceiver",
82+
MatchRE: map[string]string{
83+
"severity": "info|warning",
84+
},
85+
Continue: false,
86+
})
87+
88+
return routes
89+
}
90+
91+
func inhibitRules() []*config.InhibitRule {
92+
var inhibitRules []*config.InhibitRule
93+
94+
inhibitRules = append(inhibitRules, &config.InhibitRule{
95+
SourceMatch: map[string]string{
96+
"severity": "critical",
97+
},
98+
TargetMatchRE: map[string]string{
99+
"severity": "info|warning",
100+
},
101+
Equal: []string{"alertname"},
102+
})
103+
104+
inhibitRules = append(inhibitRules, &config.InhibitRule{
105+
SourceMatch: map[string]string{
106+
"severity": "warning",
107+
},
108+
TargetMatchRE: map[string]string{
109+
"severity": "info",
110+
},
111+
Equal: []string{"alertname"},
112+
})
113+
114+
return inhibitRules
115+
}
116+
117+
func criticalReceiver(ctx *common.RenderContext) *config.Receiver {
118+
if ctx.Config.Alerting.PagerDutyRoutingKey != "" {
119+
return &config.Receiver{
120+
Name: "criticalReceiver",
121+
PagerdutyConfigs: []*config.PagerdutyConfig{
122+
{
123+
VSendResolved: common.ToPointer(true),
124+
RoutingKey: ctx.Config.Alerting.PagerDutyRoutingKey,
125+
Links: []config.PagerdutyLink{
126+
{
127+
Href: "{{ .CommonAnnotations.runbook_url }}",
128+
Text: "Runbook",
129+
},
130+
{
131+
Href: queryString,
132+
Text: "Query",
133+
},
134+
{
135+
Href: "{{ .CommonAnnotations.dashboard_url}}",
136+
Text: "Dashboard",
137+
},
138+
},
139+
},
140+
},
141+
}
142+
} else {
143+
return &config.Receiver{
144+
Name: "criticalReceiver",
145+
SlackConfigs: []*config.SlackConfig{
146+
{
147+
VSendResolved: common.ToPointer(true),
148+
APIURL: "https://slack.com/api/chat.postMessage",
149+
Channel: ctx.Config.Alerting.GenericSlackChannel,
150+
Color: "{{ if eq .Status \"firing\" -}}{{ if eq .CommonLabels.severity \"warning\" -}}warning{{- else if eq .CommonLabels.severity \"critical\" -}}danger{{- else -}}#439FE0{{- end -}}{{ else -}}good{{- end }}",
151+
Title: "[{{ .CommonLabels.alertname }} {{ .Status | toUpper }} {{ if eq .Status \"firing\" }}{{ end }}]",
152+
Text: "{{ range .Alerts }}\n*Summary*: {{ .Annotations.summary }}\n*Severity: {{ .Labels.severity }}*\n*Cluster:* {{ .Labels.cluster }}\n*Alert:* {{ .Labels.alertname }}\n*Description:* {{ .Annotations.description }}\n{{ end }}",
153+
HTTPConfig: &config.HTTPClientConfig{
154+
Authorization: &config.Authorization{
155+
Credentials: ctx.Config.Alerting.SlackOAuthToken,
156+
},
157+
},
158+
Actions: slackButtons(),
159+
},
160+
},
161+
}
162+
}
163+
}
164+
165+
func defaultReceivers(ctx *common.RenderContext) []*config.Receiver {
166+
var receivers []*config.Receiver
167+
168+
receivers = append(receivers, &config.Receiver{Name: "Black_Hole"})
169+
receivers = append(receivers, &config.Receiver{
170+
Name: "genericReceiver",
171+
SlackConfigs: []*config.SlackConfig{
172+
{
173+
VSendResolved: common.ToPointer(true),
174+
APIURL: "https://slack.com/api/chat.postMessage",
175+
Channel: ctx.Config.Alerting.GenericSlackChannel,
176+
Color: "{{ if eq .Status \"firing\" -}}{{ if eq .CommonLabels.severity \"warning\" -}}warning{{- else if eq .CommonLabels.severity \"critical\" -}}danger{{- else -}}#439FE0{{- end -}}{{ else -}}good{{- end }}",
177+
Title: "[{{ .CommonLabels.alertname }} {{ .Status | toUpper }} {{ if eq .Status \"firing\" }}{{ end }}]",
178+
Text: "{{ range .Alerts }}\n*Summary*: {{ .Annotations.summary }}\n*Severity: {{ .Labels.severity }}*\n*Cluster:* {{ .Labels.cluster }}\n*Alert:* {{ .Labels.alertname }}\n*Description:* {{ .Annotations.description }}\n{{ end }}",
179+
HTTPConfig: &config.HTTPClientConfig{
180+
Authorization: &config.Authorization{
181+
Credentials: ctx.Config.Alerting.SlackOAuthToken,
182+
},
183+
},
184+
Actions: slackButtons(),
185+
},
186+
},
187+
})
188+
return receivers
189+
}
190+
191+
func teamSlackReceivers(ctx *common.RenderContext) []*config.Receiver {
192+
var receivers []*config.Receiver
193+
for _, tRoute := range ctx.Config.Alerting.TeamRoutes {
194+
var receiver config.Receiver
195+
196+
receiver.Name = fmt.Sprintf("%s-slackReceiver", tRoute.TeamLabel)
197+
receiver.SlackConfigs = []*config.SlackConfig{
198+
{
199+
VSendResolved: common.ToPointer(true),
200+
APIURL: "https://slack.com/api/chat.postMessage",
201+
Channel: tRoute.SlackChannel,
202+
Color: "{{ if eq .Status \"firing\" -}}{{ if eq .CommonLabels.severity \"warning\" -}}warning{{- else if eq .CommonLabels.severity \"critical\" -}}danger{{- else -}}#439FE0{{- end -}}{{ else -}}good{{- end }}",
203+
Title: "[{{ .CommonLabels.alertname }} {{ .Status | toUpper }} {{ if eq .Status \"firing\" }}{{ end }}]",
204+
Text: "{{ range .Alerts }}\n*Summary*: {{ .Annotations.summary }}\n*Severity: {{ .Labels.severity }}*\n*Cluster:* {{ .Labels.cluster }}\n*Alert:* {{ .Labels.alertname }}\n*Description:* {{ .Annotations.description }}\n{{ end }}",
205+
HTTPConfig: &config.HTTPClientConfig{
206+
Authorization: &config.Authorization{
207+
Credentials: ctx.Config.Alerting.SlackOAuthToken,
208+
},
209+
},
210+
Actions: slackButtons(),
211+
},
212+
}
213+
214+
receivers = append(receivers, &receiver)
215+
}
216+
return receivers
217+
}
218+
219+
func slackButtons() []config.SlackAction {
220+
return []config.SlackAction{
221+
{
222+
Type: "button",
223+
Text: "Runbbok :runbook",
224+
URL: "{{ .CommonAnnotations.runbook_url }}",
225+
},
226+
{
227+
Type: "button",
228+
Text: "Query :prometheus:",
229+
URL: queryString,
230+
},
231+
{
232+
Type: "button",
233+
Text: "Dashboard :grafana:",
234+
URL: "{{ .CommonAnnotations.dashboard_url}}",
235+
},
236+
}
237+
}

0 commit comments

Comments
 (0)