Skip to content

Commit 6421547

Browse files
muicodermuicoder
committed
Adding association between a Kafka cluster and a notifier
Signed-off-by: muicoder <[email protected]> linkedin#611 Adding WeCom/DingTalk template
1 parent e593345 commit 6421547

File tree

15 files changed

+235
-28
lines changed

15 files changed

+235
-28
lines changed

‎.github/workflows/action.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
env:
2+
BASE64manifest: IyEvYmluL3NoCgpDTUQ9JChpZiBidWlsZGFoID4vZGV2L251bGw7IHRoZW4gZWNobyBidWlsZGFoOyBlbGlmIHNlYWxvcyA+L2Rldi9udWxsOyB0aGVuIGVjaG8gc2VhbG9zOyBmaSkKTUY9Im1mOiQoZGF0ZSArJUYpIgoKUkVQTz0iJHsxOi1kb2NrZXIuaW8vYml0bmFtaS9tZXRyaWNzLXNlcnZlcjpkb2NrZXIuaW8vbXVpY29kZXIvbWV0cmljcy1zZXJ2ZXJ9IgpUQUdTPSIkezI6LTAuNi4zfSIKVEFHPSIkezM6LSRUQUdTfSIKCmlmIFsgIiR7UkVQTyU6Kn0iICE9ICIkUkVQTyIgXTsgdGhlbgogIGlmIFsgIiR7VEFHUyUsKn0iICE9ICIkVEFHUyIgXTsgdGhlbgogICAgZWNobyAiJFRBR1MiIHwgc2VkICJzfix+XG5+ZyIgfCB3aGlsZSByZWFkIC1yIHRhZzsgZG8KICAgICAgZWNobyAiJHtSRVBPJToqfTokdGFnIgogICAgZG9uZSB8ICRDTUQgcHVsbAogIGVsc2UKICAgICRDTUQgcHVsbCAtLXBvbGljeT1hbHdheXMgLS1wbGF0Zm9ybT1saW51eC9hbWQ2NCAiJHtSRVBPJToqfTokVEFHUyIKICAgICRDTUQgdGFnICIke1JFUE8lOip9OiRUQUdTIiAiJHtSRVBPIyo6fTokVEFHUy1hbWQ2NCIKICAgICRDTUQgcHVsbCAtLXBvbGljeT1hbHdheXMgLS1wbGF0Zm9ybT1saW51eC9hcm02NCAiJHtSRVBPJToqfTokVEFHUyIKICAgICRDTUQgdGFnICIke1JFUE8lOip9OiRUQUdTIiAiJHtSRVBPIyo6fTokVEFHUy1hcm02NCIKICAgIFRBR1M9IiRUQUdTLWFtZDY0LCRUQUdTLWFybTY0IgogIGZpCmZpCgplY2hvICIkVEFHUyIgfCBzZWQgInN+LH5cbn5nIiB8IHdoaWxlIHJlYWQgLXIgdGFnOyBkbwogIGVjaG8gIiR7UkVQTyMqOn06JHRhZyIKZG9uZSB8IHhhcmdzICRDTUQgbWFuaWZlc3QgY3JlYXRlIC0tYWxsICIkTUYiCiRDTUQgbWFuaWZlc3QgcHVzaCAtLWFsbCAiJE1GIiAiZG9ja2VyOi8vJHtSRVBPIyo6fTokVEFHIgokQ01EIG1hbmlmZXN0IHJtICIkTUYiIHx8IHRydWUK
3+
jobs:
4+
aio-manifest:
5+
needs:
6+
- build
7+
runs-on: ubuntu-latest
8+
steps:
9+
- name: Login to DockerHub
10+
uses: docker/login-action@v2
11+
with:
12+
password: ${{ secrets.DOCKERHUB_PASSWORD }}
13+
username: ${{ secrets.DOCKERHUB_USERNAME }}
14+
- name: manifest
15+
run: echo ${{ env.BASE64manifest }} | base64 -d | sh -s docker.io/${{ secrets.DOCKERHUB_USERNAME }}/burrow action-amd64,action-arm64 action
16+
build:
17+
runs-on: ubuntu-latest
18+
steps:
19+
- name: Checkout
20+
uses: actions/checkout@v3
21+
- name: Build artifacts
22+
run: make update && git diff
23+
- name: Set up QEMU
24+
uses: docker/setup-qemu-action@v2
25+
- name: Set up Docker Buildx
26+
uses: docker/setup-buildx-action@v2
27+
- name: Login to DockerHub
28+
uses: docker/login-action@v2
29+
with:
30+
password: ${{ secrets.DOCKERHUB_PASSWORD }}
31+
username: ${{ secrets.DOCKERHUB_USERNAME }}
32+
- name: Build and push
33+
uses: docker/build-push-action@v3
34+
with:
35+
context: .
36+
file: Dockerfile
37+
platforms: ${{ matrix.os}}/${{ matrix.arch}}
38+
provenance: false
39+
pull: true
40+
push: true
41+
sbom: false
42+
tags: ${{ secrets.DOCKERHUB_USERNAME }}/burrow:action-${{ matrix.arch}}
43+
strategy:
44+
matrix:
45+
arch:
46+
- arm64
47+
- amd64
48+
os:
49+
- linux
50+
name: build
51+
on:
52+
workflow_dispatch:

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: CI
22

3-
on: [push, pull_request]
3+
on: [pull_request]
44

55
jobs:
66
test:

‎Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
# stage 1: builder
2-
FROM golang:1.20.4-alpine as builder
2+
FROM golang:1.20-alpine as builder
33

44
ENV BURROW_SRC /usr/src/Burrow/
55

6-
RUN apk add --no-cache git curl
6+
RUN apk add --no-cache git curl make
77
COPY . $BURROW_SRC
88
WORKDIR $BURROW_SRC
99

10-
RUN go mod tidy && go build -o /tmp/burrow .
10+
RUN go build -trimpath -ldflags "-extldflags -static -w -s" -o /tmp/burrow .
1111

1212
# stage 2: runner
1313
FROM alpine:3.18

‎config/burrow.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ min-distance=1
5858

5959
[notifier.default]
6060
class-name="http"
61+
cluster="local"
6162
url-open="http://someservice.example.com:1467/v1/event"
6263
interval=60
6364
timeout=5

‎config/default-dingtalk-post.tmpl‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{"msgtype": "markdown","markdown": {"title":"Kafka LagChecker", "text": "
2+
{{- $StatusURL := "https://pkg.go.dev/github.com/linkedin/Burrow/core/protocol#StatusConstant"}}
3+
{{- $FormatString := "2006-01-02 15:04:05"}}
4+
# Kafka: {{.Cluster}}
5+
ConsumerGroup: {{.Group}}{{- with .Result.Status}}
6+
{{- if eq . 0}}NotFound{{end}}
7+
{{- if eq . 1}}normal{{end}}
8+
{{- if eq . 2}}lagging{{end}}
9+
{{- if eq . 3}}abnormal{{end}}
10+
{{- end}}
11+
**Status:** Total(Partitions={{.Result.TotalPartitions}},Lag={{.Result.TotalLag}})[{{- with .Result.Status}}
12+
{{- if eq . 0}}NotFound{{end}}
13+
{{- if eq . 1}}{{.}}{{end}}
14+
{{- if eq . 2}}{{.}}{{end}}
15+
{{- if eq . 3}}{{.}}{{end}}
16+
{{- end}}]({{$StatusURL}}){{printf "%.2f" .Result.Complete}}
17+
{{- if .Result.Maxlag|maxlag}}
18+
**MaxLagDetails:**
19+
{{- with .Result.Maxlag}}
20+
{{.Topic}}[{{.Status.String}}](){{printf "%.2f" .Complete}}
21+
{{- if .Owner}}\n\tConsumerHost={{.Owner}}{{end}}
22+
{{- if .ClientID}}\n\tConsumerClientID={{.ClientID}}{{end}}
23+
\tCurrentLag={{.CurrentLag}}
24+
\tPartition={{.Partition}}
25+
{{- end}}
26+
{{- end}}
27+
{{- $TotalErrors := len .Result.Partitions}}
28+
{{- if $TotalErrors}}
29+
### {{$TotalErrors}} partitions have problems
30+
>**CountPartitions:**
31+
{{- range $k,$v := .Result.Partitions|partitioncounts}}
32+
{{- if ne $v 0}}\n\t{{$k}}={{$v}}{{end}}
33+
{{- end}}
34+
**TopicsByStatus:**
35+
{{- range $k,$v := .Result.Partitions|topicsbystatus}}
36+
\t{{$k}}={{$v}}
37+
{{- end}}
38+
**PartitionDetails:**
39+
{{- range .Result.Partitions}}
40+
{{.Topic}}[{{.Status.String}}](){{printf "%.2f" .Complete}}
41+
{{- if .Owner}}\n\tConsumerHost={{.Owner}}{{end}}
42+
{{- if .ClientID}}\n\tConsumerClientID={{.ClientID}}{{end}}
43+
\tCurrentLag={{.CurrentLag}}
44+
\tPartition={{.Partition}}
45+
\tStart={{formattimestamp .Start.Timestamp $FormatString}}
46+
\t\tOffset={{.Start.Offset}}\tLag={{.Start.Lag.Value}}
47+
\tEnd={{formattimestamp .End.Timestamp $FormatString}}
48+
\t\tOffset={{.End.Offset}}\tLag={{.End.Lag.Value}}
49+
{{- end}}
50+
{{- end}}
51+
"
52+
}}

‎config/default-wecom-post.tmpl‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{"msgtype": "markdown","markdown": {"content": "
2+
{{- $StatusURL := "https://pkg.go.dev/github.com/linkedin/Burrow/core/protocol#StatusConstant"}}
3+
{{- $FormatString := "2006-01-02 15:04:05"}}
4+
# Kafka: {{.Cluster}}
5+
ConsumerGroup: {{.Group}}{{- with .Result.Status}}
6+
{{- if eq . 0}}NotFound{{end}}
7+
{{- if eq . 1}}<font color=\"info\">normal</font>{{end}}
8+
{{- if eq . 2}}<font color=\"warning\">lagging</font>{{end}}
9+
{{- if eq . 3}}<font color=\"comment\">abnormal</font>{{end}}
10+
{{- end}}
11+
**Status:** Total(Partitions={{.Result.TotalPartitions}},Lag={{.Result.TotalLag}})[{{- with .Result.Status}}
12+
{{- if eq . 0}}NotFound{{end}}
13+
{{- if eq . 1}}<font color=\"info\">{{.}}</font>{{end}}
14+
{{- if eq . 2}}<font color=\"warning\">{{.}}</font>{{end}}
15+
{{- if eq . 3}}<font color=\"comment\">{{.}}</font>{{end}}
16+
{{- end}}]({{$StatusURL}}){{printf "%.2f" .Result.Complete}}
17+
{{- if .Result.Maxlag|maxlag}}
18+
**MaxLagDetails:**
19+
{{- with .Result.Maxlag}}
20+
{{.Topic}}[{{.Status.String}}](){{printf "%.2f" .Complete}}
21+
{{- if .Owner}}\n\tConsumerHost={{.Owner}}{{end}}
22+
{{- if .ClientID}}\n\tConsumerClientID={{.ClientID}}{{end}}
23+
\tCurrentLag={{.CurrentLag}}
24+
\tPartition={{.Partition}}
25+
{{- end}}
26+
{{- end}}
27+
{{- $TotalErrors := len .Result.Partitions}}
28+
{{- if $TotalErrors}}
29+
### <font color=\"comment\">{{$TotalErrors}} partitions have problems</font>
30+
>**CountPartitions:**
31+
{{- range $k,$v := .Result.Partitions|partitioncounts}}
32+
{{- if ne $v 0}}\n\t{{$k}}={{$v}}{{end}}
33+
{{- end}}
34+
**TopicsByStatus:**
35+
{{- range $k,$v := .Result.Partitions|topicsbystatus}}
36+
\t{{$k}}={{$v}}
37+
{{- end}}
38+
**PartitionDetails:**
39+
{{- range .Result.Partitions}}
40+
{{.Topic}}[{{.Status.String}}](){{printf "%.2f" .Complete}}
41+
{{- if .Owner}}\n\tConsumerHost={{.Owner}}{{end}}
42+
{{- if .ClientID}}\n\tConsumerClientID={{.ClientID}}{{end}}
43+
\tCurrentLag={{.CurrentLag}}
44+
\tPartition={{.Partition}}
45+
\tStart={{formattimestamp .Start.Timestamp $FormatString}}
46+
\t\tOffset={{.Start.Offset}}\tLag={{.Start.Lag.Value}}
47+
\tEnd={{formattimestamp .End.Timestamp $FormatString}}
48+
\t\tOffset={{.End.Offset}}\tLag={{.End.Lag.Value}}
49+
{{- end}}
50+
{{- end}}
51+
"
52+
}}

‎core/internal/helpers/coordinators.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,12 @@ func (m *MockModule) GetName() string {
7474
return args.String(0)
7575
}
7676

77+
// GetCluster mocks the notifier.Module GetCluster func
78+
func (m *MockModule) GetCluster() string {
79+
args := m.Called()
80+
return args.String(0)
81+
}
82+
7783
// GetGroupAllowlist mocks the notifier.Module GetGroupAllowlist func
7884
func (m *MockModule) GetGroupAllowlist() *regexp.Regexp {
7985
args := m.Called()

‎core/internal/httpserver/config.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ func (hc *Coordinator) configNotifierHTTP(w http.ResponseWriter, r *http.Request
213213
SendClose: viper.GetBool(configRoot + ".send-close"),
214214
ExtraCa: viper.GetString(configRoot + ".extra-ca"),
215215
NoVerify: viper.GetString(configRoot + ".noverify"),
216+
Cluster: viper.GetString(configRoot + ".cluster"),
216217
},
217218
Request: requestInfo,
218219
})
@@ -265,6 +266,7 @@ func (hc *Coordinator) configNotifierEmail(w http.ResponseWriter, r *http.Reques
265266
To: viper.GetString(configRoot + ".to"),
266267
ExtraCa: viper.GetString(configRoot + ".extra-ca"),
267268
NoVerify: viper.GetString(configRoot + ".noverify"),
269+
Cluster: viper.GetString(configRoot + ".cluster"),
268270
},
269271
Request: requestInfo,
270272
})

‎core/internal/httpserver/structs.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ type httpResponseConfigModuleNotifierHTTP struct {
202202
SendClose bool `json:"send-close"`
203203
ExtraCa string `json:"extra-ca"`
204204
NoVerify string `json:"noverify"`
205+
Cluster string `json:"cluster"`
205206
}
206207

207208
type httpResponseConfigModuleNotifierSlack struct {
@@ -238,6 +239,7 @@ type httpResponseConfigModuleNotifierEmail struct {
238239
To string `json:"to"`
239240
ExtraCa string `json:"extra-ca"`
240241
NoVerify string `json:"noverify"`
242+
Cluster string `json:"cluster"`
241243
}
242244

243245
type httpResponseConfigModuleNotifierNull struct {

‎core/internal/notifier/coordinator.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ import (
4848
type Module interface {
4949
protocol.Module
5050
GetName() string
51+
GetCluster() string
5152
GetGroupAllowlist() *regexp.Regexp
5253
GetGroupDenylist() *regexp.Regexp
5354
GetLogger() *zap.Logger
@@ -95,7 +96,7 @@ type Coordinator struct {
9596

9697
// getModuleForClass returns the correct module based on the passed className. As part of the Configure steps, if there
9798
// is any error, it will panic with an appropriate message describing the problem.
98-
func getModuleForClass(app *protocol.ApplicationContext, moduleName, className string, groupAllowlist, groupDenylist *regexp.Regexp, extras map[string]string, templateOpen, templateClose *template.Template) protocol.Module {
99+
func getModuleForClass(app *protocol.ApplicationContext, moduleName, className string, groupAllowlist, groupDenylist *regexp.Regexp, extras map[string]string, templateOpen, templateClose *template.Template, cluster string) protocol.Module {
99100
logger := app.Logger.With(
100101
zap.String("type", "module"),
101102
zap.String("coordinator", "notifier"),
@@ -113,6 +114,7 @@ func getModuleForClass(app *protocol.ApplicationContext, moduleName, className s
113114
extras: extras,
114115
templateOpen: templateOpen,
115116
templateClose: templateClose,
117+
cluster: cluster,
116118
}
117119
case "email":
118120
return &EmailNotifier{
@@ -123,6 +125,7 @@ func getModuleForClass(app *protocol.ApplicationContext, moduleName, className s
123125
extras: extras,
124126
templateOpen: templateOpen,
125127
templateClose: templateClose,
128+
cluster: cluster,
126129
}
127130
case "null":
128131
return &NullNotifier{
@@ -133,6 +136,7 @@ func getModuleForClass(app *protocol.ApplicationContext, moduleName, className s
133136
extras: extras,
134137
templateOpen: templateOpen,
135138
templateClose: templateClose,
139+
cluster: cluster,
136140
}
137141
default:
138142
panic("Unknown notifier className provided: " + className)
@@ -194,6 +198,8 @@ func (nc *Coordinator) Configure() {
194198
groupAllowlist = re
195199
}
196200

201+
cluster := viper.GetString(configRoot + ".cluster")
202+
197203
// Compile the denylist for the consumer groups to not notify for
198204
var groupDenylist *regexp.Regexp
199205
denylist := viper.GetString(configRoot + ".group-denylist")
@@ -227,7 +233,7 @@ func (nc *Coordinator) Configure() {
227233
templateClose = tmpl.Templates()[0]
228234
}
229235

230-
module := getModuleForClass(nc.App, name, viper.GetString(configRoot+".class-name"), groupAllowlist, groupDenylist, extras, templateOpen, templateClose)
236+
module := getModuleForClass(nc.App, name, viper.GetString(configRoot+".class-name"), groupAllowlist, groupDenylist, extras, templateOpen, templateClose, cluster)
231237
module.Configure(name, configRoot)
232238
nc.modules[name] = module
233239
interval := viper.GetInt64(configRoot + ".interval")
@@ -436,6 +442,9 @@ func (nc *Coordinator) checkAndSendResponseToModules(response *protocol.Consumer
436442
for _, genericModule := range nc.modules {
437443
module := genericModule.(Module)
438444

445+
if module.GetCluster() != "" && response.Cluster != module.GetCluster() {
446+
continue
447+
}
439448
// No allowlist means everything passes
440449
groupAllowlist := module.GetGroupAllowlist()
441450
groupDenylist := module.GetGroupDenylist()

0 commit comments

Comments
 (0)