Skip to content

Commit cf0bff8

Browse files
adinhodovicadinhodovic
committed
feat: Bump mixin dashboards
Also refactor mixins to update build files. Add ci
1 parent 80e387e commit cf0bff8

25 files changed

+4536
-1510
lines changed

‎celery-mixin/.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
vendor
2-
jsonnetfile.lock.json
2+
tmp
3+
./dashboards_out/lint
4+
.vale

‎celery-mixin/.lint‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
---
2+
exclusions:
3+
template-job-rule:
4+
reason: Jobs are set to multi in our case.
5+
target-job-rule:
6+
reason: Jobs are set to multi in our case.
7+
template-instance-rule:
8+
reason: We don't use instances.
9+
panel-datasource-rule:
10+
reason: Using a datasource for each panel.
11+
panel-title-description-rule:
12+
reason: TODO(adinhodovic)
13+
target-instance-rule:
14+
reason: We don't use instances.
15+
target-rate-interval-rule:
16+
reason: Intented 1h range.
17+
entries:

‎celery-mixin/.pint.hcl‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
rule {
2+
match {
3+
name = "CeleryTaskHighFailRate"
4+
}
5+
disable = ["promql/regexp"]
6+
}
7+
8+
rule {
9+
match {
10+
name = "CeleryHighQueueLength"
11+
}
12+
disable = ["promql/regexp"]
13+
}
14+
15+
rule {
16+
match {
17+
name = "CeleryWorkerDown"
18+
}
19+
disable = ["promql/regexp"]
20+
}

‎celery-mixin/.vale.ini‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
StylesPath = .vale/styles
2+
3+
MinAlertLevel = error
4+
5+
Packages = Readability, write-good, alex
6+
7+
[*]
8+
BasedOnStyles = Readability, write-good, alex

‎celery-mixin/Makefile‎

Lines changed: 91 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,100 @@
1-
JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 2 --string-style s --comment-style s
1+
BIN_DIR ?= $(shell pwd)/tmp/bin
22

3-
all: fmt prometheus-alerts.yaml dashboards_out lint
3+
JSONNET_VENDOR=vendor
4+
GRAFANA_DASHBOARD_LINTER_BIN=$(BIN_DIR)/dashboard-linter
5+
JB_BIN=$(BIN_DIR)/jb
6+
JSONNET_BIN=$(BIN_DIR)/jsonnet
7+
JSONNETLINT_BIN=$(BIN_DIR)/jsonnet-lint
8+
JSONNETFMT_BIN=$(BIN_DIR)/jsonnetfmt
9+
MD_FILES = $(shell find . \( -type d -name '.vale' -o -type d -name 'vendor' \) -prune -o -type f -name "*.md" -print)
10+
MARKDOWNFMT_BIN=$(BIN_DIR)/markdownfmt
11+
VALE_BIN=$(BIN_DIR)/vale
12+
PROMTOOL_BIN=$(BIN_DIR)/promtool
13+
PINT_BIN=$(BIN_DIR)/pint
14+
TOOLING=$(JB_BIN) $(JSONNETLINT_BIN) $(JSONNET_BIN) $(JSONNETFMT_BIN) $(PROMTOOL_BIN) $(GRAFANA_DASHBOARD_LINTER_BIN) $(MARKDOWNFMT_BIN) $(VALE_BIN) $(PINT_BIN)
15+
JSONNETFMT_ARGS=-n 2 --max-blank-lines 2 --string-style s --comment-style s
16+
SRC_DIR ?=dashboards
17+
OUT_DIR ?=dashboards_out
418

5-
fmt:
6-
find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
7-
xargs -n 1 -- $(JSONNET_FMT) -i
19+
.PHONY: all
20+
all: fmt generate lint test
821

9-
prometheus-alerts.yaml: mixin.libsonnet config.libsonnet $(wildcard alerts/*)
10-
jsonnet -S alerts.jsonnet > $@
22+
.PHONY: generate
23+
generate: prometheus_alerts.yaml prometheus_rules.yaml $(OUT_DIR)
1124

12-
dashboards_out: mixin.libsonnet config.libsonnet $(wildcard dashboards/*)
13-
@mkdir -p dashboards_out
14-
jsonnet -J vendor -m dashboards_out dashboards.jsonnet
25+
$(JSONNET_VENDOR): $(JB_BIN) jsonnetfile.json
26+
$(JB_BIN) install
1527

16-
lint: prometheus-alerts.yaml
17-
find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
18-
while read f; do \
19-
$(JSONNET_FMT) "$$f" | diff -u "$$f" -; \
20-
done
28+
.PHONY: fmt
29+
fmt: jsonnet-fmt markdownfmt
2130

22-
promtool check rules prometheus-alerts.yaml
31+
.PHONY: jsonnet-fmt
32+
jsonnet-fmt: $(JSONNETFMT_BIN)
33+
@find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
34+
xargs -n 1 -- $(JSONNETFMT_BIN) $(JSONNETFMT_ARGS) -i
2335

24-
test: prometheus-alerts.yaml
25-
promtool test rules tests.yaml
36+
.PHONY: markdownfmt
37+
markdownfmt: $(MARKDOWNFMT_BIN)
38+
@for file in $(MD_FILES); do $(MARKDOWNFMT_BIN) -w -gofmt $$file; done
2639

40+
prometheus_alerts.yaml: $(JSONNET_BIN) mixin.libsonnet lib/alerts.jsonnet alerts/*.libsonnet
41+
@$(JSONNET_BIN) -J vendor -S lib/alerts.jsonnet > $@
42+
43+
prometheus_rules.yaml: $(JSONNET_BIN) mixin.libsonnet lib/rules.jsonnet rules/*.libsonnet
44+
@$(JSONNET_BIN) -J vendor -S lib/rules.jsonnet > $@
45+
46+
$(OUT_DIR): $(JSONNET_BIN) $(JSONNET_VENDOR) mixin.libsonnet lib/dashboards.jsonnet $(SRC_DIR)/*.libsonnet
47+
@mkdir -p $(OUT_DIR)
48+
@$(JSONNET_BIN) -J vendor -m $(OUT_DIR) lib/dashboards.jsonnet
49+
50+
.PHONY: lint
51+
lint: jsonnet-lint alerts-lint dashboards-lint vale pint-lint
52+
53+
.PHONY: jsonnet-lint
54+
jsonnet-lint: $(JSONNETLINT_BIN) $(JSONNET_VENDOR)
55+
@find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
56+
xargs -n 1 -- $(JSONNETLINT_BIN) -J vendor
57+
58+
.PHONY: alerts-lint
59+
alerts-lint: $(PROMTOOL_BIN) prometheus_alerts.yaml prometheus_rules.yaml
60+
@$(PROMTOOL_BIN) check rules prometheus_rules.yaml
61+
@$(PROMTOOL_BIN) check rules prometheus_alerts.yaml
62+
63+
$(OUT_DIR)/.lint: $(OUT_DIR)
64+
@cp .lint $@
65+
66+
.PHONY: dashboards-lint
67+
dashboards-lint: $(GRAFANA_DASHBOARD_LINTER_BIN) $(OUT_DIR)/.lint
68+
# Replace $$interval:$$resolution var with $$__rate_interval to make dashboard-linter happy.
69+
@sed -i -e 's/$$interval:$$resolution/$$__rate_interval/g' $(OUT_DIR)/*.json
70+
@find $(OUT_DIR) -name '*.json' ! -name 'celery-tasks.json' -print0 | xargs -n 1 -0 $(GRAFANA_DASHBOARD_LINTER_BIN) lint --strict
71+
72+
.PHONY: vale
73+
vale: $(VALE_BIN)
74+
@$(VALE_BIN) sync && \
75+
$(VALE_BIN) $(MD_FILES)
76+
77+
.PHONY: pint-lint
78+
pint-lint: generate $(PINT_BIN)
79+
@# Pint will not exit with a non-zero status code if there are linting issues.
80+
@output=$$($(PINT_BIN) -n -o -l WARN lint prometheus_alerts.yaml prometheus_rules.yaml 2>&1); \
81+
if [ -n "$$output" ]; then \
82+
echo "\n$$output"; \
83+
exit 1; \
84+
fi
85+
86+
.PHONY: clean
2787
clean:
28-
rm -rf dashboards_out prometheus-alerts.yaml
88+
# Remove all files and directories ignored by git.
89+
git clean -Xfd .
90+
91+
.PHONY: test
92+
test: $(PROMTOOL_BIN) prometheus_alerts.yaml prometheus_rules.yaml
93+
@$(PROMTOOL_BIN) test rules tests/*.yaml
94+
95+
$(BIN_DIR):
96+
mkdir -p $(BIN_DIR)
97+
98+
$(TOOLING): $(BIN_DIR)
99+
@echo Installing tools from hack/tools.go
100+
@cd scripts && go list -e -mod=mod -tags tools -f '{{ range .Imports }}{{ printf "%s\n" .}}{{end}}' ./ | xargs -tI % go build -mod=mod -o $(BIN_DIR) %

‎celery-mixin/README.md‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ A set of Grafana dashboards and Prometheus alerts for Celery.
44

55
## How to use
66

7-
This mixin is designed to be vendored into the repo with your infrastructure config.
8-
To do this, use [jsonnet-bundler](https://github.com/jsonnet-bundler/jsonnet-bundler):
7+
This mixin is designed to be vendored into the repo with your infrastructure config. To do this, use [jsonnet-bundler](https://github.com/jsonnet-bundler/jsonnet-bundler):
98

109
You then have three options for deploying your dashboards
1110

@@ -15,8 +14,7 @@ You then have three options for deploying your dashboards
1514

1615
## Generate config files
1716

18-
You can manually generate the alerts, dashboards and rules files, but first you
19-
must install some tools:
17+
You can manually generate the alerts, dashboards and rules files, but first you must install some tools:
2018

2119
```sh
2220
go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb
@@ -38,10 +36,7 @@ make prometheus-alerts.yaml
3836
make dashboards_out
3937
```
4038

41-
The `prometheus-alerts.yaml` file then need to passed
42-
to your Prometheus server, and the files in `dashboards_out` need to be imported
43-
into you Grafana server. The exact details will depending on how you deploy your
44-
monitoring stack.
39+
The `prometheus-alerts.yaml` file then need to passed to your Prometheus server, and the files in `dashboards_out` need to be imported into you Grafana server. The exact details will depending on how you deploy your monitoring stack.
4540

4641
## Alerts
4742

‎celery-mixin/config.libsonnet‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,18 @@ local annotation = g.dashboard.annotation;
33

44
{
55
_config+:: {
6+
local this = self,
7+
68
// Selectors are inserted between {} in Prometheus queries.
79
celerySelector: 'job=~".*celery.*"',
810

11+
// Default datasource name
12+
datasourceName: 'default',
13+
14+
// Opt-in to multiCluster dashboards by overriding this and the clusterLabel.
15+
showMultiCluster: false,
16+
clusterLabel: 'cluster',
17+
918
grafanaUrl: 'https://grafana.com',
1019

1120
celeryIgnoredTasks: 'None',
@@ -17,8 +26,6 @@ local annotation = g.dashboard.annotation;
1726
celeryTasksOverviewUrl: '%s/d/%s/celery-tasks-overview' % [self.grafanaUrl, self.celeryTasksOverviewUid],
1827
celeryTasksByTaskUrl: '%s/d/%s/celery-tasks-by-task' % [self.grafanaUrl, self.celeryTasksByTaskUid],
1928

20-
tags: ['celery', 'celery-mixin'],
21-
2229
// If you have autoscaling workers then you maybe do not want to alert on workers that are down.
2330
celeryWorkerDownAlertEnabled: true,
2431
celeryCeleryHighQueueLengthAlertEnabled: true,
@@ -29,6 +36,17 @@ local annotation = g.dashboard.annotation;
2936
celeryHighQueueLengthThreshold: '100',
3037
celeryWorkerDownInterval: '15m',
3138

39+
dashboardIds: {
40+
'celery-tasks-overview': 'celery-tasks-overview-32s3',
41+
'celery-tasks-by-task': 'celery-tasks-by-task-32s3',
42+
},
43+
dashboardUrls: {
44+
'celery-tasks-overview': '%s/d/%s/celery-tasks-overview' % [this.grafanaUrl, this.dashboardIds['celery-tasks-overview']],
45+
'celery-tasks-by-task': '%s/d/%s/celery-tasks-by-task' % [this.grafanaUrl, this.dashboardIds['celery-tasks-by-task']],
46+
},
47+
48+
tags: ['celery', 'celery-mixin'],
49+
3250
// Custom annotations to display in graphs
3351
annotation: {
3452
enabled: false,

0 commit comments

Comments
 (0)