Add pint linter for mixin alerts (#1410)

* Update consul and minio

* Add update-mixins target

* Add validate dashboards check

There is no validation against grafana schema when dashboards are generated. So let's try to import them into Grafana instance with grizzly to see if there are any issues.

* Rename to test mixin

* Fix minio

* Update consul

* Fix jaeger

* Update minio

* Fix for grizzly

* Merge ci jobs

* Add pint alerts ci

* Update README

* Fix velero-2 linting

* change to "make lint"

* Update ci (whitespace)

* Update ci

* Revert names to fix ci

Author: v-zhuravlev

.github/workflows/test-mixins.yml

@@ -46,7 +46,7 @@ jobs:
         run: echo '${{ steps.changed-mixins.outputs.all_changed_files }}'
 
   lint-mixin:
-    name: Test Mixin
+    name: Test mixin
     runs-on: ubuntu-latest
     permissions:
       issues: write
@@ -82,13 +82,13 @@ jobs:
       - name: Install CI dependencies
         run: make install-ci-deps
 
-      - name: Install Mixin dependencies
+      - name: Install mixin dependencies
         working-directory: ./${{ matrix.mixin }}
         run: make build
 
-      - name: Lint Mixin with Mixtool
+      - name: Lint mixin
         working-directory: ./${{ matrix.mixin }}
-        run: mixtool lint mixin.libsonnet
+        run: make lint
 
       - name: Check for unexpected changes in generated files
         working-directory: ./${{ matrix.mixin }}

Makefile

@@ -24,6 +24,7 @@ install-ci-deps: install-promtool
 	go install github.com/monitoring-mixins/mixtool/cmd/mixtool@main
 	go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/[email protected]
 	go install github.com/grafana/grizzly/cmd/grr@latest
+	go install github.com/cloudflare/pint/cmd/[email protected]
 
 .PHONY: install-promtool
 install-promtool:
@@ -35,7 +36,6 @@ install-promtool:
 	@rm -rf prometheus-$(PROMTOOL_VERSION).$(UNAME_S)-$(ARCH) prometheus-$(PROMTOOL_VERSION).$(UNAME_S)-$(ARCH).tar.gz
 	@echo "promtool $(PROMTOOL_VERSION) installed successfully"
 
-
 fmt:
 	@find . -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
 			xargs -n 1 -- $(JSONNET_FMT) -i
@@ -53,17 +53,25 @@ lint-fmt:
 lint-mixins:
 	@RESULT=0; \
 	for d in $$(find . -maxdepth 1 -regex '.*-mixin\|.*-lib' -a -type d -print); do \
+		echo "=== Linting mixin: $$d ==="; \
 		if [ -e "$$d/jsonnetfile.json" ]; then \
-			echo "Installing dependencies for $$d"; \
+			echo "Installing dependencies..."; \
 			pushd "$$d" >/dev/null && jb install && popd >/dev/null; \
 		fi; \
-	done; \
-	for m in $$(find . -maxdepth 2 -name 'mixin.libsonnet' -print); do \
-			echo "Linting mixin $$m"; \
-			mixtool lint -J $$(dirname "$$m")/vendor "$$m"; \
+		if [ -e "$$d/mixin.libsonnet" ]; then \
+			echo "Running mixtool lint..."; \
+			mixtool lint -J "$$d/vendor" "$$d/mixin.libsonnet"; \
 			if [ $$? -ne 0 ]; then \
 				RESULT=1; \
 			fi; \
+		fi; \
+		if [ -d "$$d/prometheus_rules_out" ]; then \
+			echo "Running pint..."; \
+			pint lint "$$d/prometheus_rules_out/"; \
+			if [ $$? -ne 0 ]; then \
+				RESULT=1; \
+			fi; \
+		fi; \
 	done; \
 	exit $$RESULT
 

Makefile_mixin

@@ -17,13 +17,17 @@ fmt:
 		xargs -n 1 -- $(JSONNET_FMT) -i
 
 .PHONY: lint
-lint: build
-	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+lint: build lint-pint
+	@find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
 		while read f; do \
 			$(JSONNET_FMT) "$$f" | diff -u "$$f" -; \
 		done
 	mixtool lint mixin.libsonnet
 
+.PHONY: lint-pint
+lint-pint: prometheus_alerts.yaml prometheus_rules.yaml
+	pint lint prometheus_rules_out/
+
 dashboards_out: mixin.libsonnet $(wildcard **/*.libsonnet) $(wildcard **/*.jsonnet) $(wildcard panels/*) $(wildcard signals/*)
 	@mkdir -p dashboards_out
 	mixtool generate dashboards mixin.libsonnet -d dashboards_out

README.md

@@ -20,7 +20,7 @@ Based on format described [here](https://monitoring.mixins.dev/):
 * [`memcached-mixin`](memcached-mixin/): A set of reuseable and extensible dashboards
   for Memcached.
 
-* [`nodejs-mixin`](nodejs-mixin/): A set of reusable and extensible dashboards
+* [`nodejs-mixin`](nodejs-mixin/): A set of reuseable and extensible dashboards
   for Node.js.
 
 * [`caddy-mixin`](caddy-mixin/): A set of reusable and extensible dashboards
@@ -30,6 +30,13 @@ Based on format described [here](https://monitoring.mixins.dev/):
 
 You can find more in directories with `-mixin` suffix.
 
+### Linting
+
+The monitoring mixins in this repository use two linting tools to ensure quality and consistency:
+
+* [mixtool](https://github.com/monitoring-mixins/mixtool): Validates the structure and syntax of monitoring mixins, ensuring they follow the standard mixin format.
+* [pint](https://github.com/cloudflare/pint): Lints Prometheus rules and alerts to catch common mistakes and enforce best practices.
+
 ## Observability libraries
 
 Observability library is a flexible format to describe dashboards and alerts in a modular way so libraries can be imported into one another or into monitoring-mixins. Observability libraries can be found in folders with `-observ-lib` suffix. [Common library](https://github.com/grafana/jsonnet-libs/tree/master/common-lib) is also used to apply consistent style options.

velero-2-mixin/dashboards_out/clusterOverview

@@ -541,7 +541,7 @@
                "label": "Job",
                "multi": true,
                "name": "job",
-               "query": "label_values(velero_backup_success_total{job=~\"integrations/velero\"}, job)",
+               "query": "label_values(velero_backup_success_total{job=\"integrations/velero\"}, job)",
                "refresh": 2,
                "sort": 1,
                "type": "query"
@@ -556,7 +556,7 @@
                "label": "Cluster",
                "multi": true,
                "name": "cluster",
-               "query": "label_values(velero_backup_success_total{job=~\"integrations/velero\",job=~\"$job\"}, cluster)",
+               "query": "label_values(velero_backup_success_total{job=\"integrations/velero\",job=~\"$job\"}, cluster)",
                "refresh": 2,
                "sort": 1,
                "type": "query"

velero-2-mixin/dashboards_out/logs

@@ -160,7 +160,7 @@
                      "type": "loki",
                      "uid": "${loki_datasource}"
                   },
-                  "expr": "sum by (level) (count_over_time({job=~\"integrations/velero\",job=~\"$job\",instance=~\"$instance\",level=~\"$level\"}\n|~ \"$regex_search\"\n\n[$__auto]))\n",
+                  "expr": "sum by (level) (count_over_time({job=\"integrations/velero\",job=~\"$job\",instance=~\"$instance\",level=~\"$level\"}\n|~ \"$regex_search\"\n\n[$__auto]))\n",
                   "legendFormat": "{{ level }}"
                }
             ],
@@ -202,7 +202,7 @@
                      "type": "loki",
                      "uid": "${loki_datasource}"
                   },
-                  "expr": "{job=~\"integrations/velero\",job=~\"$job\",instance=~\"$instance\",level=~\"$level\"} \n|~ \"$regex_search\"\n\n\n"
+                  "expr": "{job=\"integrations/velero\",job=~\"$job\",instance=~\"$instance\",level=~\"$level\"} \n|~ \"$regex_search\"\n\n\n"
                }
             ],
             "title": "Logs",
@@ -233,7 +233,7 @@
                "label": "Job",
                "multi": true,
                "name": "job",
-               "query": "label_values({job=~\"integrations/velero\"}, job)",
+               "query": "label_values({job=\"integrations/velero\"}, job)",
                "refresh": 2,
                "sort": 1,
                "type": "query"
@@ -248,7 +248,7 @@
                "label": "Instance",
                "multi": true,
                "name": "instance",
-               "query": "label_values({job=~\"integrations/velero\",job=~\"$job\"}, instance)",
+               "query": "label_values({job=\"integrations/velero\",job=~\"$job\"}, instance)",
                "refresh": 2,
                "sort": 1,
                "type": "query"
@@ -263,7 +263,7 @@
                "label": "Level",
                "multi": true,
                "name": "level",
-               "query": "label_values({job=~\"integrations/velero\",job=~\"$job\",instance=~\"$instance\"}, level)",
+               "query": "label_values({job=\"integrations/velero\",job=~\"$job\",instance=~\"$instance\"}, level)",
                "refresh": 2,
                "sort": 1,
                "type": "query"

velero-2-mixin/mixin.libsonnet

@@ -4,7 +4,7 @@ local velero =
   velerolib.new()
   + velerolib.withConfigMixin(
     {
-      filteringSelector: 'job=~"integrations/velero"',
+      filteringSelector: 'job="integrations/velero"',
       uid: 'velero',
       enableLokiLogs: true,
     }