Add alert for falco rule violations

falco/falco.yaml.tmpl

@@ -0,0 +1,16 @@
+# PROMETHEUS RULES
+# DO NOT REMOVE line above, used in `pre-commit` hook
+
+groups:
+  - name: falco
+    # Based on https://stackoverflow.com/a/68718620
+    rules:
+      - alert: FalcoRuleViolation
+        expr: (sum(falco_events{} or falco_events{} * 0) by (k8s_pod_name, k8s_ns_name, priority, rule) - sum(falco_events{} offset 5m or falco_events{} * 0) by (k8s_pod_name,  k8s_ns_name, priority, rule)) > 0
+        for: 1m
+        labels:
+          team: infra
+        annotations:
+          summary: Falco Rule Violation
+          impact: "Rule: {{ $labels.rule }} violation in {{ $labels.k8s_ns_name }}/{{ $labels.k8s_pod_name }}, priority: {{ $labels.priority }}"
+          action: For more information check https://falco-falcosidekick-ui.$ENVIRONMENT.$PROVIDER.uw.systems/events

falco/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+configMapGenerator:
+  - files:
+      - falco.yaml.tmpl=falco.yaml.tmpl
+    name: alert-templates-falco
+
+patches:
+  - path: volume.yaml

falco/volume.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: thanos-rule
+spec:
+  template:
+    spec:
+      initContainers:
+        - name: render-alerts
+          volumeMounts:
+            - name: rule-templates-falco
+              mountPath: /var/thanos/rule-templates/falco
+      volumes:
+        - name: rule-templates-falco
+          configMap:
+            name: alert-templates-falco