KFLUXINFRA-1986: Update Kueue alerting rules (#7381)

- Include the rules in the platform Prometheus. When included
  in the UWM Prometheus a label with the namespace name (tekton-kueue)
  is added to the rules, which breaks them.

- Don't use critical severity so the alerts won't show up for
  SRE platform (they are not interested in those alerts and shouldn't
  do anything with them).

- Add an alert for the success rate of requests to the tekton-kueue
  mutating webhook servers.

Signed-off-by: Gal Ben Haim <[email protected]>

Author: gbenhaim

components/kueue/development/tekton-kueue-monitoring/kueue-prometheus-alerts.yaml

@@ -32,7 +32,7 @@ spec:
         ) * 100 < 100
       for: 5m
       labels:
-        severity: critical
+        severity: warning
         component: kueue
       annotations:
         summary: "Kueue deployment {{ $labels.deployment }} has unavailable replicas"
@@ -115,7 +115,7 @@ spec:
       expr: max by (status) (kueue_cluster_queue_status{cluster_queue="cluster-pipeline-queue", status="active"}) != 1
       for: 2m
       labels:
-        severity: critical
+        severity: warning
         component: kueue
       annotations:
         summary: "Kueue cluster queue is not active"
@@ -196,7 +196,7 @@ spec:
       expr: up{job=~".*kueue.*"} == 0
       for: 5m
       labels:
-        severity: critical
+        severity: warning
         component: kueue
       annotations:
         summary: "Kueue metrics endpoint is down"
@@ -217,3 +217,24 @@ spec:
         runbook_url: https://gitlab.cee.redhat.com/konflux/docs/sop/-/blob/main/infra/queue.md?ref_type=heads
         alert_team_handle: <!subteam^S05Q1P4Q2TG>
         team: konflux-infra
+
+    - alert: KueueMutatingWebhookLowSuccessRate
+      expr: |
+        100 * sum(increase(apiserver_admission_webhook_request_total{
+          name="pipelinerun-kueue-defaulter.tekton-kueue.io", code=~"2.."
+        }[10m]))
+        /
+        sum(increase(apiserver_admission_webhook_request_total{
+          name="pipelinerun-kueue-defaulter.tekton-kueue.io"
+        }[10m]))
+        < 99
+      for: 10m
+      labels:
+        severity: warning
+        component: kueue
+      annotations:
+        summary: "Kueue mutating webhook success rate is below 99%"
+        description: "The mutating webhook 'pipelinerun-kueue-defaulter.tekton-kueue.io' has had a success rate below 99% over the past 10 minutes. Possible causes include webhook errors, rejections, or unreachability (e.g., code=600)."
+        runbook_url: https://gitlab.cee.redhat.com/konflux/docs/sop/-/blob/main/infra/queue.md?ref_type=heads
+        alert_team_handle: <!subteam^S05Q1P4Q2TG>
+        team: konflux-infra

components/kueue/development/tekton-kueue/kustomization.yaml

@@ -19,3 +19,12 @@ configMapGenerator:
     behavior: replace
     files:
       - config.yaml
+
+patches:
+  - target:
+      kind: Namespace
+      name: tekton-kueue
+    patch: |-
+      - op: add
+        path: /metadata/labels/openshift.io~1cluster-monitoring
+        value: "true"

components/kueue/staging/base/tekton-kueue-monitoring/kueue-prometheus-alerts.yaml

@@ -32,7 +32,7 @@ spec:
         ) * 100 < 100
       for: 5m
       labels:
-        severity: critical
+        severity: warning
         component: kueue
       annotations:
         summary: "Kueue deployment {{ $labels.deployment }} has unavailable replicas"
@@ -115,7 +115,7 @@ spec:
       expr: max by (status) (kueue_cluster_queue_status{cluster_queue="cluster-pipeline-queue", status="active"}) != 1
       for: 2m
       labels:
-        severity: critical
+        severity: warning
         component: kueue
       annotations:
         summary: "Kueue cluster queue is not active"
@@ -196,7 +196,7 @@ spec:
       expr: up{job=~".*kueue.*"} == 0
       for: 5m
       labels:
-        severity: critical
+        severity: warning
         component: kueue
       annotations:
         summary: "Kueue metrics endpoint is down"
@@ -217,3 +217,24 @@ spec:
         runbook_url: https://gitlab.cee.redhat.com/konflux/docs/sop/-/blob/main/infra/queue.md?ref_type=heads
         alert_team_handle: <!subteam^S05Q1P4Q2TG>
         team: konflux-infra
+
+    - alert: KueueMutatingWebhookLowSuccessRate
+      expr: |
+        100 * sum(increase(apiserver_admission_webhook_request_total{
+          name="pipelinerun-kueue-defaulter.tekton-kueue.io", code=~"2.."
+        }[10m]))
+        /
+        sum(increase(apiserver_admission_webhook_request_total{
+          name="pipelinerun-kueue-defaulter.tekton-kueue.io"
+        }[10m]))
+        < 99
+      for: 10m
+      labels:
+        severity: warning
+        component: kueue
+      annotations:
+        summary: "Kueue mutating webhook success rate is below 99%"
+        description: "The mutating webhook 'pipelinerun-kueue-defaulter.tekton-kueue.io' has had a success rate below 99% over the past 10 minutes. Possible causes include webhook errors, rejections, or unreachability (e.g., code=600)."
+        runbook_url: https://gitlab.cee.redhat.com/konflux/docs/sop/-/blob/main/infra/queue.md?ref_type=heads
+        alert_team_handle: <!subteam^S05Q1P4Q2TG>
+        team: konflux-infra

components/kueue/staging/base/tekton-kueue/kustomization.yaml

@@ -33,3 +33,10 @@ patches:
     kind: Deployment
     name: tekton-kueue-controller-manager
     version: v1
+- target:
+    kind: Namespace
+    name: tekton-kueue
+  patch: |-
+    - op: add
+      path: /metadata/labels/openshift.io~1cluster-monitoring
+      value: "true"