Rely on kube_job_status_start_time for alerting rather than for clause

Signed-off-by: Arunprasad Rajkumar <[email protected]>

Author: arajkumar

alerts/apps_alerts.libsonnet

@@ -1,6 +1,7 @@
 {
   _config+:: {
     kubeStateMetricsSelector: error 'must provide selector for kube-state-metrics',
+    kubeJobTimeoutDuration: error 'must provide value for kubeJobTimeoutDuration',
     namespaceSelector: null,
     prefixedNamespaceSelector: if self.namespaceSelector != null then self.namespaceSelector + ',' else '',
   },
@@ -237,16 +238,15 @@
           {
             alert: 'KubeJobNotCompleted',
             expr: |||
-              time() - kube_job_status_start_time{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s}
+              time() - max by(namespace, job_name) (kube_job_status_start_time{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s}
                 and
-              kube_job_status_active{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s} > 0
+              kube_job_status_active{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s} > 0) > %(kubeJobTimeoutDuration)s
             ||| % $._config,
-            'for': '12h',
             labels: {
               severity: 'warning',
             },
             annotations: {
-              description: 'Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than 12 hours to complete.',
+              description: 'Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "%(kubeJobTimeoutDuration)s" | humanizeDuration }} to complete.' % $._config,
               summary: 'Job did not complete in time',
             },
           },

config.libsonnet

@@ -106,5 +106,8 @@
     // so that specific storage alerts will not fire.With the default selector, adding a label `exclude-from-alerts: 'true'`
     // to the PVC will have the desired effect.
     pvExcludedSelector: 'label_excluded_from_alerts="true"',
+
+    // Default timeout value for k8s Jobs. The jobs which are active beyond this duration would trigger KubeJobNotCompleted alert.
+    kubeJobTimeoutDuration: 12 * 60 * 60,
   },
 }

runbook.md

@@ -58,7 +58,7 @@ This page collects this repositories alerts and begins the process of describing
 + *Severity*: warning
 
 ##### Alert Name: "KubeJobNotCompleted"
-+ *Message*: `Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than 12h to complete.`
++ *Message*: `Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "%(kubeJobTimeoutDuration)s" | humanizeDuration }} to complete.`
 + *Severity*: warning
 + *Action*: Check the job using `kubectl describe job <job>` and look at the pod logs using `kubectl logs <pod>` for further information.
 

tests.yaml

@@ -846,30 +846,22 @@ tests:
 - interval: 1m
   input_series:
   - series: 'kube_job_status_start_time{namespace="ns1", job="kube-state-metrics", instance="instance1", job_name="job1"}'
-    values: '0+0x740'
+    values: '0+0x200 _x500 0+0x40'
   - series: 'kube_job_status_active{namespace="ns1", job="kube-state-metrics", instance="instance1", job_name="job1"}'
-    values: '1+0x740'
-  promql_expr_test:
-  - eval_time: 12h
-    expr: time() - kube_job_status_start_time and kube_job_status_active > 0
-    exp_samples:
-    - value: 43200
-      labels: '{namespace="ns1", job="kube-state-metrics", instance="instance1", job_name="job1"}'
+    values: '1x200 _x500 1x40'
   alert_rule_test:
   - eval_time: 6h
     alertname: KubeJobNotCompleted
-  - eval_time: 12h
+  - eval_time: 12h1m
     alertname: KubeJobNotCompleted
     exp_alerts:
     - exp_labels:
         namespace: ns1
         job_name: job1
         severity: warning
-        instance: instance1
-        job: kube-state-metrics
       exp_annotations:
         summary: "Job did not complete in time"
-        description: "Job ns1/job1 is taking more than 12 hours to complete."
+        description: "Job ns1/job1 is taking more than 12h 0m 0s to complete."
         runbook_url: "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobnotcompleted"
 
 - interval: 1m