add CsvAbnornalReplacing alerts

Signed-off-by: Ben Ye <[email protected]>

Author: yeya24

deploy/chart/templates/0000_90_olm_01-prometheus-rule.yaml

@@ -9,12 +9,23 @@ metadata:
     role: alert-rules
 spec:
   groups:
-    - name: olm.failing_operators.rules
+    - name: olm.csv_abnormal.rules
       rules:
-        - alert: FailingOperator
+        - alert: CsvAbnormalFailedOver2Min
+          expr: csv_abnormal{phase=~"^Failed$",exported_namespace=~"(^openshift.*|^kube.*|^redhat.*|^default$)"}
+          for: 2m
+          labels:
+            severity: warning
+            namespace: "{{ "{{ $labels.namespace }}" }}"
           annotations:
             message: Failed to install Operator {{ printf "{{ $labels.name }}"  }} version {{ printf "{{ $labels.version }}"  }}. Reason-{{ printf "{{ $labels.reason }}" }}
-          expr: csv_abnormal{phase="Failed"}
+
+        - alert: CsvAbnormalOver30Min
+          expr: csv_abnormal{phase=~"(^Replacing$|^Pending$|^Deleting$|^Unknown$)",exported_namespace=~"(^openshift.*|^kube.*|^redhat.*|^default$)"}
+          for: 30m
           labels:
             severity: warning
+            namespace: "{{ "{{ $labels.namespace }}" }}"
+          annotations:
+            message: Failed to install Operator {{ printf "{{ $labels.name }}"  }} version {{ printf "{{ $labels.version }}"  }}. Phase-{{ printf "{{ $labels.phase }}" }} Reason-{{ printf "{{ $labels.reason }}" }}
 {{ end }}

manifests/0000_90_olm_01-prometheus-rule.yaml

@@ -8,11 +8,22 @@ metadata:
     role: alert-rules
 spec:
   groups:
-    - name: olm.failing_operators.rules
+    - name: olm.csv_abnormal.rules
       rules:
-        - alert: FailingOperator
+        - alert: CsvAbnormalFailedOver2Min
+          expr: csv_abnormal{phase=~"^Failed$",exported_namespace=~"(^openshift.*|^kube.*|^redhat.*|^default$)"}
+          for: 2m
+          labels:
+            severity: warning
+            namespace: "{{ $labels.namespace }}"
           annotations:
             message: Failed to install Operator {{ $labels.name }} version {{ $labels.version }}. Reason-{{ $labels.reason }}
-          expr: csv_abnormal{phase="Failed"}
+
+        - alert: CsvAbnormalOver30Min
+          expr: csv_abnormal{phase=~"(^Replacing$|^Pending$|^Deleting$|^Unknown$)",exported_namespace=~"(^openshift.*|^kube.*|^redhat.*|^default$)"}
+          for: 30m
           labels:
             severity: warning
+            namespace: "{{ $labels.namespace }}"
+          annotations:
+            message: Failed to install Operator {{ $labels.name }} version {{ $labels.version }}. Phase-{{ $labels.phase }} Reason-{{ $labels.reason }}