address review comments

Author: ceelias

monitoring/alerting/rules/database_alerts.yaml

@@ -8,11 +8,9 @@ groups:
       - title: Catalog DB Connections High
         annotations:
           description:
-            "Checks the in-use catalog database connections > 21. The default
+            Checks the in-use catalog database connections > 21. The default
             db connection pool is 22.  If it reaches the limit, the rabbitmq queues starts
             to fill up with ready messages causing issues with Model Studio pipelines.
-
-            Click on the  URL on how to remediate the issue."
           summary:
             The active catalog database connections > 21. If it reaches the max.
             db connections, it will impact the rabbitmq queues.
@@ -210,3 +208,92 @@ groups:
         labels:
           severity: warning
         uid: postgres_connection_utilization
+      - title: Crunchy Backrest Repo
+        annotations:
+          description: Checks to see /pgbackrest/repo1 filesystem is more than 50% full.
+          summary:
+            /pgbackrest/repo1 storage > 50% full in the pgbackrest repo. This typically
+            happens when the archived WAL logs are increasing and not being expired and
+            cleared.
+        condition: C
+        data:
+          - datasourceUid: prometheus
+            model:
+              editorMode: code
+              expr:
+                ((kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"sas-crunchy-platform-postgres-repo1"}
+                - kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"sas-crunchy-platform-postgres-repo1"})
+                / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"sas-crunchy-platform-postgres-repo1"})
+                * 100
+              instant: true
+              intervalMs: 1000
+              legendFormat: __auto
+              maxDataPoints: 43200
+              range: false
+              refId: A
+            refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+          - datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params: []
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                      - B
+                  reducer:
+                    params: []
+                    type: last
+                  type: query
+              datasource:
+                type: __expr__
+                uid: __expr__
+              expression: A
+              intervalMs: 1000
+              maxDataPoints: 43200
+              reducer: last
+              refId: B
+              type: reduce
+            refId: B
+            relativeTimeRange:
+              from: 600
+              to: 0
+          - datasourceUid: __expr__
+            model:
+              conditions:
+                - evaluator:
+                    params:
+                      - 50
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                      - C
+                  reducer:
+                    params: []
+                    type: last
+                  type: query
+              datasource:
+                type: __expr__
+                uid: __expr__
+              expression: B
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: C
+              type: threshold
+            refId: C
+            relativeTimeRange:
+              from: 600
+              to: 0
+        execErrState: Error
+        for: 5m
+        isPaused: false
+        labels: {}
+        noDataState: NoData
+        uid: abe80c6a-3add-477a-b228-f8283704570f

monitoring/alerting/rules/other_alerts.yaml

@@ -5,98 +5,6 @@ groups:
     folder: Other Alerts
     orgId: 1
     rules:
-      - title: Crunchy Backrest Repo
-        annotations:
-          description:
-            "Checks to see /pgbackrest/repo1 filesystem is more than 50% full.
-
-            Go to the  URL to follow the troubleshooting steps."
-          summary:
-            /pgbackrest/repo1 storage > 50% full in the pgbackrest repo. This typically
-            happens when the archived WAL logs are increasing and not being expired and
-            cleared.
-        condition: C
-        data:
-          - datasourceUid: prometheus
-            model:
-              editorMode: code
-              expr:
-                ((kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"sas-crunchy-platform-postgres-repo1"}
-                - kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"sas-crunchy-platform-postgres-repo1"})
-                / kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"sas-crunchy-platform-postgres-repo1"})
-                * 100
-              instant: true
-              intervalMs: 1000
-              legendFormat: __auto
-              maxDataPoints: 43200
-              range: false
-              refId: A
-            refId: A
-            relativeTimeRange:
-              from: 600
-              to: 0
-          - datasourceUid: __expr__
-            model:
-              conditions:
-                - evaluator:
-                    params: []
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                      - B
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-              datasource:
-                type: __expr__
-                uid: __expr__
-              expression: A
-              intervalMs: 1000
-              maxDataPoints: 43200
-              reducer: last
-              refId: B
-              type: reduce
-            refId: B
-            relativeTimeRange:
-              from: 600
-              to: 0
-          - datasourceUid: __expr__
-            model:
-              conditions:
-                - evaluator:
-                    params:
-                      - 50
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                      - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-              datasource:
-                type: __expr__
-                uid: __expr__
-              expression: B
-              intervalMs: 1000
-              maxDataPoints: 43200
-              refId: C
-              type: threshold
-            refId: C
-            relativeTimeRange:
-              from: 600
-              to: 0
-        execErrState: Error
-        for: 5m
-        isPaused: false
-        labels: {}
-        noDataState: NoData
-        uid: abe80c6a-3add-477a-b228-f8283704570f
       - title: NFS Share Usage High
         annotations:
           description: Checks if the NFS share attached to CAS is > 85% full.

monitoring/alerting/rules/viya_platform_alerts.yaml

@@ -99,10 +99,8 @@ groups:
       - title: RabbitMQ Ready Queue Backlog
         annotations:
           description:
-            Checks for accumulation of Rabbitmq ready messages > 10,000. It
-            could impact Model Studio pipelines. Follow the steps in the  url to help
-            troubleshoot. The  covers potential orphan queues and/or bottlenecking of
-            queues due to catalog service.
+            Checks for accumulation of Rabbitmq ready messages > 10,000. The  covers potential orphan
+            queues and/or bottlenecking of queues due to catalog service.
           summary:
             Rabbitmq ready messages > 10,000. This means there is a large backlog
             of messages due to high activity (which can be temporary) or something has