Merge pull request #709 from stackhpc/rabbitmq-alert-fixes

Improve RabbitMQ alerts

Author: priteau

etc/kayobe/kolla/config/prometheus/rabbitmq.rules

@@ -5,69 +5,76 @@
 groups:
 - name: rabbitmq.rules
   rules:
-  - alert: RabbitmqNodeDown
-    expr: sum(rabbitmq_build_info{host_name!=""}) < 3
+  - alert: RabbitMQNodeDown
+    expr: sum(rabbitmq_build_info{instance!=""}) < 3
     for: 30m
     labels:
       severity: critical
     annotations:
-      description: Rabbitmq node down on {{ $labels.host_name }}
-  - alert: RabbitmqConsumersLowUtilization
+      description: RabbitMQ node down on {{ $labels.instance }}
+  - alert: RabbitMQConsumersLowUtilization
     expr: rabbitmq_queue_consumer_utilisation < 0.4
     for: 5m
     labels:
       severity: warning
     annotations:
-      description: RabbitMQ consumers message consumption speed is low on {{ $labels.host_name }}
-  - alert: RabbitmqNodeNotDistributed
+      description: RabbitMQ consumers message consumption speed is low on {{ $labels.instance }}
+  - alert: RabbitMQNodeNotDistributed
     expr: erlang_vm_dist_node_state < 3
     for: 5m
     labels:
       severity: critical
     annotations:
-      description: Rabbitmq node not distributed on node {{ $labels.host_name }}
-  - alert: RabbitmqMemoryHigh
+      description: RabbitMQ node not distributed on node {{ $labels.instance }}
+  - alert: RabbitMQMemoryHigh
     expr: rabbitmq_process_resident_memory_bytes / rabbitmq_resident_memory_limit_bytes
       * 100 > 90
     for: 2m
     labels:
       severity: warning
     annotations:
-      description: Rabbitmq memory too high on {{ $labels.host_name }}
-  - alert: RabbitmqFileDescriptorsUsage
+      description: RabbitMQ memory too high on {{ $labels.instance }}
+  - alert: RabbitMQFileDescriptorsUsage
     expr: rabbitmq_process_open_fds / rabbitmq_process_max_fds * 100 > 90
     for: 2m
     labels:
       severity: warning
     annotations:
-      description: Rabbitmq file descriptors usage on {{ $labels.host_name }}
-  - alert: RabbitmqTooMuchUnack
-    expr: sum(rabbitmq_queue_messages_unacked) BY (queue) > 1000
+      description: RabbitMQ file descriptors usage on {{ $labels.instance }}
+  - alert: RabbitMQTooMuchReady
+    expr: rabbitmq_queue_messages_ready > 100
     for: 1m
     labels:
       severity: warning
     annotations:
-      description: Rabbitmq too much unack on {{ $labels.host_name }}
-  - alert: RabbitmqTooMuchConnections
+      description: RabbitMQ too much ready on {{ $labels.instance }}
+  - alert: RabbitMQTooMuchUnack
+    expr: rabbitmq_queue_messages_unacked > 100
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      description: RabbitMQ too much unack on {{ $labels.instance }}
+  - alert: RabbitMQTooMuchConnections
     expr: rabbitmq_connections > 1000
     for: 2m
     labels:
       severity: warning
     annotations:
-      description: Rabbitmq too much connections on {{ $labels.host_name }}
-  - alert: RabbitmqNoQueueConsumer
+      description: RabbitMQ too much connections on {{ $labels.instance }}
+  - alert: RabbitMQNoQueueConsumer
     expr: rabbitmq_queue_consumers < 1
     for: 1m
     labels:
       severity: warning
     annotations:
-      description: Rabbitmq no queue consumer on {{ $labels.host_name }}
-  - alert: RabbitmqUnroutableMessages
+      description: RabbitMQ no queue consumer on {{ $labels.instance }}
+  - alert: RabbitMQUnroutableMessages
     expr: increase(rabbitmq_channel_messages_unroutable_returned_total[1m]) > 0 or  increase(rabbitmq_channel_messages_unroutable_dropped_total[1m]) > 0
     for: 2m
     labels:
       severity: warning
     annotations:
-      description: Rabbitmq unroutable messages on {{ $labels.host_name }}
+      description: RabbitMQ unroutable messages on {{ $labels.instance }}
 
 {% endraw %}

releasenotes/notes/rabbimq-too-many-ready-c18ecea9a13c8f2a.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Raises an alert when the count of RabbitMQ ready messages increases above a
+    threshold.