Merge pull request #1121 from rackerlabs/PUC-717-20250801

fix(rabbitmq): Sets rabbitmq cluster replicas to 3 to match OpenStack helm values

Author: cardoe

components/openstack/openstack-cluster.yaml

@@ -1,7 +1,20 @@
+---
 apiVersion: rabbitmq.com/v1beta1
 kind: RabbitmqCluster
 metadata:
   name: rabbitmq
   annotations:
     # do not allow ArgoCD to delete our cluster
     argocd.argoproj.io/sync-options: Delete=false
+spec:
+  replicas: 3
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: pdb-rabbitmq
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: rabbitmq

operators/rabbitmq-system/cluster-operator/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - https://github.com/rabbitmq/cluster-operator/releases/download/v2.14.0/cluster-operator.yml
+  - https://github.com/rabbitmq/cluster-operator/releases/download/v2.15.0/cluster-operator.yml
 
 # this needs to be here to remove the namespace from the above
 # because the cluster-operator defines the same namespace and it would conflict

operators/rabbitmq-system/monitoring/kustomization.yaml

@@ -17,5 +17,4 @@ resources:
   - ./rules/rabbitmq/cluster-alarms.yml
   - ./rules/rabbitmq/insufficient-established-erlang-distribution-links.yml
   - ./rules/rabbitmq/persistent-volume-missing.yml
-  - ./rules/rabbitmq/tcp-sockets-near-limit.yml
   - ./rules/rabbitmq-cluster-operator/unavailable-replicas.yml

operators/rabbitmq-system/monitoring/rules/rabbitmq-per-object/queue-has-no-consumers.yml

@@ -14,7 +14,7 @@ spec:
       expr: |
           (
             ((rabbitmq_detailed_queue_consumers{vhost="/", queue=~".*"} == 0) + rabbitmq_detailed_queue_messages) > 0
-          ) * on (instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info
+          ) * on (instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
       for: 10m
       annotations:
         description: |

operators/rabbitmq-system/monitoring/rules/rabbitmq-per-object/queue-is-growing.yml

@@ -15,7 +15,7 @@ spec:
       expr: |
           (
               avg_over_time(rabbitmq_detailed_queue_messages[10m]) - avg_over_time(rabbitmq_detailed_queue_messages[10m] offset 1m) > 1
-          ) * on (instance, job) group_left(rabbitmq_cluster) rabbitmq_identity_info
+          ) * on (instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
       for: 10m
       annotations:
         description: |

operators/rabbitmq-system/monitoring/rules/rabbitmq/cluster-alarms.yml

@@ -14,7 +14,7 @@ spec:
       expr: |
         max by(rabbitmq_cluster) (
           max_over_time(rabbitmq_alarms_memory_used_watermark[5m])
-          * on(instance) group_left(rabbitmq_cluster, rabbitmq_node, pod) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
+          * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
         ) > 0
       keep_firing_for: 5m
       annotations:
@@ -30,7 +30,7 @@ spec:
       expr: |
         max by(rabbitmq_cluster) (
           max_over_time(rabbitmq_alarms_free_disk_space_watermark[5m])
-          * on(instance) group_left(rabbitmq_cluster, rabbitmq_node, pod) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
+          * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
         ) > 0
       keep_firing_for: 5m
       annotations:
@@ -46,7 +46,7 @@ spec:
       expr: |
         max by(rabbitmq_cluster) (
           max_over_time(rabbitmq_alarms_file_descriptor_limit[5m])
-          * on(instance) group_left(rabbitmq_cluster, rabbitmq_node, pod) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
+          * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
         ) > 0
       keep_firing_for: 5m
       annotations:

operators/rabbitmq-system/monitoring/rules/rabbitmq/container-restarts.yml

@@ -12,9 +12,9 @@ spec:
     rules:
     - alert: ContainerRestarts
       expr: |
-        increase(kube_pod_container_status_restarts_total[10m]) * on(namespace, pod, container) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)
-        >=
-        1
+        increase(kube_pod_container_status_restarts_total[10m])
+        * on(namespace, pod, container) group_left(rabbitmq_cluster) max by (namespace, pod, container, rabbitmq_cluster) (rabbitmq_identity_info)
+        >= 1
       for: 5m
       annotations:
         description: |

operators/rabbitmq-system/monitoring/rules/rabbitmq/file-descriptors-near-limit.yml

@@ -12,9 +12,9 @@ spec:
     rules:
     - alert: FileDescriptorsNearLimit
       expr: |
-        sum by(namespace, rabbitmq_cluster, pod, rabbitmq_node) (max_over_time(rabbitmq_process_open_fds[5m]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node, pod) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster))
+        sum by(namespace, rabbitmq_cluster, pod, rabbitmq_node) (max_over_time(rabbitmq_process_open_fds[5m]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) max by (instance, rabbitmq_node, rabbitmq_cluster) (rabbitmq_identity_info))
         /
-        sum by(namespace, rabbitmq_cluster, pod, rabbitmq_node) (rabbitmq_process_max_fds  * on(instance) group_left(rabbitmq_cluster, rabbitmq_node, pod) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster))
+        sum by(namespace, rabbitmq_cluster, pod, rabbitmq_node) (rabbitmq_process_max_fds * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) max by (instance, rabbitmq_node, rabbitmq_cluster) (rabbitmq_identity_info))
         > 0.8
       for: 10m
       annotations:

operators/rabbitmq-system/monitoring/rules/rabbitmq/high-connection-churn.yml

@@ -13,15 +13,27 @@ spec:
     - alert: HighConnectionChurn
       expr: |
         (
-          sum(rate(rabbitmq_connections_closed_total[5m]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)) by(namespace, rabbitmq_cluster)
+          sum by (namespace, rabbitmq_cluster) (
+            rate(rabbitmq_connections_closed_total[5m])
+            * on (instance) group_left (rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
+          )
           +
-          sum(rate(rabbitmq_connections_opened_total[5m]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)) by(namespace, rabbitmq_cluster)
+          sum by (namespace, rabbitmq_cluster) (
+            rate(rabbitmq_connections_opened_total[5m])
+            * on (instance) group_left (rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
+          )
         )
         /
-        sum (rabbitmq_connections * on(instance) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)) by (namespace, rabbitmq_cluster)
+        sum by (namespace, rabbitmq_cluster) (
+          rabbitmq_connections
+          * on (instance) group_left (rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
+        )
         > 0.1
         unless
-        sum (rabbitmq_connections * on(instance) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)) by (namespace, rabbitmq_cluster)
+        sum by (namespace, rabbitmq_cluster) (
+          rabbitmq_connections
+          * on (instance) group_left (rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)
+        )
         < 100
       for: 10m
       annotations:

operators/rabbitmq-system/monitoring/rules/rabbitmq/insufficient-established-erlang-distribution-links.yml

@@ -13,11 +13,11 @@ spec:
     - alert: InsufficientEstablishedErlangDistributionLinks
       # erlang_vm_dist_node_state: 1=pending, 2=up_pending, 3=up
       expr: |
-        count by (namespace, rabbitmq_cluster) (erlang_vm_dist_node_state * on(instance) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster) == 3)
+        count by (namespace, rabbitmq_cluster) (erlang_vm_dist_node_state * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info) == 3)
         <
-         count by (namespace, rabbitmq_cluster) (rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster))
+         count by (namespace, rabbitmq_cluster) (rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info))
          *
-        (count by (namespace, rabbitmq_cluster) (rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster) max(rabbitmq_identity_info) by (namespace, pod, container, rabbitmq_cluster)) -1 )
+        (count by (namespace, rabbitmq_cluster) (rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster) max by (instance, rabbitmq_cluster) (rabbitmq_identity_info)) -1 )
       for: 10m
       annotations:
         description: |