Merge pull request ceph#61634 from VallariAg/wip-vallari-nvme-maxgroup-alert

monitoring: add NVMeoFMaxGatewayGroups alert

Author: VallariAg

monitoring/ceph-mixin/config.libsonnet

@@ -10,6 +10,7 @@
     CephRBDMirrorImageTransferBandwidthThreshold: 0.8,
     CephRBDMirrorImagesPerDaemonThreshold: 100,
     NVMeoFMaxGatewaysPerGroup: 8,
+    NVMeoFMaxGatewayGroups: 4,
     NVMeoFMaxGatewaysPerCluster: 32,
     NVMeoFHighGatewayCPU: 80,
     NVMeoFMaxSubsystemsPerGateway: 128,

monitoring/ceph-mixin/prometheus_alerts.libsonnet

@@ -885,6 +885,16 @@
             description: 'You may create many gateways in a gateway group, but %(NVMeoFMaxGatewaysPerGroup)d is the tested limit' % $._config,
           },
         },
+        {
+          alert: 'NVMeoFMaxGatewayGroups',
+          'for': '1m',
+          expr: 'count(count by (group, cluster) (ceph_nvmeof_gateway_info)) by (cluster) > %.2f' % [$._config.NVMeoFMaxGatewayGroups],
+          labels: { severity: 'warning', type: 'ceph_default' },
+          annotations: {
+            summary: 'Max gateway groups exceeded%(cluster)s' % $.MultiClusterSummary(),
+            description: 'You may create many gateway groups, but %(NVMeoFMaxGatewayGroups)d is the tested limit' % $._config,
+          },
+        },
         {
           alert: 'NVMeoFSingleGatewayGroup',
           'for': '5m',

monitoring/ceph-mixin/prometheus_alerts.yml

@@ -792,6 +792,15 @@ groups:
         labels:
           severity: "warning"
           type: "ceph_default"
+      - alert: "NVMeoFMaxGatewayGroups"
+        annotations:
+          description: "You may create many gateway groups, but 4 is the tested limit"
+          summary: "Max gateway groups exceeded on cluster {{ $labels.cluster }}"
+        expr: "count(count by (group, cluster) (ceph_nvmeof_gateway_info)) by (cluster) > 4.00"
+        for: "1m"
+        labels:
+          severity: "warning"
+          type: "ceph_default"
       - alert: "NVMeoFSingleGatewayGroup"
         annotations:
           description: "Although a single member gateway group is valid, it should only be used for test purposes"

monitoring/ceph-mixin/tests_alerts/test_alerts.yml

@@ -2450,6 +2450,38 @@ tests:
           summary: "Max gateways within a gateway group (group-1) exceeded on cluster mycluster"
           description: "You may create many gateways in a gateway group, but 8 is the tested limit"
 
+
+ # NVMeoFMaxGatewayGroups
+ - interval: 1m
+   input_series:
+    - series: 'ceph_nvmeof_gateway_info{group="group-1",addr="1.1.1.1",cluster="mycluster"}'
+      values: '1+0x20'
+    - series: 'ceph_nvmeof_gateway_info{group="group-2",addr="1.1.1.2",cluster="mycluster"}'
+      values: '1+0x20'
+    - series: 'ceph_nvmeof_gateway_info{group="group-3",addr="1.1.1.3",cluster="mycluster"}'
+      values: '1+0x20'
+    - series: 'ceph_nvmeof_gateway_info{group="group-4",addr="1.1.1.9",cluster="mycluster"}'
+      values: '1+0x20'
+    - series: 'ceph_nvmeof_gateway_info{group="group-5",addr="1.1.1.12",cluster="mycluster"}'
+      values: '1+0x20'
+   promql_expr_test:
+     - expr: count(count by (group, cluster) (ceph_nvmeof_gateway_info)) by (cluster) > 4.00
+       eval_time: 1m
+       exp_samples:
+         - labels: '{cluster="mycluster"}'
+           value: 5
+   alert_rule_test:
+    - eval_time: 5m
+      alertname: NVMeoFMaxGatewayGroups
+      exp_alerts:
+      - exp_labels:
+          severity: warning
+          cluster: mycluster
+          type: ceph_default
+        exp_annotations:
+          summary: "Max gateway groups exceeded on cluster mycluster"
+          description: "You may create many gateway groups, but 4 is the tested limit"
+
  # NVMeoFSingleGatewayGroup
  - interval: 1m
    input_series: