add rule for critical distributor inflight push request alert

Author: Tyler Reid

cortex-mixin/alerts/alerts.libsonnet

@@ -352,6 +352,30 @@
         },
       ],
     },
+    {
+      name: 'cortex_distributor_inflight_push_request_alert',
+      rules: [
+        {
+          alert: 'CortexDistributorReachingInflightPushRequestLimits',
+          expr: |||
+            (
+                (cortex_distributor_inflight_push_requests / ignoring(limit) cortex_distributor_instance_limits{limit="max_inflight_push_requests"})
+                and ignoring (limit)
+                (cortex_distributor_instance_limits{limit="max_inflight_push_requests"} > 0)
+            ) > 0.9
+          |||,
+          'for': '5m',
+          labels: {
+            severity: 'critical',
+          },
+          annotations: {
+            message: |||
+              Distributor {{ $labels.job }}/{{ $labels.instance }} has reached {{ $value | humanizePercentage }} of its series limit.
+            |||,
+          },
+        },
+      ],
+    },
     {
       name: 'cortex_wal_alerts',
       rules: [

cortex-mixin/docs/playbooks.md

@@ -108,6 +108,9 @@ How to **fix**:
 1. Ensure shuffle-sharding is enabled in the Cortex cluster
 1. Assuming shuffle-sharding is enabled, scaling up ingesters will lower the number of tenants per ingester. However, the effect of this change will be visible only after `-blocks-storage.tsdb.close-idle-tsdb-timeout` period so you may have to temporarily increase the limit
 
+### CortexDistributorReachingInflightPushRequestLimits
+  _TODO: this playbook has not been written yet._
+
 ### CortexRequestLatency
 
 This alert fires when a specific Cortex route is experiencing an high latency.