refs(metric_alerts): Keep track of pending snapshot queue size (#21351)

We want to be able to track this so that we can alert if the queue grows too large.

Author: wedamija

src/sentry/incidents/tasks.py

@@ -204,6 +204,16 @@ def process_pending_incident_snapshots(next_id=None):
     """
     from sentry.incidents.logic import create_incident_snapshot
 
+    if next_id is None:
+        # When next_id is None we know we just started running the task. Take the count
+        # of total pending snapshots so that we can alert if we notice the queue
+        # constantly growing.
+        metrics.incr(
+            "incidents.pending_snapshots",
+            amount=PendingIncidentSnapshot.objects.count(),
+            sample_rate=1.0,
+        )
+
     now = timezone.now()
     pending_snapshots = PendingIncidentSnapshot.objects.filter(target_run_date__lte=now)
     if next_id is not None: