feat(crons): Use timeout_at for timed out check-ins (#52570)

Queries off of timestamp of `timeout_at` for in progress check-ins

Author: rjo100

src/sentry/monitors/tasks.py

@@ -1,5 +1,4 @@
 import logging
-from datetime import timedelta
 
 from django.utils import timezone
 
@@ -102,21 +101,13 @@ def check_monitors(current_datetime=None):
         except Exception:
             logger.exception("Exception in check_monitors - mark missed")
 
-    qs = MonitorCheckIn.objects.filter(status=CheckInStatus.IN_PROGRESS).select_related(
-        "monitor", "monitor_environment"
-    )[:CHECKINS_LIMIT]
+    qs = MonitorCheckIn.objects.filter(
+        status=CheckInStatus.IN_PROGRESS, timeout_at__lte=current_datetime
+    ).select_related("monitor", "monitor_environment")[:CHECKINS_LIMIT]
     metrics.gauge("sentry.monitors.tasks.check_monitors.timeout_count", qs.count())
     # check for any monitors which are still running and have exceeded their maximum runtime
     for checkin in qs:
         try:
-            timeout = timedelta(
-                minutes=(checkin.monitor.config or {}).get("max_runtime") or TIMEOUT
-            )
-            # Check against date_updated to allow monitors to run for longer as
-            # long as they continue to send heart beats updating the checkin
-            if checkin.date_updated > current_datetime - timeout:
-                continue
-
             monitor_environment = checkin.monitor_environment
             logger.info(
                 "monitor_environment.checkin-timeout",
@@ -136,7 +127,16 @@ def check_monitors(current_datetime=None):
             if not has_newer_result:
                 monitor_environment.mark_failed(
                     reason=MonitorFailure.DURATION,
-                    occurrence_context={"duration": (timeout.seconds // 60) % 60},
+                    occurrence_context={
+                        "duration": (checkin.monitor.config or {}).get("max_runtime") or TIMEOUT
+                    },
                 )
         except Exception:
             logger.exception("Exception in check_monitors - mark timeout")
+
+    # safety check for check-ins stuck in the backlog
+    backlog_count = MonitorCheckIn.objects.filter(
+        status=CheckInStatus.IN_PROGRESS, timeout_at__isnull=True
+    ).count()
+    if backlog_count:
+        logger.exception(f"Exception in check_monitors - backlog count {backlog_count} is > 0")

tests/sentry/monitors/test_tasks.py

@@ -206,6 +206,7 @@ def test_timeout_with_no_future_complete_checkin(self):
             status=CheckInStatus.IN_PROGRESS,
             date_added=check_in_24hr_ago,
             date_updated=check_in_24hr_ago,
+            timeout_at=check_in_24hr_ago + timedelta(minutes=30),
         )
         # We started another checkin right now
         checkin2 = MonitorCheckIn.objects.create(
@@ -215,6 +216,7 @@ def test_timeout_with_no_future_complete_checkin(self):
             status=CheckInStatus.IN_PROGRESS,
             date_added=next_checkin_ts,
             date_updated=next_checkin_ts,
+            timeout_at=next_checkin_ts + timedelta(minutes=30),
         )
 
         assert checkin1.date_added == checkin1.date_updated == check_in_24hr_ago
@@ -268,6 +270,7 @@ def test_timeout_with_future_complete_checkin(self):
             status=CheckInStatus.IN_PROGRESS,
             date_added=check_in_24hr_ago,
             date_updated=check_in_24hr_ago,
+            timeout_at=check_in_24hr_ago + timedelta(minutes=30),
         )
         checkin2 = MonitorCheckIn.objects.create(
             monitor=monitor,
@@ -276,6 +279,7 @@ def test_timeout_with_future_complete_checkin(self):
             status=CheckInStatus.OK,
             date_added=next_checkin_ts,
             date_updated=next_checkin_ts,
+            timeout_at=next_checkin_ts + timedelta(minutes=30),
         )
 
         assert checkin1.date_added == checkin1.date_updated == check_in_24hr_ago
@@ -321,6 +325,7 @@ def test_timeout_via_max_runtime_configuration(self):
             status=CheckInStatus.IN_PROGRESS,
             date_added=next_checkin_ts,
             date_updated=next_checkin_ts,
+            timeout_at=next_checkin_ts + timedelta(minutes=60),
         )
 
         assert checkin.date_added == checkin.date_updated == next_checkin_ts
@@ -424,6 +429,7 @@ def test_timeout_exception_handling(self, logger):
             status=CheckInStatus.IN_PROGRESS,
             date_added=check_in_24hr_ago,
             date_updated=check_in_24hr_ago,
+            timeout_at=check_in_24hr_ago + timedelta(minutes=30),
         )
 
         # This monitor will be fine
@@ -448,6 +454,7 @@ def test_timeout_exception_handling(self, logger):
             status=CheckInStatus.IN_PROGRESS,
             date_added=check_in_24hr_ago,
             date_updated=check_in_24hr_ago,
+            timeout_at=check_in_24hr_ago + timedelta(minutes=30),
         )
         checkin2 = MonitorCheckIn.objects.create(
             monitor=monitor,
@@ -456,6 +463,7 @@ def test_timeout_exception_handling(self, logger):
             status=CheckInStatus.IN_PROGRESS,
             date_added=next_checkin_ts,
             date_updated=next_checkin_ts,
+            timeout_at=next_checkin_ts + timedelta(minutes=30),
         )
 
         assert checkin1.date_added == checkin1.date_updated == check_in_24hr_ago