Merge pull request #118 from MannLabs/add_instrument_stall

add instrument stall alert

Author: mschwoer

monitoring/alert_manager.py

@@ -10,6 +10,7 @@
     DiskSpaceAlert,
     HealthCheckFailedAlert,
     InstrumentFilePileUpAlert,
+    InstrumentStallAlert,
     PumpPressureAlert,
     RawFileErrorAlert,
     S3UploadFailureAlert,
@@ -45,6 +46,7 @@ def __init__(self):
             HealthCheckFailedAlert(),
             StatusPileUpAlert(),
             InstrumentFilePileUpAlert(),
+            InstrumentStallAlert(),
             RawFileErrorAlert(),
             S3UploadFailureAlert(),
             WebAppHealthAlert(),

monitoring/alerts/__init__.py

@@ -4,6 +4,7 @@
 from .disk_space_alert import DiskSpaceAlert
 from .health_check_failed_alert import HealthCheckFailedAlert
 from .instrument_file_pile_up_alert import InstrumentFilePileUpAlert
+from .instrument_stall_alert import InstrumentStallAlert
 from .pump_pressure_alert import PumpPressureAlert
 from .raw_file_error_alert import RawFileErrorAlert
 from .s3_upload_failure_alert import S3UploadFailureAlert
@@ -16,6 +17,7 @@
     "DiskSpaceAlert",
     "HealthCheckFailedAlert",
     "InstrumentFilePileUpAlert",
+    "InstrumentStallAlert",
     "PumpPressureAlert",
     "RawFileErrorAlert",
     "S3UploadFailureAlert",

monitoring/alerts/config.py

@@ -37,6 +37,10 @@
 }
 INSTRUMENT_FILE_MIN_AGE_HOURS = 6  # Only consider files older than 6 hours
 
+# Instrument stall alert configuration
+INSTRUMENT_STALL_THRESHOLD_HOURS = 2  # How long without a new file before alerting
+INSTRUMENT_STALL_INSTRUMENT_IDS: set[str] = set()  # Instruments to monitor
+
 # Pump pressure alert configuration
 PUMP_PRESSURE_LOOKBACK_DAYS = 1
 PUMP_PRESSURE_WINDOW_SIZE = 5  # Number of samples to compare
@@ -75,3 +79,4 @@ class Cases:
     S3_UPLOAD_FAILURE = "s3_upload_failure"
     WEBAPP_HEALTH = "webapp_health"
     PUMP_PRESSURE_INCREASE = "pump_pressure_increase"
+    INSTRUMENT_STALL = "instrument_stall"

monitoring/alerts/instrument_stall_alert.py

@@ -0,0 +1,81 @@
+"""Instrument stall alert checker."""
+
+import logging
+from datetime import datetime, timedelta
+
+import pytz
+
+from shared.db.models import KrakenStatus, RawFile
+
+from .base_alert import BaseAlert
+from .config import (
+    INSTRUMENT_STALL_INSTRUMENT_IDS,
+    INSTRUMENT_STALL_THRESHOLD_HOURS,
+    Cases,
+)
+
+
+class InstrumentStallAlert(BaseAlert):
+    """Check if no new file has been acquired for a configured time period."""
+
+    def __init__(self):
+        """Initialize with empty alerted file names tracker."""
+        self._alerted_file_names: dict[str, str] = {}
+
+    @property
+    def name(self) -> str:
+        """Return the case name for this alert type."""
+        return Cases.INSTRUMENT_STALL
+
+    def _get_issues(
+        self, status_objects: list[KrakenStatus]
+    ) -> list[tuple[str, tuple[str, datetime]]]:
+        """Check for instruments with no recent file acquisition."""
+        del status_objects
+
+        if not INSTRUMENT_STALL_INSTRUMENT_IDS:
+            return []
+
+        now = datetime.now(pytz.UTC)
+        threshold = now - timedelta(hours=INSTRUMENT_STALL_THRESHOLD_HOURS)
+
+        issues = []
+        for instrument_id in INSTRUMENT_STALL_INSTRUMENT_IDS:
+            latest = (
+                RawFile.objects.filter(instrument_id=instrument_id)
+                .order_by("-created_at")
+                .only("original_name", "created_at")
+                .first()
+            )
+
+            if not latest:
+                logging.debug(f"No raw files found for instrument {instrument_id}")
+                continue
+
+            created_at = latest.created_at
+            if created_at.tzinfo is None:
+                created_at = pytz.utc.localize(created_at)
+
+            if created_at < threshold:
+                if self._alerted_file_names.get(instrument_id) == latest.original_name:
+                    continue
+                issues.append((instrument_id, (latest.original_name, created_at)))
+                self._alerted_file_names[instrument_id] = latest.original_name
+            else:
+                self._alerted_file_names.pop(instrument_id, None)
+
+        return issues
+
+    def format_message(self, issues: list[tuple[str, tuple[str, datetime]]]) -> str:
+        """Format no new file acquired message."""
+        now = datetime.now(pytz.UTC)
+        lines = []
+        for instrument_id, (original_name, created_at) in issues:
+            hours_ago = (now - created_at).total_seconds() / 3600
+            lines.append(
+                f'- `{instrument_id}`: last file "{original_name}" '
+                f"acquired at {created_at.strftime('%Y-%m-%d %H:%M:%S')} UTC "
+                f"({hours_ago:.1f} hours ago)"
+            )
+        instruments_str = "\n".join(lines)
+        return f"No new file acquired:\n{instruments_str}"