Merge pull request #2543 from getsentry/log-digest-state-error

Catch and log digest state exceptions.

Author: ted kaemming

src/sentry/digests/backends/base.py

@@ -17,6 +17,13 @@ def load(options):
 }
 
 
+class InvalidState(Exception):
+    """
+    An error that is raised when an action cannot be performed on a
+    timeline in it's current state.
+    """
+
+
 class Backend(object):
     """
     A digest backend coordinates the addition of records to timelines, as well

src/sentry/digests/backends/redis.py

@@ -16,7 +16,10 @@
     Record,
     ScheduleEntry,
 )
-from sentry.digests.backends.base import Backend
+from sentry.digests.backends.base import (
+    Backend,
+    InvalidState,
+)
 from sentry.utils.cache import Lock
 from sentry.utils.redis import (
     check_cluster_versions,
@@ -398,8 +401,12 @@ def digest(self, key, minimum_delay=None):
         connection = self.cluster.get_local_client_for_key(timeline_key)
 
         with Lock(timeline_key, nowait=True, timeout=30):
+            # Check to ensure the timeline is in the correct state ("ready")
+            # before sending. This acts as a throttling mechanism to prevent
+            # sending a digest before it's next scheduled delivery time in a
+            # race condition scenario.
             if connection.zscore(make_schedule_key(self.namespace, SCHEDULE_STATE_READY), key) is None:
-                raise Exception('Cannot digest timeline, timeline is not in the ready state.')
+                raise InvalidState('Timeline is not in the ready state.')
 
             with connection.pipeline() as pipeline:
                 pipeline.watch(digest_key)  # This shouldn't be necessary, but better safe than sorry?

src/sentry/tasks/digests.py

@@ -4,6 +4,7 @@
 import time
 
 from sentry.digests import get_option_key
+from sentry.digests.backends.base import InvalidState
 from sentry.digests.notifications import (
     build_digest,
     split_key,
@@ -58,8 +59,13 @@ def deliver_digest(key, schedule_timestamp=None):
         project,
         get_option_key(plugin.get_conf_key(), 'minimum_delay')
     )
-    with digests.digest(key, minimum_delay=minimum_delay) as records:
-        digest = build_digest(project, records)
+
+    try:
+        with digests.digest(key, minimum_delay=minimum_delay) as records:
+            digest = build_digest(project, records)
+    except InvalidState as error:
+        logger.info('Skipped digest delivery: %s', error, exc_info=True)
+        return
 
     if digest:
         plugin.notify_digest(project, digest)