Fix race condition for schedule_changed() (#182)

* retry later in case of InterfaceError in schedule_changed()

* make logs less noisy

Author: rsichnyi

django_celery_beat/schedulers.py

@@ -40,7 +40,7 @@
 """
 
 logger = get_logger(__name__)
-debug, info = logger.debug, logger.info
+debug, info, warning = logger.debug, logger.info, logger.warning
 
 
 class ModelEntry(ScheduleEntry):
@@ -221,8 +221,9 @@ def all_as_schedule(self):
         return s
 
     def schedule_changed(self):
-        close_old_connections()
         try:
+            close_old_connections()
+
             # If MySQL is running with transaction isolation level
             # REPEATABLE-READ (default), then we won't see changes done by
             # other transactions until the current transaction is
@@ -236,6 +237,13 @@ def schedule_changed(self):
         except DatabaseError as exc:
             logger.exception('Database gave error: %r', exc)
             return False
+        except InterfaceError:
+            warning(
+                'DatabaseScheduler: InterfaceError in schedule_changed(), '
+                'waiting to retry in next call...'
+            )
+            return False
+
         try:
             if ts and ts > (last if last else ts):
                 return True
@@ -264,10 +272,16 @@ def sync(self):
                     _tried.add(name)
                 except (KeyError, ObjectDoesNotExist) as exc:
                     _failed.add(name)
-        except (DatabaseError, InterfaceError) as exc:
+        except DatabaseError as exc:
+            logger.exception('Database error while sync: %r', exc)
+        except InterfaceError:
+            warning(
+                'DatabaseScheduler: InterfaceError in sync(), '
+                'waiting to retry in next call...'
+            )
+        finally:
             # retry later, only for the failed ones
             self._dirty |= _failed
-            logger.exception('Database error while sync: %r', exc)
 
     def update_from_dict(self, mapping):
         s = {}