Make backlog processing instantaneous

This commit removes all the code which skips backlog manually, since I
discovered if you pass a negative offset to `getUpdates` all the old
messages are automatically discarded. Because of this new method, the
updates skipping process is now instantaneous.

Author: Pietro Albini

botogram/runner/processes.py

@@ -186,7 +186,7 @@ def loop(self):
             return
 
         try:
-            updates, backlog = self.fetcher.fetch()
+            updates = self.fetcher.fetch()
         except updates_module.AnotherInstanceRunningError:
             self.handle_another_instance()
             return
@@ -196,26 +196,17 @@ def loop(self):
             self.logger.debug("Exception content: %s" % str(e))
             return
 
-        if backlog:
-            if len(backlog) == 1:
-                self.logger.debug("Skipped update #%s because it's coming "
-                                  "from the backlog" % backlog[0].update_id)
-            else:
-                self.logger.debug("Skipped updates #%s to #%s because they're "
-                                  "coming from the backlog" % (
-                                      backlog[0].update_id,
-                                      backlog[-1].update_id
-                                  ))
-
-        if updates:
-            result = []
-            for update in updates:
-                update.set_api(None)
-                result.append(jobs.Job(self.bot_id, jobs.process_update, {
-                    "update": update,
-                }))
-
-            self.ipc.command("jobs.bulk_put", result)
+        if not updates:
+            return
+
+        result = []
+        for update in updates:
+            update.set_api(None)
+            result.append(jobs.Job(self.bot_id, jobs.process_update, {
+                "update": update,
+            }))
+
+        self.ipc.command("jobs.bulk_put", result)
 
     def handle_another_instance(self):
         """Code run when another instance of the bot is running"""

botogram/updates.py

@@ -6,8 +6,6 @@
     Released under the MIT license
 """
 
-import time
-
 from . import objects
 from . import api
 
@@ -31,7 +29,6 @@ class UpdatesFetcher:
     def __init__(self, bot):
         self._bot = bot
         self._last_id = -1
-        self._started_at = time.time()
         self._backlog_processed = False
 
         # Don't treat backlog as backlog if bot.process_backlog is True
@@ -55,6 +52,15 @@ def _fetch_updates(self, timeout):
 
     def fetch(self, timeout=1):
         """Fetch the latest updates"""
+        if not self._backlog_processed:
+            # Just erase all the previous messages
+            self._bot.api.call("getUpdates", {
+                "offset": -1,
+                "timeout": 0,
+            }, expect=objects.Updates)
+
+            self._backlog_processed = True
+
         updates = self._fetch_updates(timeout)
 
         # If there are no updates just ignore this block
@@ -63,59 +69,7 @@ def fetch(self, timeout=1):
         except IndexError:
             pass
 
-        if self._backlog_processed:
-            return updates, []
-
-        # Now start to filter backlog from messages to process
-        # This is faster than a plain check for every item with a for, and
-        # helps a lot if you have a *really* long backlog
-
-        if not updates:
-            self._backlog_processed = True
-            return [], []
-
-        to_check = len(updates) - 1
-        check_chunk = len(updates)
-        direction = -1  # -1 is backward, 1 is forward
-        last = 0
-
-        while True:
-            # Check if the current message to check is from the backlog
-            # Adjust the check direction accordingly
-            update = updates[to_check]
-            if update.message.date < self._started_at:
-                direction = 1
-            else:
-                direction = -1
-
-            # The next chunk to check is exactly half of the previous one
-            check_chunk = int(check_chunk / 2)
-            if check_chunk == 0:
-                check_chunk = 1
-
-            # If the last one was from the backlog, the current one is not and
-            # this is at the max precision (step of 1), then consider it as
-            # found
-            if direction == 1 and last == -1 and check_chunk == 1:
-                to_check += 1  # Return to the previous state
-                self._backlog_processed = True
-                break
-            last = direction
-
-            # Set the next update to check
-            to_check += direction * check_chunk
-
-            # If the next update to check is outside the bounds of the list
-            # just break the loop (and say the backlog is processed if no
-            # updates are from it)
-            if to_check < 0:
-                self._backlog_processed = True
-                break
-            elif to_check >= len(updates):
-                break
-
-        # The first is the updates to process, the second the backlog
-        return updates[to_check:], updates[:to_check]
+        return updates
 
     def block_until_alone(self, treshold=4, check_timeout=1, when_stop=None):
         """Returns when this one is the only instance of the bot"""

docs/changelog.rst

@@ -65,7 +65,8 @@ Changes
 Performance improvements
 ------------------------
 
-* Updates initial filtering and queueing performance improved
+* Updates queueing performance improved
+* Backlog processing is now instantaneous
 
 Deprecated features
 -------------------