Add storage method to fetch all timestamps in one query (#3518)

* Add storage method to fetch all timestamps in one query

* Fix missing covered line

* Order by -last_modified by default

Author: leplatrem

kinto/core/storage/__init__.py

@@ -87,6 +87,21 @@ def resource_timestamp(self, resource_name, parent_id):
         """
         raise NotImplementedError
 
+    def all_resources_timestamps(self, resource_name):
+        """Get the highest timestamp of every objects in this `resource_name` for
+        each `parent_id`.
+
+        .. note::
+
+            This should take deleted objects into account.
+
+        :param str resource_name: the resource name.
+
+        :returns: the latest timestamp of the resource by `parent_id`.
+        :rtype: dict[str, int]
+        """
+        raise NotImplementedError
+
     def create(
         self,
         resource_name,

kinto/core/storage/memory.py

@@ -153,6 +153,10 @@ def resource_timestamp(self, resource_name, parent_id):
             raise exceptions.ReadonlyError(message=error_msg)
         return self.bump_and_store_timestamp(resource_name, parent_id)
 
+    @synchronized
+    def all_resources_timestamps(self, resource_name):
+        return {k: v[resource_name] for k, v in self._timestamps.items() if resource_name in v}
+
     def bump_and_store_timestamp(
         self, resource_name, parent_id, obj=None, modified_field=None, last_modified=None
     ):

kinto/core/storage/postgresql/__init__.py

@@ -247,6 +247,36 @@ def resource_timestamp(self, resource_name, parent_id):
 
         return obj.last_epoch
 
+    def all_resources_timestamps(self, resource_name):
+        query = """
+        WITH existing_timestamps AS (
+          -- Timestamp of latest object by parent_id.
+          (
+            SELECT parent_id, MAX(last_modified) AS last_modified
+            FROM objects
+            WHERE resource_name = :resource_name
+            GROUP BY parent_id
+          )
+          -- Timestamp of resources without sub-objects.
+          UNION
+          (
+            SELECT parent_id, last_modified
+            FROM timestamps
+            WHERE resource_name = :resource_name
+          )
+        )
+        SELECT parent_id, MAX(as_epoch(last_modified)) AS last_modified
+          FROM existing_timestamps
+          GROUP BY parent_id
+          ORDER BY last_modified DESC
+        """
+        with self.client.connect(readonly=True) as conn:
+            result = conn.execute(sa.text(query), dict(resource_name=resource_name))
+            rows = result.fetchmany(self._max_fetch_size + 1)
+
+        results = {r[0]: r[1] for r in rows}
+        return results
+
     @deprecate_kwargs({"collection_id": "resource_name", "record": "obj"})
     def create(
         self,

kinto/core/storage/testing.py

@@ -783,6 +783,39 @@ def test_timestamp_are_incremented_on_delete(self):
         after = self.storage.resource_timestamp(**self.storage_kw)
         self.assertTrue(before < after)
 
+    def test_all_timestamps_by_parent_id(self):
+        self.storage.create(obj={"id": "main"}, resource_name="bucket", parent_id="")
+        self.storage.create(obj={"id": "cid1"}, resource_name="collection", parent_id="/main")
+        self.storage.create(obj={"id": "cid2"}, resource_name="collection", parent_id="/main")
+        self.storage.create(obj={}, resource_name="record", parent_id="/main/cid2")
+        self.storage.create(obj={}, resource_name="record", parent_id="/main/cid2")
+
+        self.assertEqual(
+            {
+                "": self.storage.resource_timestamp(resource_name="bucket", parent_id=""),
+            },
+            self.storage.all_resources_timestamps(resource_name="bucket"),
+        )
+        self.assertEqual(
+            {
+                "/main": self.storage.resource_timestamp(
+                    resource_name="collection", parent_id="/main"
+                ),
+            },
+            self.storage.all_resources_timestamps(resource_name="collection"),
+        )
+        self.assertEqual(
+            {
+                "/main/cid1": self.storage.resource_timestamp(
+                    resource_name="record", parent_id="/main/cid1"
+                ),
+                "/main/cid2": self.storage.resource_timestamp(
+                    resource_name="record", parent_id="/main/cid2"
+                ),
+            },
+            self.storage.all_resources_timestamps(resource_name="record"),
+        )
+
     @skip_if_ci
     def test_timestamps_are_unique(self):  # pragma: no cover
         obtained = []

tests/core/test_storage.py

@@ -66,6 +66,7 @@ def test_mandatory_overrides(self):
             (self.storage.initialize_schema,),
             (self.storage.flush,),
             (self.storage.resource_timestamp, "", ""),
+            (self.storage.all_resources_timestamps, ""),
             (self.storage.create, "", "", {}),
             (self.storage.get, "", "", ""),
             (self.storage.update, "", "", "", {}),