Merge pull request #212 from josephroque/marker-based-pagination

Limit/Offset and Marker Based pagination

Author: Jeff-Meadows

boxsdk/object/folder.py

@@ -9,6 +9,8 @@
 from boxsdk.object.group import Group
 from boxsdk.object.item import Item
 from boxsdk.object.user import User
+from boxsdk.pagination.limit_offset_based_object_collection import LimitOffsetBasedObjectCollection
+from boxsdk.pagination.marker_based_object_collection import MarkerBasedObjectCollection
 from boxsdk.util.api_call_decorator import api_call
 from boxsdk.util.text_enum import TextEnum
 
@@ -154,6 +156,69 @@ def get_items(self, limit, offset=0, fields=None):
         response = box_response.json()
         return [self.translator.translate(item['type'])(self._session, item['id'], item) for item in response['entries']]
 
+    @api_call
+    def get_items_limit_offset(self, limit=None, offset=0, fields=None):
+        """
+        Get the items in a folder using limit-offset paging.
+
+        :param limit:
+            The maximum number of items to return per page. If not specified, then will use the server-side default.
+        :type limit:
+            `int` or None
+        :param offset:
+            The index at which to start returning items.
+        :type offset:
+            `int`
+        :param fields:
+            List of fields to request.
+        :type fields:
+            `Iterable` of `unicode`
+        :returns:
+            An iterator of the items in the folder.
+        :rtype:
+            :class:`BoxObjectCollection`
+        """
+        return LimitOffsetBasedObjectCollection(
+            self.session,
+            self.get_url('items'),
+            limit=limit,
+            fields=fields,
+            offset=offset,
+            return_full_pages=False,
+        )
+
+    @api_call
+    def get_items_marker(self, limit=None, marker=None, fields=None):
+        """
+        Get the items in a folder using marker-based paging.
+
+        :param limit:
+            The maximum number of items to return per page. If not specified, then will use the server-side default.
+        :type limit:
+            `int` or None
+        :param marker:
+            The offset index to start paging from.
+        :type marker:
+            `str` or None
+        :param fields:
+            List of fields to request.
+        :type fields:
+            `Iterable` of `unicode`
+        :returns:
+            An iterator of the items in the folder.
+        :rtype:
+            :class:`BoxObjectCollection`
+        """
+        return MarkerBasedObjectCollection(
+            self.session,
+            self.get_url('items'),
+            limit=limit,
+            fields=fields,
+            marker=marker,
+            return_full_pages=False,
+            supports_limit_offset_paging=True,
+        )
+
     @api_call
     def upload_stream(
             self,

boxsdk/pagination/__init__.py

@@ -0,0 +1,3 @@
+# coding: utf-8
+
+from __future__ import unicode_literals

boxsdk/pagination/box_object_collection.py

@@ -0,0 +1,194 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+from abc import ABCMeta, abstractmethod
+import collections
+
+from six import add_metaclass
+
+from boxsdk.pagination.page import Page
+
+
+@add_metaclass(ABCMeta)
+class BoxObjectCollection(collections.Iterator, object):
+    """
+    An iterator that represents a collection of Box objects (BaseObject).
+
+    A BoxObjectCollection instance contains everything it needs in order to retrieve and page through
+    responses from Box API endpoints that return collections of Box objects.
+
+    This class only has two public methods:
+
+    1). next(), which returns either a Page (sequence of BaseObjects) or individual BaseObjects based on
+    the constructor argument 'return_full_pages'.
+
+    2). next_pointer(), which returns the pointer (either an offset or a marker, based on the endpoint) that
+    will be used to retrieve the next page of Box objects. This pointer can be used when requesting new
+    BoxObjectCollection instances that start off from a particular page, instead of from the very beginning.
+    """
+    def __init__(
+            self,
+            session,
+            url,
+            limit=None,
+            fields=None,
+            additional_params=None,
+            return_full_pages=False,
+    ):
+        """
+        :param session:
+            The Box session used to make requests.
+        :type session:
+            :class:`BoxSession`
+        :param url:
+            The endpoint url to hit.
+        :type url:
+            `unicode`
+        :param limit:
+            The number of entries for each page to return. The default, as well as the upper limit of this value,
+            differs by endpoint. See https://developer.box.com/reference. If limit is set to None, then the default
+            limit (returned by Box in the response) is used.
+        :type limit:
+            `int` or None
+        :param fields:
+            List of fields to request. If None, will return the default fields for the object.
+        :type fields:
+            `Iterable` of `unicode` or None
+        :param additional_params:
+            Additional HTTP params to send in the request.
+        :type additional_params:
+            `dict` or None
+        :param return_full_pages:
+            If True, then the returned iterator for this collection will return full pages of Box objects on each
+            call to next(). If False, the iterator will return a single Box object on each next() call.
+        :type return_full_pages:
+            `bool`
+        """
+        super(BoxObjectCollection, self).__init__()
+        self._session = session
+        self._url = url
+        self._limit = limit
+        self._fields = fields
+        self._additional_params = additional_params
+        self._return_full_pages = return_full_pages
+        self._has_retrieved_all_items = False
+        self._all_items = None
+
+    def next(self):
+        """
+        Returns either a Page (a Sequence of BaseObjects) or a BaseObject depending on self._return_full_pages.
+
+        Invoking this method may make an API call to Box. Any exceptions that can occur while making requests
+        may be raised in this method.
+
+        :rtype:
+            :class:`Page` or :class:`BaseObject`
+        """
+        if self._all_items is None:
+            self._all_items = self._items_generator()
+        return next(self._all_items)
+
+    __next__ = next
+
+    def _items_generator(self):
+        """
+        :rtype:
+            :class:`Page` or :class:`BaseObject`
+        """
+        while not self._has_retrieved_all_items:
+            response_object = self._load_next_page()
+
+            # If the limit was not specified, then it should default to whatever the server tells us.
+            if self._limit is None:
+                self._limit = response_object['limit']
+
+            self._update_pointer_to_next_page(response_object)
+            self._has_retrieved_all_items = not self._has_more_pages(response_object)
+            page = Page(self._session, response_object)
+
+            if self._return_full_pages:
+                yield page
+            else:
+                # It's possible for the Box API to return 0 items in a page, even if there are more items to be
+                # retrieved on subsequent pages. When self._return_full_pages is True, then yielding a 0-item
+                # page is fine because that's what the page returned.
+                # But when we are iterating over individual items, and not pages, it's odd to yield a sequence of
+                # Nones (for that page that had 0 items). So instead, we continue to request more pages until we
+                # have Box objects to yield.
+                if not page:
+                    continue
+                for entry in page:
+                    yield entry
+
+    def _load_next_page(self):
+        """
+        Request the next page of entries from Box. Raises any network-related exceptions, including BoxAPIException.
+        Returns a parsed dictionary of the JSON response from Box
+
+        :rtype:
+            `dict`
+        """
+        params = {}
+        if self._limit is not None:
+            params['limit'] = self._limit
+        if self._fields:
+            params['fields'] = ','.join(self._fields)
+        if self._additional_params:
+            params.update(self._additional_params)
+        params.update(self._next_page_pointer_params())
+        box_response = self._session.get(self._url, params=params)
+        return box_response.json()
+
+    @abstractmethod
+    def _update_pointer_to_next_page(self, response_object):
+        """
+        Update the internal pointer attribute of this class to what will be used to request the next page
+        of Box objects.
+
+        A "pointer" can either be a marker (for marker-based paging) or an offset (for limit-offset paging).
+
+        :param response_object:
+            The parsed HTTP response from Box after requesting more pages.
+        :type response_object:
+            `dict`
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _has_more_pages(self, response_object):
+        """
+        Are there more pages of entries to query Box for? This gets invoked after self._update_pointer_to_next_page().
+
+        :param response_object:
+            The parsed HTTP response from Box after requesting more pages.
+        :type response_object:
+            `dict`
+        :rtype:
+            `bool`
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _next_page_pointer_params(self):
+        """
+        The dict of HTTP params that specify which page of Box objects to retrieve.
+
+        :rtype:
+            `dict`
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def next_pointer(self):
+        """
+        The pointer that will be used to request the next page of Box objects.
+
+        For limit-offset based paging, this is an offset. For marker-based paging, this is a marker.
+
+        The pointer only gets progressed upon successful page requests to Box.
+
+        :rtype:
+            varies
+        """
+        raise NotImplementedError

boxsdk/pagination/limit_offset_based_object_collection.py

@@ -0,0 +1,83 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+from .box_object_collection import BoxObjectCollection
+
+
+class LimitOffsetBasedObjectCollection(BoxObjectCollection):
+    """
+    An iterator of Box objects (BaseObjects) that were retrieved from a Box API endpoint that supports
+    limit-offset type of pagination.
+
+    See https://developer.box.com/reference#pagination for more details.
+    """
+
+    def __init__(
+            self,
+            session,
+            url,
+            limit=None,
+            fields=None,
+            additional_params=None,
+            return_full_pages=False,
+            offset=0,
+    ):
+        """
+        :param offset:
+            The offset index to start paging from.
+        :type offset:
+            `int`
+        """
+        super(LimitOffsetBasedObjectCollection, self).__init__(
+            session,
+            url,
+            limit=limit,
+            fields=fields,
+            additional_params=additional_params,
+            return_full_pages=return_full_pages,
+        )
+        self._offset = offset
+
+    def _update_pointer_to_next_page(self, response_object):
+        """Baseclass override."""
+        total_count = response_object['total_count']
+
+        # The API might use a lower limit than the client asked for, if the
+        # client asked for a limit above the maximum limit for that endpoint.
+        # The API is supposed to respond with the limit that it actually used.
+        # If that is given, then use that limit for the offset calculation, and
+        # also for the remainder of the paging.
+        #
+        # Similarly, the API reports the offset that it used. In theory, this
+        # should always be the same as what was requested. But just in case, do
+        # the same thing with offset.
+        if 'limit' in response_object:
+            self._limit, old_limit = int(response_object['limit']), self._limit
+
+            # If the API erroneously sends a bad value for limit, we want to
+            # avoid getting into an infinite chain of API calls. So abort with
+            # a runtime error.
+            if self._limit <= 0 < old_limit:
+                self._offset = total_count  # Disable additional paging.
+                raise RuntimeError('API returned limit={0}, cannot continue paging'.format(self._limit))
+
+        if 'offset' in response_object:
+            self._offset = int(response_object['offset'])
+
+        if total_count >= self._offset + self._limit:
+            self._offset += self._limit
+        else:
+            self._offset = total_count
+
+    def _has_more_pages(self, response_object):
+        """Baseclass override."""
+        return self._offset < response_object['total_count']
+
+    def _next_page_pointer_params(self):
+        """Baseclass override."""
+        return {'offset': self._offset}
+
+    def next_pointer(self):
+        """Baseclass override."""
+        return self._offset