[Confluence] Add _get_paged function for internal handling of paged APIs. (#937)

* Add `_get_paged` function for internal handling of paged APIs.

* Pin black to old version because 2.7 support is removed in LATEST version. An update will result in a lot of changes in the code.

Author: Spacetown

atlassian/confluence.py

@@ -45,6 +45,46 @@ def _create_body(body, representation):
 
         return {representation: {"value": body, "representation": representation}}
 
+    def _get_paged(self, url, params=None, data=None, flags=None, trailing=None, absolute=False):
+        """
+        Used to get the paged data
+
+        :param url: string:                        The url to retrieve
+        :param params: dict (default is None):     The parameters
+        :param data: dict (default is None):       The data
+        :param flags: string[] (default is None):  The flags
+        :param trailing: bool (default is None):   If True, a trailing slash is added to the url
+        :param absolute: bool (default is False):  If True, the url is used absolute and not relative to the root
+
+        :return: A generator object for the data elements
+        """
+
+        if params is None:
+            params = {}
+
+        while True:
+            response = self.get(url, trailing=trailing, params=params, data=data, flags=flags, absolute=absolute)
+            if "results" not in response:
+                return
+
+            for value in response.get("results", []):
+                yield value
+
+            # According to Cloud and Server documentation the links are returned the same way:
+            # https://developer.atlassian.com/cloud/confluence/rest/api-group-content/#api-wiki-rest-api-content-get
+            # https://developer.atlassian.com/server/confluence/pagination-in-the-rest-api/
+            url = response.get("_links", {}).get("next")
+            if url is None:
+                break
+            # From now on we have absolute URLs with parameters
+            absolute = True
+            # Params are now provided by the url
+            params = {}
+            # Trailing should not be added as it is already part of the url
+            trailing = False
+
+        return
+
     def page_exists(self, space, title):
         try:
             if self.get_page_by_title(space, title):
@@ -79,7 +119,13 @@ def get_page_child_by_type(self, page_id, type="page", start=None, limit=None, e
         log.info(url)
 
         try:
-            response = self.get(url, params=params)
+            if not self.advanced_mode and start is None and limit is None:
+                return self._get_paged(url, params=params)
+            else:
+                response = self.get(url, params=params)
+                if self.advanced_mode:
+                    return response
+                return response.get("results")
         except HTTPError as e:
             if e.response.status_code == 404:
                 # Raise ApiError as the documented reason is ambiguous
@@ -91,10 +137,6 @@ def get_page_child_by_type(self, page_id, type="page", start=None, limit=None, e
 
             raise
 
-        if self.advanced_mode:
-            return response
-        return response.get("results")
-
     def get_child_title_list(self, page_id, type="page", start=None, limit=None):
         """
         Find a list of Child title

requirements-dev.txt

@@ -4,7 +4,7 @@ pytest
 pytest-cov
 flake8
 flake8-no-fstring
-black
+black==21.12b0
 tox
 coverage
 codecov

tox.ini

@@ -34,7 +34,7 @@ commands = pylint {[base]linting_targets}
 [testenv:black]
 basepython = python3
 skip_install = true
-deps = black
+deps = black==21.12b0
 commands = black --check --diff {[base]linting_targets}
 
 [testenv:mypy]