Support paginated methods in async client (#1615)

* Support paginated methods in async client

To be able to support both sync and async in paginated methods, we need
to push the pagination logic to the client so that the coroutine
handling can be handled there and the top level method retains the lower
level returned type. This refactors it a bit to be able to do that.

* Less duplication

* Remove some more duplication

* Remove specific method

* Don't pass parameters that breaks data handling

Author: therve

.generator/src/generator/templates/api.j2

@@ -204,17 +204,19 @@ class {{ classname }}:
         local_page_size = get_attribute_from_path(kwargs, "{{ pagination.limitParam|attribute_path }}", {{ get_default(operation, pagination.limitParam) }})
         endpoint = self._{{ operation.operationId|safe_snake_case }}_endpoint
         set_attribute_from_path(kwargs, "{{ pagination.limitParam|attribute_path }}", local_page_size, endpoint.params_map)
-        while True:
-            response = endpoint.call_with_http_info(**kwargs)
-            for item in get_attribute_from_path(response, "{{ pagination.resultsPath|attribute_path }}"):
-                yield item
-            if len(get_attribute_from_path(response, "{{ pagination.resultsPath|attribute_path }}")) < local_page_size:
-                break
-            {%- if pagination.pageOffsetParam %}
-            set_attribute_from_path(kwargs, "{{ pagination.pageOffsetParam|attribute_path }}", get_attribute_from_path(kwargs, "{{ pagination.pageOffsetParam|attribute_path }}", 0) + local_page_size, endpoint.params_map)
-            {%- endif %}
+        pagination = {
+            "limit_value": local_page_size,
+            "results_path": "{{ pagination.resultsPath|attribute_path }}",
             {%- if pagination.cursorParam %}
-            set_attribute_from_path(kwargs, "{{ pagination.cursorParam|attribute_path }}", get_attribute_from_path(response, "{{ pagination.cursorPath }}"), endpoint.params_map)
+            "cursor_param": "{{ pagination.cursorParam|attribute_path }}",
+            "cursor_path": "{{ pagination.cursorPath }}",
+            {%- endif %}
+            {%- if pagination.pageOffsetParam %}
+            "page_offset_param": "{{ pagination.pageOffsetParam|attribute_path }}",
             {%- endif %}
+            "endpoint": endpoint,
+            "kwargs": kwargs,
+        }
+        return endpoint.call_with_http_info_paginated(pagination)
     {%- endif %}
 {% endfor %}

.generator/src/generator/templates/api_client.j2

@@ -28,8 +28,9 @@ from {{ package }}.model_utils import (
     deserialize_file,
     file_type,
     model_to_dict,
-    none_type,
     validate_and_convert_types,
+    get_attribute_from_path,
+    set_attribute_from_path,
 )
 
 
@@ -335,6 +336,61 @@ class ApiClient:
             check_type,
         )
 
+    def call_api_paginated(
+        self,
+        resource_path: str,
+        method: str,
+        pagination: dict,
+        response_type: Optional[Tuple[Any]] = None,
+        request_timeout: Optional[Union[int, float, Tuple[Union[int, float], Union[int, float]]]] = None,
+        host: Optional[str] = None,
+        check_type: Optional[bool] = None,
+    ):
+        params = pagination["endpoint"].gather_params(pagination["kwargs"])
+        while True:
+            response = self.call_api(
+                resource_path,
+                method,
+                params["path"],
+                params["query"],
+                params["header"],
+                body=params["body"],
+                post_params=params["form"],
+                files=params["file"],
+                response_type=response_type,
+                check_type=check_type,
+                return_http_data_only=True,
+                preload_content=True,
+                request_timeout=request_timeout,
+                host=host,
+                collection_formats=params["collection_format"],
+            )
+            for item in get_attribute_from_path(response, pagination["results_path"]):
+                yield item
+            if len(get_attribute_from_path(response, pagination["results_path"])) < pagination["limit_value"]:
+                break
+
+            params = self._update_paginated_params(pagination, response)
+
+    def _update_paginated_params(self, pagination, response):
+        if "page_offset_param" in pagination:
+            set_attribute_from_path(
+                pagination["kwargs"],
+                pagination["page_offset_param"],
+                get_attribute_from_path(pagination["kwargs"], pagination["page_offset_param"], 0)
+                + pagination["limit_value"],
+                pagination["endpoint"].params_map,
+            )
+        else:
+            set_attribute_from_path(
+                pagination["kwargs"],
+                pagination["cursor_param"],
+                get_attribute_from_path(response, pagination["cursor_path"]),
+                pagination["endpoint"].params_map,
+            )
+
+        return pagination["endpoint"].gather_params(pagination["kwargs"])
+
     def parameters_to_tuples(self, params, collection_formats) -> List[Tuple[str, Any]]:
         """Get parameters as list of tuples, formatting collections.
 
@@ -550,6 +606,42 @@ class AsyncApiClient(ApiClient):
             return return_data
         return (return_data, response.status_code, response.headers)
 
+    async def call_api_paginated(
+        self,
+        resource_path: str,
+        method: str,
+        pagination: dict,
+        response_type: Optional[Tuple[Any]] = None,
+        request_timeout: Optional[Union[int, float, Tuple[Union[int, float], Union[int, float]]]] = None,
+        host: Optional[str] = None,
+        check_type: Optional[bool] = None,
+    ):
+        params = pagination["endpoint"].get_pagination_params(pagination["kwargs"])
+        while True:
+            response = await self.call_api(
+                resource_path,
+                method,
+                params["path"],
+                params["query"],
+                params["header"],
+                body=params["body"],
+                post_params=params["form"],
+                files=params["file"],
+                response_type=response_type,
+                check_type=check_type,
+                return_http_data_only=True,
+                preload_content=True,
+                request_timeout=request_timeout,
+                host=host,
+                collection_formats=params["collection_format"],
+            )
+            for item in get_attribute_from_path(response, pagination["results_path"]):
+                yield item
+            if len(get_attribute_from_path(response, pagination["results_path"])) < pagination["limit_value"]:
+                break
+
+            params = self._update_paginated_params(pagination, response)
+
 
 class Endpoint:
     def __init__(
@@ -617,7 +709,7 @@ class Endpoint:
             )
             kwargs[key] = fixed_val
 
-    def _gather_params(self, kwargs):
+    def gather_params(self, kwargs):
         params = {"body": None, "collection_format": {}, "file": {}, "form": [], "header": {}, "path": {}, "query": []}
 
         for param_name, param_value in kwargs.items():
@@ -645,10 +737,20 @@ class Endpoint:
                 if collection_format:
                     params["collection_format"][base_name] = collection_format
 
-        return params
+        accept_headers_list = self.headers_map["accept"]
+        if accept_headers_list:
+            params["header"]["Accept"] = self.api_client.select_header_accept(accept_headers_list)
 
-    def call_with_http_info(self, **kwargs):
+        content_type_headers_list = self.headers_map.get("content_type")
+        if content_type_headers_list:
+            header_list = self.api_client.select_header_content_type(content_type_headers_list)
+            params["header"]["Content-Type"] = header_list
+
+        self.update_params_for_auth(params["header"], params["query"])
 
+        return params
+
+    def _validate_and_get_host(self, kwargs):
         is_unstable = self.api_client.configuration.unstable_operations.get(
             "{}.{}".format(self.settings["version"], self.settings["operation_id"])
         )
@@ -699,18 +801,12 @@ class Endpoint:
 
         self._validate_inputs(kwargs)
 
-        params = self._gather_params(kwargs)
+        return host
 
-        accept_headers_list = self.headers_map["accept"]
-        if accept_headers_list:
-            params["header"]["Accept"] = self.api_client.select_header_accept(accept_headers_list)
-
-        content_type_headers_list = self.headers_map.get("content_type")
-        if content_type_headers_list:
-            header_list = self.api_client.select_header_content_type(content_type_headers_list)
-            params["header"]["Content-Type"] = header_list
+    def call_with_http_info(self, **kwargs):
+        host = self._validate_and_get_host(kwargs)
 
-        self.update_params_for_auth(params["header"], params["query"])
+        params = self.gather_params(kwargs)
 
         return self.api_client.call_api(
             self.settings["endpoint_path"],
@@ -730,6 +826,19 @@ class Endpoint:
             collection_formats=params["collection_format"],
         )
 
+    def call_with_http_info_paginated(self, pagination):
+        host = self._validate_and_get_host(pagination["kwargs"])
+
+        return self.api_client.call_api_paginated(
+            self.settings["endpoint_path"],
+            self.settings["http_method"],
+            response_type=self.settings["response_type"],
+            check_type=self.api_client.configuration.check_return_type,
+            request_timeout=self.api_client.configuration.request_timeout,
+            host=host,
+            pagination=pagination
+        )
+
     def update_params_for_auth(self, headers, queries) -> None:
         """Updates header and query params based on authentication setting.