Skip to content

Commit 74829d3

Browse files
api-clients-generation-pipeline[bot]therveci.datadog-api-spec
authored
Support pagination in Python (#957)
* Implement offset pagination * Progress * More progress * Remove default limit * Doc * Don't repeat desc * Fix rebase * Add test facilities * Remove tabs * Review * Don't reference application/json * Regenerate client from commit 65921537 of spec repo Co-authored-by: Thomas Hervé <[email protected]> Co-authored-by: api-clients-generation-pipeline[bot] <54105614+api-clients-generation-pipeline[bot]@users.noreply.github.com> Co-authored-by: ci.datadog-api-spec <[email protected]>
1 parent 492e2f7 commit 74829d3

21 files changed

+701
-63
lines changed

‎.apigentools-info

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@
44
"spec_versions": {
55
"v1": {
66
"apigentools_version": "1.6.2",
7-
"regenerated": "2022-04-19 14:40:07.070967",
8-
"spec_repo_commit": "86a023ae"
7+
"regenerated": "2022-04-19 15:28:15.502011",
8+
"spec_repo_commit": "65921537"
99
},
1010
"v2": {
1111
"apigentools_version": "1.6.2",
12-
"regenerated": "2022-04-19 14:40:07.085526",
13-
"spec_repo_commit": "86a023ae"
12+
"regenerated": "2022-04-19 15:28:15.516188",
13+
"spec_repo_commit": "65921537"
1414
}
1515
}
1616
}

‎.generator/src/generator/cli.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ def cli(specs, output):
3131
env.filters["camel_case"] = formatter.camel_case
3232
env.filters["collection_format"] = openapi.collection_format
3333
env.filters["format_value"] = formatter.format_value
34+
env.filters["attribute_path"] = formatter.attribute_path
3435
env.filters["parameter_schema"] = openapi.parameter_schema
3536
env.filters["parameters"] = openapi.parameters
3637
env.filters["return_type"] = openapi.return_type
@@ -51,6 +52,8 @@ def cli(specs, output):
5152
env.globals["get_oneof_types"] = openapi.get_oneof_types
5253
env.globals["get_oneof_models"] = openapi.get_oneof_models
5354
env.globals["type_to_python"] = openapi.type_to_python
55+
env.globals["get_default"] = openapi.get_default
56+
env.globals["get_type_at_path"] = openapi.get_type_at_path
5457

5558
api_j2 = env.get_template("api.j2")
5659
apis_j2 = env.get_template("apis.j2")

‎.generator/src/generator/formatter.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,7 @@ def format_value(value, quotes='"'):
4949
def get_name(schema):
5050
if hasattr(schema, "__reference__"):
5151
return schema.__reference__["$ref"].split("/")[-1]
52+
53+
54+
def attribute_path(attribute):
55+
return ".".join(attribute_name(a) for a in attribute.split("."))

‎.generator/src/generator/openapi.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -569,3 +569,34 @@ def request(self):
569569
return Schema(
570570
next(iter(self.spec["requestBody"]["content"].values()))["schema"]
571571
)
572+
573+
574+
def get_default(operation, attribute_path):
575+
attrs = attribute_path.split(".")
576+
for name, parameter in parameters(operation):
577+
if name == attrs[0]:
578+
break
579+
if name == attribute_path:
580+
# We found a top level attribute matching the full path, let's use the default
581+
return parameter["schema"]["default"]
582+
583+
if name == "body":
584+
parameter = next(iter(parameter["content"].values()))["schema"]
585+
for attr in attrs[1:]:
586+
parameter = parameter["properties"][attr]
587+
return parameter["default"]
588+
589+
590+
def get_type_at_path(operation, attribute_path):
591+
content = None
592+
for code, response in operation.get("responses", {}).items():
593+
if int(code) >= 300:
594+
continue
595+
for content in response.get("content", {}).values():
596+
if "schema" in content:
597+
break
598+
if content is None:
599+
raise RuntimeError("Default response not found")
600+
for attr in attribute_path.split("."):
601+
content = content["schema"]["properties"][attr]
602+
return get_type_for_items(content)

‎.generator/src/generator/templates/api.j2

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ from {{ package }}.api_client import ApiClient, Endpoint as _Endpoint
55
from {{ package }}.model_utils import (
66
date,
77
datetime,
8+
set_attribute_from_path,
9+
get_attribute_from_path,
810
file_type,
911
none_type,
1012
)
@@ -185,4 +187,59 @@ class {{ classname }}:
185187
kwargs["{{ name|attribute_name }}"] = {{ name|attribute_name }}
186188
{% endfor %}
187189
return self._{{ operation.operationId|safe_snake_case }}_endpoint.call_with_http_info(**kwargs)
190+
{%- if operation["x-pagination"] %}
191+
{%- set pagination = operation["x-pagination"] %}
192+
193+
def {{ operation.operationId|safe_snake_case }}_with_pagination(self, {% for name, parameter in operation|parameters if parameter.required %}{{name|attribute_name}}, {% endfor %}**kwargs):
194+
"""{{ operation.summary|indent(8) }}.
195+
196+
Provide a paginated version of :meth:`{{ operation.operationId|safe_snake_case }}`, returning all items.
197+
{# keep new line #}
198+
{%- for name, parameter in operation|parameters if parameter.required %}
199+
{%- if parameter.description %}
200+
:param {{ name|attribute_name }}: {{ parameter.description|indent(12) }}{% if parameter.default %} Defaults to {{ parameter.default }}.{% endif %}{% endif %}
201+
:type {{ name|attribute_name }}: {{ get_type_for_parameter(parameter) }}
202+
{%- endfor %}
203+
{%- for name, parameter in operation|parameters if not parameter.required %}
204+
{%- if parameter.description %}
205+
:param {{ name|attribute_name }}: {{ parameter.description|indent(12) }}{%- if parameter.default %} If omitted the server will use the default value of {{ parameter.default }}.{% endif %}{% endif %}
206+
:type {{ name|attribute_name }}: {{ get_type_for_parameter(parameter) }}, optional
207+
{%- endfor %}
208+
:param _request_timeout: Timeout setting for this request. If one
209+
number provided, it will be total request timeout. It can also be a
210+
pair (tuple) of (connection, read) timeouts. Default is None.
211+
:type _request_timeout: float/tuple
212+
:param _check_input_type: Specifies if type checking should be done one
213+
the data sent to the server. Default is True.
214+
:type _check_input_type: bool
215+
:param _check_return_type: Specifies if type checking should be done
216+
one the data received from the server. Default is True.
217+
:type _check_return_type: bool
218+
:param _host_index: Specifies the index of the server that we want to
219+
use. Default is read from the configuration.
220+
:type _host_index: int/None
221+
222+
:return: A generator of paginated results.
223+
:rtype: collections.abc.Iterable[{{ get_type_at_path(operation, pagination.resultsPath) }}]
224+
"""
225+
kwargs = self._{{ operation.operationId|safe_snake_case }}_endpoint.default_arguments(kwargs)
226+
{%- for name, parameter in operation|parameters if parameter.required %}
227+
kwargs["{{ name|attribute_name }}"] = {{ name|attribute_name }}
228+
{% endfor %}
229+
page_size = get_attribute_from_path(kwargs, "{{ pagination.limitParam|attribute_path }}", {{ get_default(operation, pagination.limitParam) }})
230+
endpoint = self._{{ operation.operationId|safe_snake_case }}_endpoint
231+
set_attribute_from_path(kwargs, "{{ pagination.limitParam|attribute_path }}", page_size, endpoint.params_map)
232+
while True:
233+
response = endpoint.call_with_http_info(**kwargs)
234+
for item in get_attribute_from_path(response, "{{ pagination.resultsPath|attribute_path }}"):
235+
yield item
236+
if len(get_attribute_from_path(response, "{{ pagination.resultsPath|attribute_path }}")) < page_size:
237+
break
238+
{%- if pagination.pageOffsetParam %}
239+
set_attribute_from_path(kwargs, "{{ pagination.pageOffsetParam|attribute_path }}", get_attribute_from_path(kwargs, "{{ pagination.pageOffsetParam|attribute_path }}", 0) + page_size, endpoint.params_map)
240+
{%- endif %}
241+
{%- if pagination.cursorParam %}
242+
set_attribute_from_path(kwargs, "{{ pagination.cursorParam|attribute_path }}", get_attribute_from_path(response, "{{ pagination.cursorPath }}"), endpoint.params_map)
243+
{%- endif %}
244+
{%- endif %}
188245
{% endfor %}

‎.generator/src/generator/templates/configuration.j2

Lines changed: 49 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -37,30 +37,30 @@ class _UnstableOperations:
3737
def __getitem__(self, key):
3838
if key in self.values:
3939
return self.values[key]
40-
for version in ("v1", "v2"):
41-
version_key = f"{version}.{key}"
42-
if version_key in self.values:
43-
return self.values[version_key]
40+
for version in ("v1", "v2"):
41+
version_key = f"{version}.{key}"
42+
if version_key in self.values:
43+
return self.values[version_key]
4444
raise KeyError(f"Unknown unstable operation {key}")
4545

4646
def __setitem__(self, key, value):
4747
if key in self.values:
4848
self.values[key] = value
49-
for version in ("v1", "v2"):
50-
version_key = f"{version}.{key}"
51-
if version_key in self.values:
52-
self.values[version_key] = value
53-
break
54-
else:
55-
raise KeyError(f"Unknown unstable operation {key}")
49+
for version in ("v1", "v2"):
50+
version_key = f"{version}.{key}"
51+
if version_key in self.values:
52+
self.values[version_key] = value
53+
break
54+
else:
55+
raise KeyError(f"Unknown unstable operation {key}")
5656

5757
def __contains__(self, key):
5858
if key in self.values:
5959
return True
60-
for version in ("v1", "v2"):
61-
version_key = f"{version}.{key}"
62-
if version_key in self.values:
63-
return True
60+
for version in ("v1", "v2"):
61+
version_key = f"{version}.{key}"
62+
if version_key in self.values:
63+
return True
6464
return False
6565

6666

@@ -435,43 +435,43 @@ class Configuration:
435435
auth = {}
436436
{%- for name, schema in specs.v2.components.securitySchemes.items() %}
437437
{%- if schema.type == "apiKey" %}
438-
if "{{name}}" in self.api_key{% if "x-auth-id-alias" in schema %} or "{{ schema["x-auth-id-alias"] }}" in self.api_key{% endif %}:
439-
auth["{{name}}"] = {
440-
"type": "api_key",
441-
"in": "{{ schema.in }}",
442-
"key": "{{ schema.name }}",
443-
"value": self.get_api_key_with_prefix(
444-
"{{name}}",{% if "x-auth-id-alias" in schema %}
445-
alias="{{ schema["x-auth-id-alias"] }}",{%- endif %}
446-
),
447-
}
438+
if "{{name}}" in self.api_key{% if "x-auth-id-alias" in schema %} or "{{ schema["x-auth-id-alias"] }}" in self.api_key{% endif %}:
439+
auth["{{name}}"] = {
440+
"type": "api_key",
441+
"in": "{{ schema.in }}",
442+
"key": "{{ schema.name }}",
443+
"value": self.get_api_key_with_prefix(
444+
"{{name}}",{% if "x-auth-id-alias" in schema %}
445+
alias="{{ schema["x-auth-id-alias"] }}",{%- endif %}
446+
),
447+
}
448448
{%- elif schema.type == "http" and schema.scheme == "basic" %}
449-
if self.username is not None and self.password is not None:
450-
auth["{{name}}"] = {
451-
"type": "basic",
452-
"in": "header",
453-
"key": "Authorization",
454-
"value": self.get_basic_auth_token()
455-
}
449+
if self.username is not None and self.password is not None:
450+
auth["{{name}}"] = {
451+
"type": "basic",
452+
"in": "header",
453+
"key": "Authorization",
454+
"value": self.get_basic_auth_token()
455+
}
456456
{%- elif schema.type == "http" and schema.scheme == "bearer" %}
457-
if self.access_token is not None:
458-
auth["{{name}}"] = {
459-
"type": "bearer",
460-
"in": "header",
461-
{% if schema.bearerFormat %}
462-
"format": "{{ schema.bearerFormat }}",
463-
{% endif %}
464-
"key": "Authorization",
465-
"value": "Bearer " + self.access_token
466-
}
457+
if self.access_token is not None:
458+
auth["{{name}}"] = {
459+
"type": "bearer",
460+
"in": "header",
461+
{% if schema.bearerFormat %}
462+
"format": "{{ schema.bearerFormat }}",
463+
{% endif %}
464+
"key": "Authorization",
465+
"value": "Bearer " + self.access_token
466+
}
467467
{%- elif schema.type == "oauth2" %}
468-
if self.access_token is not None:
469-
auth["AuthZ"] = {
470-
"type": "oauth2",
471-
"in": "header",
472-
"key": "Authorization",
473-
"value": "Bearer " + self.access_token,
474-
}
468+
if self.access_token is not None:
469+
auth["AuthZ"] = {
470+
"type": "oauth2",
471+
"in": "header",
472+
"key": "Authorization",
473+
"value": "Bearer " + self.access_token,
474+
}
475475
{%- endif %}
476476
{%- endfor %}
477477
return auth

‎.generator/src/generator/templates/model_utils.j2

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2019,4 +2019,33 @@ class UnparsedObject(ModelNormal):
20192019
for var_name, var_value in kwargs.items():
20202020
self.__dict__[var_name] = var_value
20212021
self.__dict__["_data_store"][var_name] = var_value
2022+
2023+
2024+
def get_attribute_from_path(obj, path, default=None):
2025+
"""Return an attribute at `path` from the passed object."""
2026+
for elt in path.split("."):
2027+
try:
2028+
obj = obj[elt]
2029+
except (KeyError, AttributeError):
2030+
if default is None:
2031+
raise
2032+
return default
2033+
return obj
2034+
2035+
2036+
def set_attribute_from_path(obj, path, value, params_map):
2037+
"""Set an attribute at `path` with the given value."""
2038+
elts = path.split(".")
2039+
last = elts.pop(-1)
2040+
root = None
2041+
for i, elt in enumerate(elts):
2042+
if i:
2043+
root = root.openapi_types[elt][0]
2044+
else:
2045+
root = params_map[elt]["openapi_types"][0]
2046+
try:
2047+
obj = obj[elt]
2048+
except (KeyError, AttributeError):
2049+
obj = root()
2050+
obj[last] = value
20222051
{# keep new line #}

‎src/datadog_api_client/model_utils.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2021,3 +2021,32 @@ def __init__(self, **kwargs):
20212021
for var_name, var_value in kwargs.items():
20222022
self.__dict__[var_name] = var_value
20232023
self.__dict__["_data_store"][var_name] = var_value
2024+
2025+
2026+
def get_attribute_from_path(obj, path, default=None):
2027+
"""Return an attribute at `path` from the passed object."""
2028+
for elt in path.split("."):
2029+
try:
2030+
obj = obj[elt]
2031+
except (KeyError, AttributeError):
2032+
if default is None:
2033+
raise
2034+
return default
2035+
return obj
2036+
2037+
2038+
def set_attribute_from_path(obj, path, value, params_map):
2039+
"""Set an attribute at `path` with the given value."""
2040+
elts = path.split(".")
2041+
last = elts.pop(-1)
2042+
root = None
2043+
for i, elt in enumerate(elts):
2044+
if i:
2045+
root = root.openapi_types[elt][0]
2046+
else:
2047+
root = params_map[elt]["openapi_types"][0]
2048+
try:
2049+
obj = obj[elt]
2050+
except (KeyError, AttributeError):
2051+
obj = root()
2052+
obj[last] = value

0 commit comments

Comments
 (0)