Skip to content

Commit 0af80ad

Browse files
HantingZhang2therveci.datadog-api-spec
authored
Add retry support (#1502)
* Base design * add_test * adjust test * handle nil * Fix tests * add testing sleep time without actually sleeping * Add documentation * change code generator * pre-commit fixes * change how configs are made * pre-commit fixes * update readme doc * pre-commit fixes * adds validation for backoff_base * pre-commit fixes * change generator * pre-commit fixes * add validation back * pre-commit fixes --------- Co-authored-by: Thomas Hervé <thomas.herve@datadoghq.com> Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 803beb2 commit 0af80ad

File tree

7 files changed

+288
-106
lines changed

7 files changed

+288
-106
lines changed

‎.generator/src/generator/templates/api_client.j2‎

Lines changed: 81 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -44,71 +44,99 @@ module {{ module_name }}
4444
# the data deserialized from response body (could be nil), response status code and response headers.
4545
def call_api(http_method, path, opts = {})
4646
request = build_request(http_method, path, opts)
47-
if opts[:stream_body]
48-
tempfile = nil
49-
encoding = nil
50-
51-
response = request.perform do | chunk |
52-
unless tempfile
53-
content_disposition = chunk.http_response.header['Content-Disposition']
54-
if content_disposition && content_disposition =~ /filename=/i
55-
filename = content_disposition[/filename=['"]?([^'"\s]+)['"]?/, 1]
56-
prefix = sanitize_filename(filename)
57-
else
58-
prefix = 'download-'
59-
end
60-
prefix = prefix + '-' unless prefix.end_with?('-')
61-
unless encoding
62-
encoding = chunk.encoding
47+
attempt = 0
48+
loop do
49+
if opts[:stream_body]
50+
tempfile = nil
51+
encoding = nil
52+
53+
response = request.perform do | chunk |
54+
unless tempfile
55+
content_disposition = chunk.http_response.header['Content-Disposition']
56+
if content_disposition && content_disposition =~ /filename=/i
57+
filename = content_disposition[/filename=['"]?([^'"\s]+)['"]?/, 1]
58+
prefix = sanitize_filename(filename)
59+
else
60+
prefix = 'download-'
61+
end
62+
prefix = prefix + '-' unless prefix.end_with?('-')
63+
unless encoding
64+
encoding = chunk.encoding
65+
end
66+
tempfile = Tempfile.open(prefix, @config.temp_folder_path, encoding: encoding)
67+
@tempfile = tempfile
6368
end
64-
tempfile = Tempfile.open(prefix, @config.temp_folder_path, encoding: encoding)
65-
@tempfile = tempfile
69+
chunk.force_encoding(encoding)
70+
tempfile.write(chunk)
71+
end
72+
if tempfile
73+
tempfile.close
74+
@config.logger.info "Temp file written to #{tempfile.path}, please copy the file to a proper folder "\
75+
"with e.g. `FileUtils.cp(tempfile.path, '/new/file/path')` otherwise the temp file "\
76+
"will be deleted automatically with GC. It's also recommended to delete the temp file "\
77+
"explicitly with `tempfile.delete`"
6678
end
67-
chunk.force_encoding(encoding)
68-
tempfile.write(chunk)
79+
else
80+
response = request.perform
6981
end
70-
if tempfile
71-
tempfile.close
72-
@config.logger.info "Temp file written to #{tempfile.path}, please copy the file to a proper folder "\
73-
"with e.g. `FileUtils.cp(tempfile.path, '/new/file/path')` otherwise the temp file "\
74-
"will be deleted automatically with GC. It's also recommended to delete the temp file "\
75-
"explicitly with `tempfile.delete`"
82+
83+
if @config.debugging
84+
@config.logger.debug "HTTP response body ~BEGIN~\n#{response.body}\n~END~\n"
7685
end
77-
else
78-
response = request.perform
79-
end
8086

81-
if @config.debugging
82-
@config.logger.debug "HTTP response body ~BEGIN~\n#{response.body}\n~END~\n"
83-
end
87+
unless response.success?
88+
if response.request_timeout?
89+
fail APIError.new('Connection timed out')
90+
elsif response.code == 0
91+
# Errors from libcurl will be made visible here
92+
fail APIError.new(:code => 0,
93+
:message => response.return_message)
94+
else
95+
body = response.body
96+
if response.headers['Content-Encoding'].eql?('gzip') && !(body.nil? || body.empty?) then
97+
gzip = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
98+
body = gzip.inflate(body)
99+
gzip.close
100+
end
101+
if should_retry(attempt, @config.max_retries, response.code, @config.enable_retry)
102+
sleep calculate_retry_interval(response, @config.backoff_base, @config.backoff_multiplier, attempt, @config.timeout)
103+
attempt = attempt + 1
104+
next
105+
else
106+
fail APIError.new(:code => response.code,
107+
:response_headers => response.headers,
108+
:response_body => body),
109+
response.message
110+
end
111+
end
112+
end
84113

85-
unless response.success?
86-
if response.request_timeout?
87-
fail APIError.new('Connection timed out')
88-
elsif response.code == 0
89-
# Errors from libcurl will be made visible here
90-
fail APIError.new(:code => 0,
91-
:message => response.return_message)
114+
if opts[:return_type]
115+
data = deserialize(opts[:api_version], response, opts[:return_type])
92116
else
93-
body = response.body
94-
if response.headers['Content-Encoding'].eql?('gzip') && !(body.nil? || body.empty?) then
95-
gzip = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
96-
body = gzip.inflate(body)
97-
gzip.close
98-
end
99-
fail APIError.new(:code => response.code,
100-
:response_headers => response.headers,
101-
:response_body => body),
102-
response.message
117+
data = nil
103118
end
119+
return data, response.code, response.headers
104120
end
121+
end
122+
123+
# Check if an http request should be retried
124+
def should_retry(attempt, max_retries, http_code, enable_retry)
125+
(http_code == 429 || http_code >= 500) && max_retries > attempt && enable_retry
126+
end
105127

106-
if opts[:return_type]
107-
data = deserialize(opts[:api_version], response, opts[:return_type])
128+
# Calculate the sleep interval between 2 retry attempts
129+
def calculate_retry_interval(response, backoff_base, backoff_multiplier, attempt, timeout)
130+
reset_header = response.headers['X-Ratelimit-Reset']
131+
if !reset_header.nil? && !reset_header.empty?
132+
sleep_time = reset_header.to_i
108133
else
109-
data = nil
134+
sleep_time = (backoff_multiplier**attempt) * backoff_base
135+
if timeout && timeout > 0
136+
sleep_time = [timeout, sleep_time].min
137+
end
110138
end
111-
return data, response.code, response.headers
139+
sleep_time
112140
end
113141

114142
# Build the HTTP request

‎.generator/src/generator/templates/configuration.j2‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,16 @@ module {{ module_name }}
138138
# Password for proxy server authentication
139139
attr_accessor :http_proxypass
140140

141+
# Enable retry when rate limited
142+
attr_accessor :enable_retry
143+
144+
# Retry backoff calculation parameters
145+
attr_accessor :backoff_base
146+
attr_accessor :backoff_multiplier
147+
148+
# Maximum number of retry attempts allowed
149+
attr_accessor :max_retries
150+
141151
def initialize
142152
{%- set default_server = openapi.servers[0]|format_server %}
143153
@scheme = '{{ default_server.scheme }}'
@@ -149,6 +159,10 @@ module {{ module_name }}
149159
@server_operation_variables = {}
150160
@api_key = {}
151161
@api_key_prefix = {}
162+
@enable_retry = false
163+
@backoff_base = 2
164+
@backoff_multiplier = 2
165+
@max_retries = 3
152166
@timeout = nil
153167
@client_side_validation = true
154168
@verify_ssl = true
@@ -188,6 +202,13 @@ module {{ module_name }}
188202
@@default ||= Configuration.new
189203
end
190204

205+
def backoff_base=(value)
206+
if value < 2
207+
raise ArgumentError, 'backoff_base cannot be smaller than 2'
208+
end
209+
@backoff_base = value
210+
end
211+
191212
def configure
192213
yield(self) if block_given?
193214
end

‎README.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,29 @@ api_instance.list_incidents_with_pagination() do |incident|
165165
end
166166
```
167167

168+
### Retry
169+
170+
To enable the client to retry when rate limited (status 429) or status 500 and above:
171+
172+
```ruby
173+
config = DatadogAPIClient::Configuration.new
174+
config.enable_retry = true
175+
client = DatadogAPIClient::APIClient.new(config)
176+
```
177+
178+
The interval between 2 retry attempts will be the value of the `x-ratelimit-reset` response header when available.
179+
If not, it will be :
180+
181+
```ruby
182+
(config.backoffMultiplier ** current_retry_count) * config.backoffBase
183+
```
184+
185+
The maximum number of retry attempts is `3` by default and can be modified with
186+
187+
```ruby
188+
config.maxRetries
189+
```
190+
168191
## Documentation
169192

170193
If you are interested in general documentation for all public Datadog API endpoints, checkout the [general documentation site][api docs].

‎lib/datadog_api_client/api_client.rb‎

Lines changed: 81 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -55,71 +55,99 @@ def self.default
5555
# the data deserialized from response body (could be nil), response status code and response headers.
5656
def call_api(http_method, path, opts = {})
5757
request = build_request(http_method, path, opts)
58-
if opts[:stream_body]
59-
tempfile = nil
60-
encoding = nil
61-
62-
response = request.perform do | chunk |
63-
unless tempfile
64-
content_disposition = chunk.http_response.header['Content-Disposition']
65-
if content_disposition && content_disposition =~ /filename=/i
66-
filename = content_disposition[/filename=['"]?([^'"\s]+)['"]?/, 1]
67-
prefix = sanitize_filename(filename)
68-
else
69-
prefix = 'download-'
70-
end
71-
prefix = prefix + '-' unless prefix.end_with?('-')
72-
unless encoding
73-
encoding = chunk.encoding
58+
attempt = 0
59+
loop do
60+
if opts[:stream_body]
61+
tempfile = nil
62+
encoding = nil
63+
64+
response = request.perform do | chunk |
65+
unless tempfile
66+
content_disposition = chunk.http_response.header['Content-Disposition']
67+
if content_disposition && content_disposition =~ /filename=/i
68+
filename = content_disposition[/filename=['"]?([^'"\s]+)['"]?/, 1]
69+
prefix = sanitize_filename(filename)
70+
else
71+
prefix = 'download-'
72+
end
73+
prefix = prefix + '-' unless prefix.end_with?('-')
74+
unless encoding
75+
encoding = chunk.encoding
76+
end
77+
tempfile = Tempfile.open(prefix, @config.temp_folder_path, encoding: encoding)
78+
@tempfile = tempfile
7479
end
75-
tempfile = Tempfile.open(prefix, @config.temp_folder_path, encoding: encoding)
76-
@tempfile = tempfile
80+
chunk.force_encoding(encoding)
81+
tempfile.write(chunk)
82+
end
83+
if tempfile
84+
tempfile.close
85+
@config.logger.info "Temp file written to #{tempfile.path}, please copy the file to a proper folder "\
86+
"with e.g. `FileUtils.cp(tempfile.path, '/new/file/path')` otherwise the temp file "\
87+
"will be deleted automatically with GC. It's also recommended to delete the temp file "\
88+
"explicitly with `tempfile.delete`"
7789
end
78-
chunk.force_encoding(encoding)
79-
tempfile.write(chunk)
90+
else
91+
response = request.perform
8092
end
81-
if tempfile
82-
tempfile.close
83-
@config.logger.info "Temp file written to #{tempfile.path}, please copy the file to a proper folder "\
84-
"with e.g. `FileUtils.cp(tempfile.path, '/new/file/path')` otherwise the temp file "\
85-
"will be deleted automatically with GC. It's also recommended to delete the temp file "\
86-
"explicitly with `tempfile.delete`"
93+
94+
if @config.debugging
95+
@config.logger.debug "HTTP response body ~BEGIN~\n#{response.body}\n~END~\n"
8796
end
88-
else
89-
response = request.perform
90-
end
9197

92-
if @config.debugging
93-
@config.logger.debug "HTTP response body ~BEGIN~\n#{response.body}\n~END~\n"
94-
end
98+
unless response.success?
99+
if response.request_timeout?
100+
fail APIError.new('Connection timed out')
101+
elsif response.code == 0
102+
# Errors from libcurl will be made visible here
103+
fail APIError.new(:code => 0,
104+
:message => response.return_message)
105+
else
106+
body = response.body
107+
if response.headers['Content-Encoding'].eql?('gzip') && !(body.nil? || body.empty?) then
108+
gzip = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
109+
body = gzip.inflate(body)
110+
gzip.close
111+
end
112+
if should_retry(attempt, @config.max_retries, response.code, @config.enable_retry)
113+
sleep calculate_retry_interval(response, @config.backoff_base, @config.backoff_multiplier, attempt, @config.timeout)
114+
attempt = attempt + 1
115+
next
116+
else
117+
fail APIError.new(:code => response.code,
118+
:response_headers => response.headers,
119+
:response_body => body),
120+
response.message
121+
end
122+
end
123+
end
95124

96-
unless response.success?
97-
if response.request_timeout?
98-
fail APIError.new('Connection timed out')
99-
elsif response.code == 0
100-
# Errors from libcurl will be made visible here
101-
fail APIError.new(:code => 0,
102-
:message => response.return_message)
125+
if opts[:return_type]
126+
data = deserialize(opts[:api_version], response, opts[:return_type])
103127
else
104-
body = response.body
105-
if response.headers['Content-Encoding'].eql?('gzip') && !(body.nil? || body.empty?) then
106-
gzip = Zlib::Inflate.new(Zlib::MAX_WBITS + 16)
107-
body = gzip.inflate(body)
108-
gzip.close
109-
end
110-
fail APIError.new(:code => response.code,
111-
:response_headers => response.headers,
112-
:response_body => body),
113-
response.message
128+
data = nil
114129
end
130+
return data, response.code, response.headers
115131
end
132+
end
133+
134+
# Check if an http request should be retried
135+
def should_retry(attempt, max_retries, http_code, enable_retry)
136+
(http_code == 429 || http_code >= 500) && max_retries > attempt && enable_retry
137+
end
116138

117-
if opts[:return_type]
118-
data = deserialize(opts[:api_version], response, opts[:return_type])
139+
# Calculate the sleep interval between 2 retry attempts
140+
def calculate_retry_interval(response, backoff_base, backoff_multiplier, attempt, timeout)
141+
reset_header = response.headers['X-Ratelimit-Reset']
142+
if !reset_header.nil? && !reset_header.empty?
143+
sleep_time = reset_header.to_i
119144
else
120-
data = nil
145+
sleep_time = (backoff_multiplier**attempt) * backoff_base
146+
if timeout && timeout > 0
147+
sleep_time = [timeout, sleep_time].min
148+
end
121149
end
122-
return data, response.code, response.headers
150+
sleep_time
123151
end
124152

125153
# Build the HTTP request

0 commit comments

Comments
 (0)