Skip to content

Commit cca6c5d

Browse files
smaeda-kssmaeda-ks
authored
handle rate limit and implement retry logic (#213)
* handle rate limit and implement retry logic * Add test for client options * Add more test cases for rate limit & retry options * Use monkeypatch to stub sleep * Do not use unnecessary temp variables
1 parent e0db0ca commit cca6c5d

File tree

6 files changed

+435
-16
lines changed

6 files changed

+435
-16
lines changed

‎tests/test_client.py‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
11
# Copyright (C) 2015 Twitter, Inc.
2-
3-
import pytest
4-
import requests_oauthlib
52
import responses
63

74
from tests.support import with_resource, with_fixture, characters
@@ -11,11 +8,12 @@
118
from twitter_ads.cursor import Cursor
129
from twitter_ads import API_VERSION
1310

11+
1412
@responses.activate
1513
def test_accounts_with_no_id():
1614
responses.add(responses.GET, with_resource('/' + API_VERSION + '/accounts'),
17-
body=with_fixture('accounts_all'),
18-
content_type='application/json')
15+
body=with_fixture('accounts_all'),
16+
content_type='application/json')
1917

2018
client = Client(
2119
characters(40),
@@ -29,11 +27,12 @@ def test_accounts_with_no_id():
2927
assert isinstance(cursor, Cursor)
3028
assert cursor.count == 5
3129

30+
3231
@responses.activate
3332
def test_accounts_with_id():
3433
responses.add(responses.GET, with_resource('/' + API_VERSION + '/accounts/2iqph'),
35-
body=with_fixture('accounts_load'),
36-
content_type='application/json')
34+
body=with_fixture('accounts_load'),
35+
content_type='application/json')
3736

3837
client = Client(
3938
characters(40),
@@ -46,3 +45,24 @@ def test_accounts_with_id():
4645
assert account is not None
4746
assert isinstance(account, Account)
4847
assert account.id == '2iqph'
48+
49+
50+
@responses.activate
51+
def test_accounts_with_options():
52+
53+
client = Client(
54+
characters(40),
55+
characters(40),
56+
characters(40),
57+
characters(40),
58+
options={
59+
'handle_rate_limit': True,
60+
'retry_max': 1,
61+
'retry_delay': 3000,
62+
'retry_on_status': [404, 500, 503]
63+
}
64+
)
65+
66+
assert client is not None
67+
assert isinstance(client, Client)
68+
assert len(client.options) == 4

‎tests/test_rate_limit.py‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import responses
22
import unittest
3+
import time
34

45
from tests.support import with_resource, with_fixture, characters
56

@@ -10,6 +11,251 @@
1011
from twitter_ads.http import Request
1112
from twitter_ads.resource import Resource
1213
from twitter_ads import API_VERSION
14+
from twitter_ads.error import RateLimit
15+
16+
17+
@responses.activate
18+
def test_rate_limit_handle_with_retry_success_1(monkeypatch):
19+
# scenario:
20+
# - 500 (retry) -> 429 (handle rate limit) -> 200 (end)
21+
monkeypatch.setattr(time, 'sleep', lambda s: None)
22+
23+
responses.add(responses.GET,
24+
with_resource('/' + API_VERSION + '/accounts/2iqph'),
25+
body=with_fixture('accounts_load'),
26+
content_type='application/json')
27+
28+
responses.add(responses.GET,
29+
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
30+
status=500,
31+
body=with_fixture('campaigns_all'),
32+
content_type='application/json',
33+
headers={
34+
'x-account-rate-limit-limit': '10000',
35+
'x-account-rate-limit-remaining': '0',
36+
'x-account-rate-limit-reset': '1546300800'
37+
})
38+
39+
responses.add(responses.GET,
40+
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
41+
status=429,
42+
body=with_fixture('campaigns_all'),
43+
content_type='application/json',
44+
headers={
45+
'x-account-rate-limit-limit': '10000',
46+
'x-account-rate-limit-remaining': '0',
47+
'x-account-rate-limit-reset': str(int(time.time()) + 5)
48+
})
49+
50+
responses.add(responses.GET,
51+
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
52+
status=200,
53+
body=with_fixture('campaigns_all'),
54+
content_type='application/json',
55+
headers={
56+
'x-account-rate-limit-limit': '10000',
57+
'x-account-rate-limit-remaining': '9999',
58+
'x-account-rate-limit-reset': '1546300800'
59+
})
60+
61+
client = Client(
62+
characters(40),
63+
characters(40),
64+
characters(40),
65+
characters(40),
66+
options={
67+
'handle_rate_limit': True,
68+
'retry_max': 1,
69+
'retry_delay': 3000,
70+
'retry_on_status': [500]
71+
}
72+
)
73+
74+
account = Account.load(client, '2iqph')
75+
76+
cursor = Campaign.all(account)
77+
assert len(responses.calls) == 4
78+
assert cursor is not None
79+
assert isinstance(cursor, Cursor)
80+
assert cursor.rate_limit is None
81+
assert cursor.account_rate_limit == '10000'
82+
assert cursor.account_rate_limit_remaining == '9999'
83+
assert cursor.account_rate_limit_reset == '1546300800'
84+
85+
86+
@responses.activate
87+
def test_rate_limit_handle_with_retry_success_2(monkeypatch):
88+
# scenario:
89+
# - 429 (handle rate limit) -> 500 (retry) -> 200 (end)
90+
monkeypatch.setattr(time, 'sleep', lambda s: None)
91+
92+
responses.add(responses.GET,
93+
with_resource('/' + API_VERSION + '/accounts/2iqph'),
94+
body=with_fixture('accounts_load'),
95+
content_type='application/json')
96+
97+
responses.add(responses.GET,
98+
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
99+
status=429,
100+
body=with_fixture('campaigns_all'),
101+
content_type='application/json',
102+
headers={
103+
'x-account-rate-limit-limit': '10000',
104+
'x-account-rate-limit-remaining': '0',
105+
'x-account-rate-limit-reset': '1546300800'
106+
})
107+
108+
responses.add(responses.GET,
109+
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
110+
status=500,
111+
body=with_fixture('campaigns_all'),
112+
content_type='application/json',
113+
headers={
114+
'x-account-rate-limit-limit': '10000',
115+
'x-account-rate-limit-remaining': '0',
116+
'x-account-rate-limit-reset': str(int(time.time()) + 5)
117+
})
118+
119+
responses.add(responses.GET,
120+
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
121+
status=200,
122+
body=with_fixture('campaigns_all'),
123+
content_type='application/json',
124+
headers={
125+
'x-account-rate-limit-limit': '10000',
126+
'x-account-rate-limit-remaining': '9999',
127+
'x-account-rate-limit-reset': '1546300800'
128+
})
129+
130+
client = Client(
131+
characters(40),
132+
characters(40),
133+
characters(40),
134+
characters(40),
135+
options={
136+
'handle_rate_limit': True,
137+
'retry_max': 1,
138+
'retry_delay': 3000,
139+
'retry_on_status': [500]
140+
}
141+
)
142+
143+
account = Account.load(client, '2iqph')
144+
145+
cursor = Campaign.all(account)
146+
assert len(responses.calls) == 4
147+
assert cursor is not None
148+
assert isinstance(cursor, Cursor)
149+
assert cursor.rate_limit is None
150+
assert cursor.account_rate_limit == '10000'
151+
assert cursor.account_rate_limit_remaining == '9999'
152+
assert cursor.account_rate_limit_reset == '1546300800'
153+
154+
155+
@responses.activate
156+
def test_rate_limit_handle_success(monkeypatch):
157+
monkeypatch.setattr(time, 'sleep', lambda s: None)
158+
159+
responses.add(responses.GET,
160+
with_resource('/' + API_VERSION + '/accounts/2iqph'),
161+
body=with_fixture('accounts_load'),
162+
content_type='application/json')
163+
164+
responses.add(responses.GET,
165+
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
166+
status=429,
167+
body=with_fixture('campaigns_all'),
168+
content_type='application/json',
169+
headers={
170+
'x-account-rate-limit-limit': '10000',
171+
'x-account-rate-limit-remaining': '0',
172+
'x-account-rate-limit-reset': str(int(time.time()) + 5)
173+
})
174+
175+
responses.add(responses.GET,
176+
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
177+
status=200,
178+
body=with_fixture('campaigns_all'),
179+
content_type='application/json',
180+
headers={
181+
'x-account-rate-limit-limit': '10000',
182+
'x-account-rate-limit-remaining': '9999',
183+
'x-account-rate-limit-reset': '1546300800'
184+
})
185+
186+
client = Client(
187+
characters(40),
188+
characters(40),
189+
characters(40),
190+
characters(40),
191+
options={
192+
'handle_rate_limit': True
193+
}
194+
)
195+
196+
account = Account.load(client, '2iqph')
197+
198+
cursor = Campaign.all(account)
199+
assert len(responses.calls) == 3
200+
assert cursor is not None
201+
assert isinstance(cursor, Cursor)
202+
assert cursor.rate_limit is None
203+
assert cursor.account_rate_limit == '10000'
204+
assert cursor.account_rate_limit_remaining == '9999'
205+
assert cursor.account_rate_limit_reset == '1546300800'
206+
207+
208+
@responses.activate
209+
def test_rate_limit_handle_error(monkeypatch):
210+
monkeypatch.setattr(time, 'sleep', lambda s: None)
211+
212+
responses.add(responses.GET,
213+
with_resource('/' + API_VERSION + '/accounts/2iqph'),
214+
body=with_fixture('accounts_load'),
215+
content_type='application/json')
216+
217+
responses.add(responses.GET,
218+
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
219+
status=429,
220+
body=with_fixture('campaigns_all'),
221+
content_type='application/json',
222+
headers={
223+
'x-account-rate-limit-limit': '10000',
224+
'x-account-rate-limit-remaining': '0',
225+
'x-account-rate-limit-reset': str(int(time.time()) + 5)
226+
})
227+
228+
responses.add(responses.GET,
229+
with_resource('/' + API_VERSION + '/accounts/2iqph/campaigns'),
230+
status=429,
231+
body=with_fixture('campaigns_all'),
232+
content_type='application/json',
233+
headers={
234+
'x-account-rate-limit-limit': '10000',
235+
'x-account-rate-limit-remaining': '0',
236+
'x-account-rate-limit-reset': '1546300800'
237+
})
238+
239+
client = Client(
240+
characters(40),
241+
characters(40),
242+
characters(40),
243+
characters(40),
244+
options={
245+
'handle_rate_limit': True
246+
}
247+
)
248+
249+
account = Account.load(client, '2iqph')
250+
251+
try:
252+
cursor = Campaign.all(account)
253+
except Exception as e:
254+
error = e
255+
print(error)
256+
assert len(responses.calls) == 3
257+
assert isinstance(error, RateLimit)
258+
assert error.reset_at == '1546300800'
13259

14260

15261
@responses.activate

0 commit comments

Comments
 (0)