Skip to content

Commit b2742d0

Browse files
jocejoce
committed
Harden Yahoo client requests
1 parent bda122b commit b2742d0

File tree

3 files changed

+138
-34
lines changed

3 files changed

+138
-34
lines changed

‎.cspell.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
"pageup",
3131
"referer",
3232
"renderable",
33+
"retryable",
3334
"triggerable",
3435
"watchlist",
3536
"watchlists"

‎src/calahan/_yasync_client.py‎

Lines changed: 69 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ class YAsyncClient:
4747

4848
_DEFAULT_TIMEOUT: Final[float] = 5.0
4949
_READ_TIMEOUT: Final[float] = 15.0
50+
_REQUEST_ATTEMPTS: Final[int] = 3
51+
_RETRYABLE_STATUS_CODES: Final[frozenset[int]] = frozenset({429, 502, 503, 504})
52+
_RETRY_DELAY_SECONDS: Final[float] = 0.25
5053

5154
def __init__(self, timeout: httpx.Timeout | None = None) -> None:
5255
"""Initialize the async Yahoo! Finance API client.
@@ -98,15 +101,35 @@ async def _request_or_raise(
98101
"""
99102

100103
request = self._client.get if method == "GET" else self._client.post
101-
start = time.perf_counter()
102-
try:
103-
response = await request(url, **kwargs)
104-
response.raise_for_status()
105-
except httpx.HTTPStatusError as exc:
106-
if exc.response.is_error:
104+
attempt = 1
105+
while True:
106+
start = time.perf_counter()
107+
try:
108+
response = await request(url, **kwargs)
109+
if response.is_error:
110+
response.raise_for_status()
111+
except httpx.HTTPStatusError as exc:
107112
status_code = exc.response.status_code if exc.response else -1
108113
reason = exc.response.reason_phrase if exc.response else "unknown"
109114
url_str = str(exc.request.url)
115+
if (
116+
method == "GET"
117+
and status_code in self._RETRYABLE_STATUS_CODES
118+
and attempt < self._REQUEST_ATTEMPTS
119+
):
120+
self._logger.warning(
121+
"Transient HTTP error for '%s': Status %s - %s. "
122+
"Retrying attempt %s/%s.",
123+
context,
124+
status_code,
125+
reason,
126+
attempt + 1,
127+
self._REQUEST_ATTEMPTS,
128+
)
129+
await asyncio.sleep(self._RETRY_DELAY_SECONDS * attempt)
130+
attempt += 1
131+
continue
132+
110133
self._logger.exception(
111134
"HTTP error for '%s': Status %s - %s. "
112135
"URL: %s. "
@@ -118,33 +141,46 @@ async def _request_or_raise(
118141
url_str,
119142
)
120143
raise MarketDataRequestError(status_code, url_str) from exc
121-
except httpx.TransportError as exc:
122-
self._logger.exception(
123-
"Transport error for '%s'. "
124-
"Potential causes: network connectivity issues, "
125-
"DNS resolution failure, or timeout. "
126-
"Check your internet connection.",
127-
context,
128-
)
129-
raise MarketDataUnavailableError(context) from exc
130-
except asyncio.CancelledError:
131-
self._logger.info(
132-
"Request cancelled for '%s'. "
133-
"Typically occurs during application shutdown "
134-
"or when a timeout is exceeded.",
135-
context,
136-
)
137-
raise
138-
finally:
139-
elapsed_ms = (time.perf_counter() - start) * 1_000.0
140-
self._logger.debug(
141-
"Request timing for '%s': %.1f ms (method=%s url=%s)",
142-
context,
143-
elapsed_ms,
144-
method,
145-
url,
146-
)
147-
return response # type: ignore reportPossiblyUnboundVariable
144+
except httpx.TransportError as exc:
145+
if method == "GET" and attempt < self._REQUEST_ATTEMPTS:
146+
self._logger.warning(
147+
"Transient transport error for '%s'. Retrying attempt %s/%s.",
148+
context,
149+
attempt + 1,
150+
self._REQUEST_ATTEMPTS,
151+
)
152+
await asyncio.sleep(self._RETRY_DELAY_SECONDS * attempt)
153+
attempt += 1
154+
continue
155+
156+
self._logger.exception(
157+
"Transport error for '%s'. "
158+
"Potential causes: network connectivity issues, "
159+
"DNS resolution failure, or timeout. "
160+
"Check your internet connection.",
161+
context,
162+
)
163+
raise MarketDataUnavailableError(context) from exc
164+
except asyncio.CancelledError:
165+
self._logger.info(
166+
"Request cancelled for '%s'. "
167+
"Typically occurs during application shutdown "
168+
"or when a timeout is exceeded.",
169+
context,
170+
)
171+
raise
172+
else:
173+
return response
174+
finally:
175+
elapsed_ms = (time.perf_counter() - start) * 1_000.0
176+
self._logger.debug(
177+
"Request timing for '%s': %.1f ms (method=%s url=%s attempt=%s)",
178+
context,
179+
elapsed_ms,
180+
method,
181+
url,
182+
attempt,
183+
)
148184

149185
async def _refresh_cookies(self) -> None:
150186
"""Log into Yahoo! finance and set required cookies.

‎tests/calahan/test_yasync_client.py‎

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,73 @@ async def test_safe_request_get_success(httpx_mock: HTTPXMock) -> None:
130130
assert request.method == "GET"
131131

132132

133+
@pytest.mark.asyncio
134+
async def test_safe_request_allows_redirect_response(httpx_mock: HTTPXMock) -> None:
135+
"""Return redirect responses so callers can handle consent flows."""
136+
137+
httpx_mock.add_response(
138+
url=EXAMPLE_URL,
139+
status_code=307,
140+
headers={"Location": "https://example.com/redirect"},
141+
)
142+
143+
client = YAsyncClient()
144+
response = await client._request_or_raise("GET", EXAMPLE_URL, context="ctx")
145+
146+
assert response.status_code == httpx.codes(307)
147+
assert response.is_redirect
148+
149+
150+
@pytest.mark.asyncio
151+
async def test_safe_request_retries_retryable_http_status(
152+
httpx_mock: HTTPXMock,
153+
monkeypatch: pytest.MonkeyPatch,
154+
caplog: pytest.LogCaptureFixture,
155+
) -> None:
156+
"""Retry transient GET status failures before succeeding."""
157+
158+
httpx_mock.add_response(url=EXAMPLE_URL, status_code=502)
159+
httpx_mock.add_response(url=EXAMPLE_URL, status_code=200)
160+
161+
sleep_mock = AsyncMock()
162+
monkeypatch.setattr("calahan._yasync_client.asyncio.sleep", sleep_mock)
163+
164+
client = YAsyncClient()
165+
with caplog.at_level("WARNING"):
166+
response = await client._request_or_raise("GET", EXAMPLE_URL, context="ctx")
167+
168+
expected_request_count = 2
169+
assert response.status_code == httpx.codes(200)
170+
assert "Transient HTTP error for 'ctx': Status 502" in caplog.text
171+
sleep_mock.assert_awaited_once_with(client._RETRY_DELAY_SECONDS)
172+
assert len(httpx_mock.get_requests()) == expected_request_count
173+
174+
175+
@pytest.mark.asyncio
176+
async def test_safe_request_retries_transport_error(
177+
httpx_mock: HTTPXMock,
178+
monkeypatch: pytest.MonkeyPatch,
179+
caplog: pytest.LogCaptureFixture,
180+
) -> None:
181+
"""Retry transient GET transport failures before succeeding."""
182+
183+
httpx_mock.add_exception(httpx.TransportError("fail"), url=EXAMPLE_URL)
184+
httpx_mock.add_response(url=EXAMPLE_URL, status_code=200)
185+
186+
sleep_mock = AsyncMock()
187+
monkeypatch.setattr("calahan._yasync_client.asyncio.sleep", sleep_mock)
188+
189+
client = YAsyncClient()
190+
with caplog.at_level("WARNING"):
191+
response = await client._request_or_raise("GET", EXAMPLE_URL, context="ctx")
192+
193+
expected_request_count = 2
194+
assert response.status_code == httpx.codes(200)
195+
assert "Transient transport error for 'ctx'" in caplog.text
196+
sleep_mock.assert_awaited_once_with(client._RETRY_DELAY_SECONDS)
197+
assert len(httpx_mock.get_requests()) == expected_request_count
198+
199+
133200
@pytest.mark.asyncio
134201
async def test_safe_request_handles_http_status_error(
135202
httpx_mock: HTTPXMock,
@@ -160,7 +227,7 @@ async def test_safe_request_handles_transport_error(
160227

161228
client = YAsyncClient()
162229
with caplog.at_level("ERROR"), pytest.raises(MarketDataUnavailableError):
163-
await client._request_or_raise("GET", EXAMPLE_URL, context="ctx")
230+
await client._request_or_raise("POST", EXAMPLE_URL, context="ctx")
164231

165232
assert "Transport error for 'ctx'" in caplog.text
166233

0 commit comments

Comments
 (0)