Skip to content

refactor!: Remove parse_response arg from the call method #462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 13, 2025
Merged

refactor!: Remove parse_response arg from the call method #462

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
753465b
Remove the `parse_response` parameter for the `call` method
Mantisus Aug 12, 2025
94977d2
resolve
Mantisus Aug 13, 2025
9cc1438
resolve
Mantisus Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 2 additions & 39 deletions src/apify_client/_http_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
from typing import TYPE_CHECKING, Any

import httpx
from apify_shared.utils import ignore_docs, is_content_type_json, is_content_type_text, is_content_type_xml
from apify_shared.utils import ignore_docs

from apify_client._errors import ApifyApiError, InvalidResponseBodyError, is_retryable_error
from apify_client._errors import ApifyApiError, is_retryable_error
from apify_client._logging import log_context, logger_name
from apify_client._statistics import Statistics
from apify_client._utils import retry_with_exp_backoff, retry_with_exp_backoff_async
Expand Down Expand Up @@ -64,25 +64,6 @@ def __init__(

self.stats = stats or Statistics()

@staticmethod
def _maybe_parse_response(response: httpx.Response) -> Any:
if response.status_code == HTTPStatus.NO_CONTENT:
return None

content_type = ''
if 'content-type' in response.headers:
content_type = response.headers['content-type'].split(';')[0].strip()

try:
if is_content_type_json(content_type):
return response.json()
elif is_content_type_xml(content_type) or is_content_type_text(content_type): # noqa: RET505
return response.text
else:
return response.content
except ValueError as err:
raise InvalidResponseBodyError(response) from err

@staticmethod
def _parse_params(params: dict | None) -> dict | None:
if params is None:
Expand Down Expand Up @@ -143,17 +124,13 @@ def call(
data: Any = None,
json: JSONSerializable | None = None,
stream: bool | None = None,
parse_response: bool | None = True,
timeout_secs: int | None = None,
) -> httpx.Response:
log_context.method.set(method)
log_context.url.set(url)

self.stats.calls += 1

if stream and parse_response:
raise ValueError('Cannot stream response and parse it at the same time!')

headers, params, content = self._prepare_request_call(headers, params, data, json)

httpx_client = self.httpx_client
Expand Down Expand Up @@ -190,11 +167,6 @@ def _make_request(stop_retrying: Callable, attempt: int) -> httpx.Response:
# If response status is < 300, the request was successful, and we can return the result
if response.status_code < 300: # noqa: PLR2004
logger.debug('Request successful', extra={'status_code': response.status_code})
if not stream:
_maybe_parsed_body = (
self._maybe_parse_response(response) if parse_response else response.content
)
setattr(response, '_maybe_parsed_body', _maybe_parsed_body) # noqa: B010

return response

Expand Down Expand Up @@ -239,17 +211,13 @@ async def call(
data: Any = None,
json: JSONSerializable | None = None,
stream: bool | None = None,
parse_response: bool | None = True,
timeout_secs: int | None = None,
) -> httpx.Response:
log_context.method.set(method)
log_context.url.set(url)

self.stats.calls += 1

if stream and parse_response:
raise ValueError('Cannot stream response and parse it at the same time!')

headers, params, content = self._prepare_request_call(headers, params, data, json)

httpx_async_client = self.httpx_async_client
Expand Down Expand Up @@ -283,11 +251,6 @@ async def _make_request(stop_retrying: Callable, attempt: int) -> httpx.Response
# If response status is < 300, the request was successful, and we can return the result
if response.status_code < 300: # noqa: PLR2004
logger.debug('Request successful', extra={'status_code': response.status_code})
if not stream:
_maybe_parsed_body = (
self._maybe_parse_response(response) if parse_response else response.content
)
setattr(response, '_maybe_parsed_body', _maybe_parsed_body) # noqa: B010

return response

Expand Down
33 changes: 32 additions & 1 deletion src/apify_client/_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,21 @@
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, TypeVar, cast

from apify_shared.utils import is_file_or_bytes, maybe_extract_enum_member_value
from apify_shared.utils import (
is_content_type_json,
is_content_type_text,
is_content_type_xml,
is_file_or_bytes,
maybe_extract_enum_member_value,
)

from apify_client._errors import InvalidResponseBodyError

if TYPE_CHECKING:
from collections.abc import Awaitable

from httpx import Response

from apify_client._errors import ApifyApiError

PARSE_DATE_FIELDS_MAX_DEPTH = 3
Expand Down Expand Up @@ -149,3 +159,24 @@ def encode_key_value_store_record_value(value: Any, content_type: str | None = N
value = json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False, default=str).encode('utf-8')

return (value, content_type)


def maybe_parse_response(response: Response) -> Any:
if response.status_code == HTTPStatus.NO_CONTENT:
return None

content_type = ''
if 'content-type' in response.headers:
content_type = response.headers['content-type'].split(';')[0].strip()

try:
if is_content_type_json(content_type):
response_data = response.json()
elif is_content_type_xml(content_type) or is_content_type_text(content_type):
response_data = response.text
else:
response_data = response.content
except ValueError as err:
raise InvalidResponseBodyError(response) from err
else:
return response_data
4 changes: 0 additions & 4 deletions src/apify_client/clients/resource_clients/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,6 @@ def get_items_as_bytes(
url=self._url('items'),
method='GET',
params=request_params,
parse_response=False,
)

return response.content
Expand Down Expand Up @@ -516,7 +515,6 @@ def stream_items(
method='GET',
params=request_params,
stream=True,
parse_response=False,
)
yield response
finally:
Expand Down Expand Up @@ -877,7 +875,6 @@ async def get_items_as_bytes(
url=self._url('items'),
method='GET',
params=request_params,
parse_response=False,
)

return response.content
Expand Down Expand Up @@ -973,7 +970,6 @@ async def stream_items(
method='GET',
params=request_params,
stream=True,
parse_response=False,
)
yield response
finally:
Expand Down
15 changes: 8 additions & 7 deletions src/apify_client/clients/resource_clients/key_value_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@
from apify_shared.utils import filter_out_none_values_recursively, ignore_docs, parse_date_fields

from apify_client._errors import ApifyApiError
from apify_client._utils import catch_not_found_or_throw, encode_key_value_store_record_value, pluck_data
from apify_client._utils import (
catch_not_found_or_throw,
encode_key_value_store_record_value,
maybe_parse_response,
pluck_data,
)
from apify_client.clients.base import ResourceClient, ResourceClientAsync

if TYPE_CHECKING:
Expand Down Expand Up @@ -146,7 +151,7 @@ def get_record(self, key: str, *, as_bytes: bool = False, as_file: bool = False)

return {
'key': key,
'value': response._maybe_parsed_body, # type: ignore[attr-defined] # noqa: SLF001
'value': maybe_parse_response(response),
'content_type': response.headers['content-type'],
}

Expand Down Expand Up @@ -196,7 +201,6 @@ def get_record_as_bytes(self, key: str) -> dict | None:
url=self._url(f'records/{key}'),
method='GET',
params=self._params(),
parse_response=False,
)

return {
Expand Down Expand Up @@ -228,7 +232,6 @@ def stream_record(self, key: str) -> Iterator[dict | None]:
url=self._url(f'records/{key}'),
method='GET',
params=self._params(),
parse_response=False,
stream=True,
)

Expand Down Expand Up @@ -393,7 +396,7 @@ async def get_record(self, key: str) -> dict | None:

return {
'key': key,
'value': response._maybe_parsed_body, # type: ignore[attr-defined] # noqa: SLF001
'value': maybe_parse_response(response),
'content_type': response.headers['content-type'],
}

Expand Down Expand Up @@ -443,7 +446,6 @@ async def get_record_as_bytes(self, key: str) -> dict | None:
url=self._url(f'records/{key}'),
method='GET',
params=self._params(),
parse_response=False,
)

return {
Expand Down Expand Up @@ -475,7 +477,6 @@ async def stream_record(self, key: str) -> AsyncIterator[dict | None]:
url=self._url(f'records/{key}'),
method='GET',
params=self._params(),
parse_response=False,
stream=True,
)

Expand Down
4 changes: 0 additions & 4 deletions src/apify_client/clients/resource_clients/log.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ def get_as_bytes(self, *, raw: bool = False) -> bytes | None:
url=self.url,
method='GET',
params=self._params(raw=raw),
parse_response=False,
)

return response.content # noqa: TRY300
Expand Down Expand Up @@ -105,7 +104,6 @@ def stream(self, *, raw: bool = False) -> Iterator[httpx.Response | None]:
method='GET',
params=self._params(stream=True, raw=raw),
stream=True,
parse_response=False,
)

yield response
Expand Down Expand Up @@ -166,7 +164,6 @@ async def get_as_bytes(self, *, raw: bool = False) -> bytes | None:
url=self.url,
method='GET',
params=self._params(raw=raw),
parse_response=False,
)

return response.content # noqa: TRY300
Expand Down Expand Up @@ -195,7 +192,6 @@ async def stream(self, *, raw: bool = False) -> AsyncIterator[httpx.Response | N
method='GET',
params=self._params(stream=True, raw=raw),
stream=True,
parse_response=False,
)

yield response
Expand Down
4 changes: 2 additions & 2 deletions tests/unit/test_client_errors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def test_client_apify_api_error_streamed(httpserver: HTTPServer) -> None:
httpserver.expect_request('/stream_error').respond_with_handler(streaming_handler)

with pytest.raises(ApifyApiError) as e:
client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True, parse_response=False)
client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True)

assert e.value.message == error['error']['message']
assert e.value.type == error['error']['type']
Expand All @@ -108,7 +108,7 @@ async def test_async_client_apify_api_error_streamed(httpserver: HTTPServer) ->
httpserver.expect_request('/stream_error').respond_with_handler(streaming_handler)

with pytest.raises(ApifyApiError) as e:
await client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True, parse_response=False)
await client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True)

assert e.value.message == error['error']['message']
assert e.value.type == error['error']['type']
2 changes: 1 addition & 1 deletion uv.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading