diff --git a/asyncprawcore/requestor.py b/asyncprawcore/requestor.py index cf029ef..b9d40a2 100644 --- a/asyncprawcore/requestor.py +++ b/asyncprawcore/requestor.py @@ -121,4 +121,4 @@ async def request(self, *args: Any, timeout: float | None = None, **kwargs: Any) except ResponseException as exc: raise exc except Exception as exc: # noqa: BLE001 - raise RequestException(exc, args, kwargs) from None + raise RequestException(exc, args, kwargs) from exc diff --git a/asyncprawcore/sessions.py b/asyncprawcore/sessions.py index 77b6628..4399725 100644 --- a/asyncprawcore/sessions.py +++ b/asyncprawcore/sessions.py @@ -153,14 +153,13 @@ def _log_request( log.debug("Params: %s", pformat(params)) @staticmethod - def _preprocess_dict(data: dict[str, object] | None) -> dict[str, object]: + def _preprocess_dict(data: dict[str, object]) -> dict[str, object]: new_data = {} - if data: - for key, value in data.items(): - if isinstance(value, bool): - new_data[key] = str(value).lower() - elif value is not None: - new_data[key] = str(value) if not isinstance(value, str) else value + for key, value in data.items(): + if isinstance(value, bool): + new_data[key] = str(value).lower() + elif value is not None: + new_data[key] = str(value) if not isinstance(value, str) else value return new_data @property @@ -197,7 +196,6 @@ async def _do_retry( self, *, data: list[tuple[str, object]] | None, - files: dict[str, BinaryIO | TextIO] | None, json: dict[str, object] | None, method: str, params: dict[str, object], @@ -209,7 +207,6 @@ async def _do_retry( log.warning("Retrying due to %s: %s %s", status, method, url) return await self._request_with_retries( data=data, - files=files, json=json, method=method, params=params, @@ -223,7 +220,6 @@ async def _do_retry( async def _make_request( self, data: list[tuple[str, object]] | None, - files: dict[str, BinaryIO | TextIO] | None, json: dict[str, object] | None, method: str, params: dict[str, object], @@ -237,7 +233,6 @@ async def _make_request( url, allow_redirects=False, data=data, - files=files, json=json, params=params, timeout=timeout, @@ -253,7 +248,36 @@ async def _make_request( ) yield response - def _preprocess_params(self, params: dict[str, object] | None) -> dict[str, object]: + def _preprocess_data( + self, + data: dict[str, object], + files: dict[str, BinaryIO | TextIO] | None, + ) -> dict[str, object]: + """Preprocess data and files before request. + + This is to convert requests that are formatted for the ``requests`` package to + be compatible with the ``aiohttp`` package. The motivation for this is so that + ``praw`` and ``asyncpraw`` can remain as similar as possible and thus making + contributions to ``asyncpraw`` simpler. + + This method does the following: + + - Removes keys that have a value of ``None`` from ``data``. + - Moves ``files`` into ``data``. + + :param data: Dictionary, bytes, or file-like object to send in the body of the + request. + :param files: Dictionary, mapping ``filename`` to file-like object to add to + ``data``. + + """ + if isinstance(data, dict): + data = self._preprocess_dict(data) + if files is not None: + data.update(files) + return data + + def _preprocess_params(self, params: dict[str, object]) -> dict[str, object]: """Preprocess params before request. This is to convert requests that are formatted for the ``requests`` package to @@ -275,7 +299,6 @@ async def _request_with_retries( # noqa: PLR0912 self, *, data: list[tuple[str, object]] | None, - files: dict[str, BinaryIO | TextIO] | None, json: dict[str, object] | None, method: str, params: dict[str, object], @@ -292,7 +315,6 @@ async def _request_with_retries( # noqa: PLR0912 try: async with self._make_request( data=data, - files=files, json=json, method=method, params=params, @@ -310,7 +332,6 @@ async def _request_with_retries( # noqa: PLR0912 if retry_status is not None and retry_strategy_state.should_retry_on_failure(): return await self._do_retry( data=data, - files=files, json=json, method=method, params=params, @@ -340,7 +361,6 @@ async def _request_with_retries( # noqa: PLR0912 ): return await self._do_retry( data=data, - files=files, json=json, method=method, params=params, @@ -393,7 +413,7 @@ async def request( params = self._preprocess_params(deepcopy(params) or {}) params["raw_json"] = "1" if isinstance(data, dict): - data = self._preprocess_dict(deepcopy(data)) + data = self._preprocess_data(deepcopy(data), files) data["api_type"] = "json" data_list = sorted(data.items()) else: @@ -404,7 +424,6 @@ async def request( url = urljoin(self._requestor.oauth_url, path) return await self._request_with_retries( data=data_list, - files=files, json=json, method=method, params=params, diff --git a/pyproject.toml b/pyproject.toml index a27cb17..077669e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ test = [ "pytest>=8.3.4", "pytest-asyncio==1.2.*", "pytest-vcr==1.*", - "vcrpy==7.0.0" + "vcrpy==4.3.1" ] type = [ "aiohttp<4", diff --git a/tests/integration/cassettes/TestSession.test_request__post__with_files.json b/tests/integration/cassettes/TestSession.test_request__post__with_files.json index 1014717..b8e8f07 100644 --- a/tests/integration/cassettes/TestSession.test_request__post__with_files.json +++ b/tests/integration/cassettes/TestSession.test_request__post__with_files.json @@ -27,7 +27,7 @@ "close" ], "User-Agent": [ - "asyncprawcore:test (by u/Lil_SpazJoekp) asyncprawcore/2.3.1.dev0" + "asyncprawcore:test (by u/Lil_SpazJoekp) asyncprawcore/3.0.3.dev0" ] }, "method": "POST", @@ -42,7 +42,7 @@ "bytes" ], "Cache-Control": [ - "private, max-age=3600" + "private, s-maxage=0, max-age=0, must-revalidate, no-store" ], "Connection": [ "close" @@ -54,7 +54,10 @@ "application/json; charset=UTF-8" ], "Date": [ - "Mon, 27 Nov 2023 01:54:04 GMT" + "Sat, 04 Oct 2025 21:26:58 GMT" + ], + "Expires": [ + "-1" ], "NEL": [ "{\"report_to\": \"w3-reporting-nel\", \"max_age\": 14400, \"include_subdomains\": false, \"success_fraction\": 1.0, \"failure_fraction\": 1.0}" @@ -66,7 +69,7 @@ "snooserv" ], "Set-Cookie": [ - "edgebucket=47Z4cbkEIRoMLAEduI; Domain=reddit.com; Max-Age=63071999; Path=/; secure" + "edgebucket=LcttArzS78Dk2wvBkG; Domain=reddit.com; Max-Age=63071999; Path=/; secure" ], "Strict-Transport-Security": [ "max-age=31536000; includeSubdomains" @@ -118,10 +121,10 @@ "bearer " ], "Cookie": [ - "edgebucket=47Z4cbkEIRoMLAEduI" + "edgebucket=LcttArzS78Dk2wvBkG" ], "User-Agent": [ - "asyncprawcore:test (by u/Lil_SpazJoekp) asyncprawcore/2.3.1.dev0" + "asyncprawcore:test (by u/Lil_SpazJoekp) asyncprawcore/3.0.3.dev0" ] }, "method": "POST", @@ -148,7 +151,7 @@ "application/json; charset=UTF-8" ], "Date": [ - "Mon, 27 Nov 2023 01:54:04 GMT" + "Sat, 04 Oct 2025 21:26:58 GMT" ], "Expires": [ "-1" @@ -163,9 +166,9 @@ "snooserv" ], "Set-Cookie": [ - "loid=00000000003ebyblla.2.1552433321352.Z0FBQUFBQmxZX2E4NVJaUjQzYm82ZUE0aEpxSnJyMlZEc3NNc21hdVl6NGN1MU9ZWnRaLW1CNnBsVmQtdHdiTFdBdnVlX3M1V3VfemlSdU9VMTlkMkdURzA4bmIzSXh4MFU1Z1pFQXk3ajhrcEU3dzEySVVGN2gwZkdGX29UbUJlNkx0bGN5bmJPMGY; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Wed, 26-Nov-2025 01:54:04 GMT; secure", - "redesign_optout=true; Domain=reddit.com; Max-Age=94607999; Path=/; expires=Thu, 26-Nov-2026 01:54:04 GMT; secure", - "session_tracker=rkcihjnenmqghcdkif.0.1701050044379.Z0FBQUFBQmxZX2E4SG45MHJFN3FZenBZaWZmNV9IRVNCUHg2VWdRdGdDdzdvbUpheHdvanU5d1lFODUyUUh5R3JlTUUxd3p1MWdNcmZHU3dwNVMxVkphY2ZpZFFXLTFwSGtwQktPY1hNMERTaTJSeFVUM1lSTzY4UVhUWktPMUR5UG5mSmF5NVRydHo; Domain=reddit.com; Max-Age=7199; Path=/; expires=Mon, 27-Nov-2023 03:54:04 GMT; secure" + "loid=00000000003ebyblla.2.1552433321352.Z0FBQUFBQm80WkVpVFlkajgzWlBlb3VRQ29Zc09wUTJtSi1vQTZiWTNENFhpSVhCeW9SeXlJd2wwQmU3dW91d19qa21ReGhnLUJOaVhsNGlXb0RoWnNraFRCMVAyR3NsMktSWG4zbWJPMTlaeHhLenBHXzhLQlA3MGo5NHVYWUhkMXlmUWxKZUQtdGs; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Mon, 04-Oct-2027 21:26:58 GMT; secure", + "redesign_optout=true; Domain=reddit.com; Max-Age=94607999; Path=/; expires=Tue, 03-Oct-2028 21:26:58 GMT; secure", + "session_tracker=dmmnbkoichealejbmc.0.1759613218619.Z0FBQUFBQm80WkVpTWszMTFMczlvQWpMdEd6LUxqYTNFQUZOSHBSaDVEdU9WRFhuLUJZbjNCMEpCN0xNU1NteWV6NXFVNkhXajFRNlpPenVXeXlzdF9DdjQ2Z1ZJMWdHUjZDMHdDNGdXV25ZZDFyejlkRzlOWU5Ud05iZ01FOU5YRjYwMUhtWmtLNXA; Domain=reddit.com; Max-Age=7199; Path=/; expires=Sat, 04-Oct-2025 23:26:58 GMT; secure" ], "Strict-Transport-Security": [ "max-age=31536000; includeSubdomains" @@ -186,13 +189,13 @@ "1; mode=block" ], "x-ratelimit-remaining": [ - "992" + "997.0" ], "x-ratelimit-reset": [ - "356" + "181" ], "x-ratelimit-used": [ - "4" + "3" ], "x-ua-compatible": [ "IE=edge" @@ -206,6 +209,6 @@ } } ], - "recorded_at": "2023-11-27T01:54:04", + "recorded_at": "2025-10-04T21:26:58", "version": 1 } diff --git a/tests/utils.py b/tests/utils.py index 9cd23c5..c2b0675 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -30,9 +30,10 @@ def filter_access_token(response): # pragma: no cover if "api/v1/access_token" not in request_uri or response["status"]["code"] != 200: return response body = response["body"]["string"].decode() + json_body = json.loads(body) for token_key in ["access", "refresh"]: try: - token = json.loads(body)[f"{token_key}_token"] + token = json_body[f"{token_key}_token"] except (KeyError, TypeError, ValueError): continue response["body"]["string"] = response["body"]["string"].replace( diff --git a/uv.lock b/uv.lock index 5873a4a..f025374 100644 --- a/uv.lock +++ b/uv.lock @@ -191,7 +191,7 @@ test = [ { name = "pytest", specifier = ">=8.3.4" }, { name = "pytest-asyncio", specifier = "==1.2.*" }, { name = "pytest-vcr", specifier = "==1.*" }, - { name = "vcrpy", specifier = "==7.0.0" }, + { name = "vcrpy", specifier = "==4.3.1" }, ] type = [ { name = "aiohttp", specifier = "<4" }, @@ -952,6 +952,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083, upload-time = "2025-07-29T22:32:33.881Z" }, ] +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + [[package]] name = "tomli" version = "2.2.1" @@ -1074,17 +1083,18 @@ wheels = [ [[package]] name = "vcrpy" -version = "7.0.0" +version = "4.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, - { name = "urllib3" }, + { name = "six" }, + { name = "urllib3", marker = "python_full_version < '3.10'" }, { name = "wrapt" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/d3/856e06184d4572aada1dd559ddec3bedc46df1f2edc5ab2c91121a2cccdb/vcrpy-7.0.0.tar.gz", hash = "sha256:176391ad0425edde1680c5b20738ea3dc7fb942520a48d2993448050986b3a50", size = 85502, upload-time = "2024-12-31T00:07:57.894Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/d2/07e523cf0532eaee956f40ec17fde9b7f41827f5deaac0d38db5fb10af18/vcrpy-4.3.1.tar.gz", hash = "sha256:24e2d450bf1c2f9f9b4246ee91beb7d58f862a9f2f030514b14783b83c5146ec", size = 81973, upload-time = "2023-05-26T16:04:28.405Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/13/5d/1f15b252890c968d42b348d1e9b0aa12d5bf3e776704178ec37cceccdb63/vcrpy-7.0.0-py2.py3-none-any.whl", hash = "sha256:55791e26c18daa363435054d8b35bd41a4ac441b6676167635d1b37a71dbe124", size = 42321, upload-time = "2024-12-31T00:07:55.277Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d0/11a43916c8c2187685a7143aed5209169e0a484c7c1a061882c4f5d77532/vcrpy-4.3.1-py2.py3-none-any.whl", hash = "sha256:35398f1b373f32340f39d735ea45f40d679ace316f3dddf8cbcbc2f120e6d1d0", size = 40925, upload-time = "2023-05-26T16:04:26.361Z" }, ] [[package]]