chore: replace respx mocks with pytest-httpserver in tests (#454)

- Replace `respx` mocks with `pytest-httpserver` in tests

Author: Mantisus

pyproject.toml

@@ -52,11 +52,13 @@ dev = [
     "pytest-timeout>=2.4.0",
     "pytest-xdist~=3.8.0",
     "pytest~=8.4.0",
+    "pytest-httpserver>=1.1.3",
     "redbaron~=0.9.0",
     "respx~=0.22.0",
     "ruff~=0.12.0",
     "setuptools", # setuptools are used by pytest but not explicitly required
     "types-colorama~=0.4.15.20240106",
+    "werkzeug~=3.0.0", # Werkzeug is used by pytest-httpserver
 ]
 
 [tool.hatch.build.targets.wheel]

tests/integration/__init__.py

@@ -0,0 +1,33 @@
+from collections.abc import Iterable, Iterator
+from logging import getLogger
+
+import pytest
+from pytest_httpserver import HTTPServer
+
+
+@pytest.fixture(scope='session')
+def make_httpserver() -> Iterable[HTTPServer]:
+    werkzeug_logger = getLogger('werkzeug')
+    werkzeug_logger.disabled = True
+
+    server = HTTPServer(threaded=True, host='127.0.0.1')
+    server.start()
+    yield server
+    server.clear()  # type: ignore[no-untyped-call]
+    if server.is_running():
+        server.stop()  # type: ignore[no-untyped-call]
+
+
+@pytest.fixture
+def httpserver(make_httpserver: HTTPServer) -> Iterable[HTTPServer]:
+    server = make_httpserver
+    yield server
+    server.clear()  # type: ignore[no-untyped-call]
+
+
+@pytest.fixture
+def patch_basic_url(httpserver: HTTPServer, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    server_url = httpserver.url_for('/').removesuffix('/')
+    monkeypatch.setattr('apify_client.client.DEFAULT_API_URL', server_url)
+    yield
+    monkeypatch.undo()

tests/unit/__init__.py

@@ -1,14 +1,22 @@
+from __future__ import annotations
+
 import json
-from collections.abc import AsyncIterator, Generator, Iterator
+import time
+from typing import TYPE_CHECKING
 
-import httpx
 import pytest
-import respx
+from werkzeug import Response
 
 from apify_client._errors import ApifyApiError
 from apify_client._http_client import HTTPClient, HTTPClientAsync
 
-_TEST_URL = 'http://example.com'
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from pytest_httpserver import HTTPServer
+    from werkzeug import Request
+
+_TEST_PATH = '/errors'
 _EXPECTED_MESSAGE = 'some_message'
 _EXPECTED_TYPE = 'some_type'
 _EXPECTED_DATA = {
@@ -27,86 +35,80 @@
 
 
 @pytest.fixture
-def mocked_response() -> Generator[respx.MockRouter]:
-    response_content = json.dumps(
-        {'error': {'message': _EXPECTED_MESSAGE, 'type': _EXPECTED_TYPE, 'data': _EXPECTED_DATA}}
+def test_endpoint(httpserver: HTTPServer) -> str:
+    httpserver.expect_request(_TEST_PATH).respond_with_json(
+        {'error': {'message': _EXPECTED_MESSAGE, 'type': _EXPECTED_TYPE, 'data': _EXPECTED_DATA}}, status=400
     )
-    with respx.mock() as respx_mock:
-        respx_mock.get(_TEST_URL).mock(return_value=httpx.Response(400, content=response_content))
-        yield respx_mock
+    return str(httpserver.url_for(_TEST_PATH))
+
+
+def streaming_handler(_request: Request) -> Response:
+    """Handler for streaming log requests."""
 
+    def generate_response() -> Iterator[bytes]:
+        for i in range(len(RAW_ERROR)):
+            yield RAW_ERROR[i : i + 1]
+            time.sleep(0.01)
 
-@pytest.mark.usefixtures('mocked_response')
-def test_client_apify_api_error_with_data() -> None:
+    return Response(
+        response=(RAW_ERROR[i : i + 1] for i in range(len(RAW_ERROR))),
+        status=403,
+        mimetype='application/octet-stream',
+        headers={'Content-Length': str(len(RAW_ERROR))},
+    )
+
+
+def test_client_apify_api_error_with_data(test_endpoint: str) -> None:
     """Test that client correctly throws ApifyApiError with error data from response."""
     client = HTTPClient()
 
     with pytest.raises(ApifyApiError) as e:
-        client.call(method='GET', url=_TEST_URL)
+        client.call(method='GET', url=test_endpoint)
 
     assert e.value.message == _EXPECTED_MESSAGE
     assert e.value.type == _EXPECTED_TYPE
     assert e.value.data == _EXPECTED_DATA
 
 
-@pytest.mark.usefixtures('mocked_response')
-async def test_async_client_apify_api_error_with_data() -> None:
+async def test_async_client_apify_api_error_with_data(test_endpoint: str) -> None:
     """Test that async client correctly throws ApifyApiError with error data from response."""
     client = HTTPClientAsync()
 
     with pytest.raises(ApifyApiError) as e:
-        await client.call(method='GET', url=_TEST_URL)
+        await client.call(method='GET', url=test_endpoint)
 
     assert e.value.message == _EXPECTED_MESSAGE
     assert e.value.type == _EXPECTED_TYPE
     assert e.value.data == _EXPECTED_DATA
 
 
-def test_client_apify_api_error_streamed() -> None:
+def test_client_apify_api_error_streamed(httpserver: HTTPServer) -> None:
     """Test that client correctly throws ApifyApiError when the response has stream."""
 
     error = json.loads(RAW_ERROR.decode())
 
-    class ByteStream(httpx._types.SyncByteStream):
-        def __iter__(self) -> Iterator[bytes]:
-            yield RAW_ERROR
-
-        def close(self) -> None:
-            pass
-
-    stream_url = 'http://some-stream-url.com'
-
     client = HTTPClient()
 
-    with respx.mock() as respx_mock:
-        respx_mock.get(url=stream_url).mock(return_value=httpx.Response(stream=ByteStream(), status_code=403))
-        with pytest.raises(ApifyApiError) as e:
-            client.call(method='GET', url=stream_url, stream=True, parse_response=False)
+    httpserver.expect_request('/stream_error').respond_with_handler(streaming_handler)
+
+    with pytest.raises(ApifyApiError) as e:
+        client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True, parse_response=False)
 
     assert e.value.message == error['error']['message']
     assert e.value.type == error['error']['type']
 
 
-async def test_async_client_apify_api_error_streamed() -> None:
+async def test_async_client_apify_api_error_streamed(httpserver: HTTPServer) -> None:
     """Test that async client correctly throws ApifyApiError when the response has stream."""
 
     error = json.loads(RAW_ERROR.decode())
 
-    class AsyncByteStream(httpx._types.AsyncByteStream):
-        async def __aiter__(self) -> AsyncIterator[bytes]:
-            yield RAW_ERROR
-
-        async def aclose(self) -> None:
-            pass
-
-    stream_url = 'http://some-stream-url.com'
-
     client = HTTPClientAsync()
 
-    with respx.mock() as respx_mock:
-        respx_mock.get(url=stream_url).mock(return_value=httpx.Response(stream=AsyncByteStream(), status_code=403))
-        with pytest.raises(ApifyApiError) as e:
-            await client.call(method='GET', url=stream_url, stream=True, parse_response=False)
+    httpserver.expect_request('/stream_error').respond_with_handler(streaming_handler)
+
+    with pytest.raises(ApifyApiError) as e:
+        await client.call(method='GET', url=httpserver.url_for('/stream_error'), stream=True, parse_response=False)
 
     assert e.value.message == error['error']['message']
     assert e.value.type == error['error']['type']

tests/unit/conftest.py

@@ -1,9 +1,16 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
 import pytest
-import respx
 
 import apify_client
 from apify_client import ApifyClient, ApifyClientAsync
 
+if TYPE_CHECKING:
+    from pytest_httpserver import HTTPServer
+
 _PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT = """{
   "data": {
     "processedRequests": [
@@ -25,12 +32,11 @@
 }"""
 
 
-@respx.mock
-async def test_batch_not_processed_raises_exception_async() -> None:
+@pytest.mark.usefixtures('patch_basic_url')
+async def test_batch_not_processed_raises_exception_async(httpserver: HTTPServer) -> None:
     """Test that client exceptions are not silently ignored"""
-    client = ApifyClientAsync(token='')
-
-    respx.route(method='POST', host='api.apify.com').mock(return_value=respx.MockResponse(401))
+    client = ApifyClientAsync(token='placeholder_token')
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(status=401)
     requests = [
         {'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},
         {'uniqueKey': 'http://example.com/2', 'url': 'http://example.com/2', 'method': 'GET'},
@@ -41,12 +47,12 @@ async def test_batch_not_processed_raises_exception_async() -> None:
         await rq_client.batch_add_requests(requests=requests)
 
 
-@respx.mock
-async def test_batch_processed_partially_async() -> None:
-    client = ApifyClientAsync(token='')
+@pytest.mark.usefixtures('patch_basic_url')
+async def test_batch_processed_partially_async(httpserver: HTTPServer) -> None:
+    client = ApifyClientAsync(token='placeholder_token')
 
-    respx.route(method='POST', host='api.apify.com').mock(
-        return_value=respx.MockResponse(200, content=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT)
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(
+        status=200, response_data=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT
     )
     requests = [
         {'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},
@@ -59,12 +65,12 @@ async def test_batch_processed_partially_async() -> None:
     assert response['unprocessedRequests'] == [requests[1]]
 
 
-@respx.mock
-def test_batch_not_processed_raises_exception_sync() -> None:
+@pytest.mark.usefixtures('patch_basic_url')
+def test_batch_not_processed_raises_exception_sync(httpserver: HTTPServer) -> None:
     """Test that client exceptions are not silently ignored"""
-    client = ApifyClient(token='')
+    client = ApifyClient(token='placeholder_token')
 
-    respx.route(method='POST', host='api.apify.com').mock(return_value=respx.MockResponse(401))
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(status=401)
     requests = [
         {'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},
         {'uniqueKey': 'http://example.com/2', 'url': 'http://example.com/2', 'method': 'GET'},
@@ -75,12 +81,12 @@ def test_batch_not_processed_raises_exception_sync() -> None:
         rq_client.batch_add_requests(requests=requests)
 
 
-@respx.mock
-async def test_batch_processed_partially_sync() -> None:
-    client = ApifyClient(token='')
+@pytest.mark.usefixtures('patch_basic_url')
+async def test_batch_processed_partially_sync(httpserver: HTTPServer) -> None:
+    client = ApifyClient(token='placeholder_token')
 
-    respx.route(method='POST', host='api.apify.com').mock(
-        return_value=respx.MockResponse(200, content=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT)
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(
+        status=200, response_data=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT
     )
     requests = [
         {'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},