chore: replace respx mocks with pytest-httpserver in tests

pyproject.toml

@@ -52,6 +52,7 @@ dev = [
     "pytest-timeout>=2.4.0",
     "pytest-xdist~=3.8.0",
     "pytest~=8.4.0",
+    "pytest-httpserver>=1.1.3",
     "redbaron~=0.9.0",
     "respx~=0.22.0",
     "ruff~=0.12.0",

tests/unit/conftest.py

@@ -0,0 +1,33 @@
+from collections.abc import Iterable, Iterator
+from logging import getLogger
+
+import pytest
+from pytest_httpserver import HTTPServer
+
+
+@pytest.fixture(scope='session')
+def make_httpserver() -> Iterable[HTTPServer]:
+    werkzeug_logger = getLogger('werkzeug')
+    werkzeug_logger.disabled = True
+
+    server = HTTPServer(threaded=True, host='127.0.0.1')
+    server.start()
+    yield server
+    server.clear()  # type: ignore[no-untyped-call]
+    if server.is_running():
+        server.stop()  # type: ignore[no-untyped-call]
+
+
+@pytest.fixture
+def httpserver(make_httpserver: HTTPServer) -> Iterable[HTTPServer]:
+    server = make_httpserver
+    yield server
+    server.clear()  # type: ignore[no-untyped-call]
+
+
+@pytest.fixture
+def patch_basic_url(httpserver: HTTPServer, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
+    server_url = httpserver.url_for('/').removesuffix('/')
+    monkeypatch.setattr('apify_client.client.DEFAULT_API_URL', server_url)
+    yield
+    monkeypatch.undo()

tests/unit/test_client_errors.py

@@ -1,49 +1,49 @@
-import json
-from collections.abc import Generator
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
-import httpx
 import pytest
-import respx
 
 from apify_client._errors import ApifyApiError
 from apify_client._http_client import HTTPClient, HTTPClientAsync
 
-_TEST_URL = 'http://example.com'
+if TYPE_CHECKING:
+    from pytest_httpserver import HTTPServer
+
+_TEST_PATH = '/errors'
 _EXPECTED_MESSAGE = 'some_message'
 _EXPECTED_TYPE = 'some_type'
 _EXPECTED_DATA = {
     'invalidItems': {'0': ["should have required property 'name'"], '1': ["should have required property 'name'"]}
 }
 
 
-@pytest.fixture(autouse=True)
-def mocked_response() -> Generator[respx.MockRouter]:
-    response_content = json.dumps(
-        {'error': {'message': _EXPECTED_MESSAGE, 'type': _EXPECTED_TYPE, 'data': _EXPECTED_DATA}}
+@pytest.fixture
+def test_endpoint(httpserver: HTTPServer) -> str:
+    httpserver.expect_request(_TEST_PATH).respond_with_json(
+        {'error': {'message': _EXPECTED_MESSAGE, 'type': _EXPECTED_TYPE, 'data': _EXPECTED_DATA}}, status=400
     )
-    with respx.mock() as respx_mock:
-        respx_mock.get(_TEST_URL).mock(return_value=httpx.Response(400, content=response_content))
-        yield respx_mock
+    return str(httpserver.url_for(_TEST_PATH))
 
 
-def test_client_apify_api_error_with_data() -> None:
+def test_client_apify_api_error_with_data(test_endpoint: str) -> None:
     """Test that client correctly throws ApifyApiError with error data from response."""
     client = HTTPClient()
 
     with pytest.raises(ApifyApiError) as e:
-        client.call(method='GET', url=_TEST_URL)
+        client.call(method='GET', url=test_endpoint)
 
     assert e.value.message == _EXPECTED_MESSAGE
     assert e.value.type == _EXPECTED_TYPE
     assert e.value.data == _EXPECTED_DATA
 
 
-async def test_async_client_apify_api_error_with_data() -> None:
+async def test_async_client_apify_api_error_with_data(test_endpoint: str) -> None:
     """Test that async client correctly throws ApifyApiError with error data from response."""
     client = HTTPClientAsync()
 
     with pytest.raises(ApifyApiError) as e:
-        await client.call(method='GET', url=_TEST_URL)
+        await client.call(method='GET', url=test_endpoint)
 
     assert e.value.message == _EXPECTED_MESSAGE
     assert e.value.type == _EXPECTED_TYPE

tests/unit/test_client_request_queue.py

@@ -1,9 +1,16 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
 import pytest
-import respx
 
 import apify_client
 from apify_client import ApifyClient, ApifyClientAsync
 
+if TYPE_CHECKING:
+    from pytest_httpserver import HTTPServer
+
 _PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT = """{
   "data": {
     "processedRequests": [
@@ -25,12 +32,11 @@
 }"""
 
 
-@respx.mock
-async def test_batch_not_processed_raises_exception_async() -> None:
+@pytest.mark.usefixtures('patch_basic_url')
+async def test_batch_not_processed_raises_exception_async(httpserver: HTTPServer) -> None:
     """Test that client exceptions are not silently ignored"""
-    client = ApifyClientAsync(token='')
-
-    respx.route(method='POST', host='api.apify.com').mock(return_value=respx.MockResponse(401))
+    client = ApifyClientAsync(token='placeholder_token')
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(status=401)
     requests = [
         {'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},
         {'uniqueKey': 'http://example.com/2', 'url': 'http://example.com/2', 'method': 'GET'},
@@ -41,12 +47,12 @@ async def test_batch_not_processed_raises_exception_async() -> None:
         await rq_client.batch_add_requests(requests=requests)
 
 
-@respx.mock
-async def test_batch_processed_partially_async() -> None:
-    client = ApifyClientAsync(token='')
+@pytest.mark.usefixtures('patch_basic_url')
+async def test_batch_processed_partially_async(httpserver: HTTPServer) -> None:
+    client = ApifyClientAsync(token='placeholder_token')
 
-    respx.route(method='POST', host='api.apify.com').mock(
-        return_value=respx.MockResponse(200, content=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT)
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(
+        status=200, response_data=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT
     )
     requests = [
         {'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},
@@ -59,12 +65,12 @@ async def test_batch_processed_partially_async() -> None:
     assert response['unprocessedRequests'] == [requests[1]]
 
 
-@respx.mock
-def test_batch_not_processed_raises_exception_sync() -> None:
+@pytest.mark.usefixtures('patch_basic_url')
+def test_batch_not_processed_raises_exception_sync(httpserver: HTTPServer) -> None:
     """Test that client exceptions are not silently ignored"""
-    client = ApifyClient(token='')
+    client = ApifyClient(token='placeholder_token')
 
-    respx.route(method='POST', host='api.apify.com').mock(return_value=respx.MockResponse(401))
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(status=401)
     requests = [
         {'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},
         {'uniqueKey': 'http://example.com/2', 'url': 'http://example.com/2', 'method': 'GET'},
@@ -75,12 +81,12 @@ def test_batch_not_processed_raises_exception_sync() -> None:
         rq_client.batch_add_requests(requests=requests)
 
 
-@respx.mock
-async def test_batch_processed_partially_sync() -> None:
-    client = ApifyClient(token='')
+@pytest.mark.usefixtures('patch_basic_url')
+async def test_batch_processed_partially_sync(httpserver: HTTPServer) -> None:
+    client = ApifyClient(token='placeholder_token')
 
-    respx.route(method='POST', host='api.apify.com').mock(
-        return_value=respx.MockResponse(200, content=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT)
+    httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(
+        status=200, response_data=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT
     )
     requests = [
         {'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},

tests/unit/test_client_timeouts.py

@@ -1,10 +1,13 @@
 from __future__ import annotations
 
+import time
 from functools import partial
+from typing import TYPE_CHECKING
+from unittest.mock import Mock
 
 import pytest
 import respx
-from httpx import Request, Response, TimeoutException
+from werkzeug import Response as WerkzeugResponse
 
 from apify_client import ApifyClient
 from apify_client._http_client import HTTPClient, HTTPClientAsync
@@ -13,13 +16,17 @@
 from apify_client.clients.resource_clients import dataset, request_queue
 from apify_client.clients.resource_clients import key_value_store as kvs
 
+if TYPE_CHECKING:
+    from httpx import Request, Response
+    from pytest_httpserver import HTTPServer
+    from werkzeug import Request as WerkzeugRequest
+
 
 class EndOfTestError(Exception):
     """Custom exception that is raised after the relevant part of the code is executed to stop the test."""
 
 
-@respx.mock
-async def test_dynamic_timeout_async_client() -> None:
+async def test_dynamic_timeout_async_client(httpserver: HTTPServer) -> None:
     """Tests timeout values for request with retriable errors.
 
     Values should increase with each attempt, starting from initial call value and bounded by the client timeout value.
@@ -28,27 +35,35 @@ async def test_dynamic_timeout_async_client() -> None:
     call_timeout = 1
     client_timeout = 5
     expected_timeouts = iter((call_timeout, 2, 4, client_timeout))
+    retry_counter_mock = Mock()
+
+    def slow_handler(_request: WerkzeugRequest) -> WerkzeugResponse:
+        timeout = next(expected_timeouts)
+        should_raise = next(should_raise_error)
+        # Counter for retries
+        retry_counter_mock()
+
+        if should_raise:
+            # We expect longer than the client is willing to wait. This will cause a timeout on the client side.
+            time.sleep(timeout + 0.02)
 
-    def check_timeout(request: Request) -> Response:
-        expected_timeout = next(expected_timeouts)
-        assert request.extensions['timeout'] == {
-            'connect': expected_timeout,
-            'pool': expected_timeout,
-            'read': expected_timeout,
-            'write': expected_timeout,
-        }
-        if next(should_raise_error):
-            raise TimeoutException('This error can be retried')
-        return Response(200)
-
-    respx.get('https://example.com').mock(side_effect=check_timeout)
-    await HTTPClientAsync(timeout_secs=client_timeout).call(
-        method='GET', url='https://example.com', timeout_secs=call_timeout
+        return WerkzeugResponse('200 OK')
+
+    httpserver.expect_request('/async_timeout', method='GET').respond_with_handler(slow_handler)
+
+    server_url = str(httpserver.url_for('/async_timeout'))
+    response = await HTTPClientAsync(timeout_secs=client_timeout).call(
+        method='GET', url=server_url, timeout_secs=call_timeout
     )
 
+    # Check that the retry counter was called the expected number of times
+    # (4 times: 3 retries + 1 final successful call)
+    assert retry_counter_mock.call_count == 4
+    # Check that the response is successful
+    assert response.status_code == 200
 
-@respx.mock
-def test_dynamic_timeout_sync_client() -> None:
+
+def test_dynamic_timeout_sync_client(httpserver: HTTPServer) -> None:
     """Tests timeout values for request with retriable errors.
 
     Values should increase with each attempt, starting from initial call value and bounded by the client timeout value.
@@ -57,21 +72,31 @@ def test_dynamic_timeout_sync_client() -> None:
     call_timeout = 1
     client_timeout = 5
     expected_timeouts = iter((call_timeout, 2, 4, client_timeout))
+    retry_counter_mock = Mock()
+
+    def slow_handler(_request: WerkzeugRequest) -> WerkzeugResponse:
+        timeout = next(expected_timeouts)
+        should_raise = next(should_raise_error)
+        # Counter for retries
+        retry_counter_mock()
+
+        if should_raise:
+            # We expect longer than the client is willing to wait. This will cause a timeout on the client side.
+            time.sleep(timeout + 0.02)
+
+        return WerkzeugResponse('200 OK')
+
+    httpserver.expect_request('/sync_timeout', method='GET').respond_with_handler(slow_handler)
+
+    server_url = str(httpserver.url_for('/sync_timeout'))
 
-    def check_timeout(request: Request) -> Response:
-        expected_timeout = next(expected_timeouts)
-        assert request.extensions['timeout'] == {
-            'connect': expected_timeout,
-            'pool': expected_timeout,
-            'read': expected_timeout,
-            'write': expected_timeout,
-        }
-        if next(should_raise_error):
-            raise TimeoutException('This error can be retired')
-        return Response(200)
+    response = HTTPClient(timeout_secs=client_timeout).call(method='GET', url=server_url, timeout_secs=call_timeout)
 
-    respx.get('https://example.com').mock(side_effect=check_timeout)
-    HTTPClient(timeout_secs=client_timeout).call(method='GET', url='https://example.com', timeout_secs=call_timeout)
+    # Check that the retry counter was called the expected number of times
+    # (4 times: 3 retries + 1 final successful call)
+    assert retry_counter_mock.call_count == 4
+    # Check that the response is successful
+    assert response.status_code == 200
 
 
 def assert_timeout(expected_timeout: int, request: Request) -> Response:
@@ -122,6 +147,8 @@ def assert_timeout(expected_timeout: int, request: Request) -> Response:
 ]
 
 
+# This test will probably need to be reworked or skipped when switching to `impit`.
+# Without the mock library, it's difficult to reproduce, maybe with monkeypatch?
 @pytest.mark.parametrize(
     ('client_type', 'method', 'expected_timeout', 'kwargs'),
     _timeout_params,
@@ -139,6 +166,8 @@ def test_specific_timeouts_for_specific_endpoints_sync(
         getattr(client, method)(**kwargs)
 
 
+# This test will probably need to be reworked or skipped when switching to `impit`.
+# Without the mock library, it's difficult to reproduce, maybe with monkeypatch?
 @pytest.mark.parametrize(
     ('client_type', 'method', 'expected_timeout', 'kwargs'),
     _timeout_params,