fix: possible infinity loop in Apify-Scrapy proxy middleware

Author: vdusek

src/apify/scrapy/middlewares/apify_proxy.py

@@ -93,7 +93,7 @@ def process_exception(
         request: Request,
         exception: Exception,
         spider: Spider,
-    ) -> None | Request:
+    ) -> None:
         """Process an exception that occurs during request processing.
 
         Args:
@@ -102,8 +102,9 @@ def process_exception(
             spider: Scrapy Spider object.
 
         Returns:
-            If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline.
-            Return None otherwise to allow the continuation of request processing.
+            Returning None, meaning Scrapy will continue processing this exception, executing any other
+            process_exception() methods of installed middleware, until no middleware is left and the default
+            exception handling kicks in.
         """
         Actor.log.debug(
             f'ApifyHttpProxyMiddleware.process_exception: request={request}, exception={exception}, spider={spider}',
@@ -114,9 +115,6 @@ def process_exception(
                 f'ApifyHttpProxyMiddleware: TunnelError occurred for request="{request}", '
                 'reason="{exception}", skipping...'
             )
-            return request
-
-        return None
 
     async def _get_new_proxy_url(self: ApifyHttpProxyMiddleware) -> ParseResult:
         """Get a new proxy URL.

tests/unit/scrapy/middlewares/test_apify_proxy.py

@@ -129,24 +129,14 @@ async def mock_get_new_proxy_url() -> ParseResult:
 
 
 @pytest.mark.parametrize(
-    ('exception', 'none_returned_values_is_expected'),
-    [
-        (TunnelError(), False),
-        (ValueError(), True),
-    ],
+    'exception',
+    [TunnelError(), ValueError()],
 )
 def test__process_exception(
     middleware: ApifyHttpProxyMiddleware,
     spider: DummySpider,
     dummy_request: Request,
     exception: Exception,
-    *,
-    none_returned_values_is_expected: bool,
 ) -> None:
-    returned_value = middleware.process_exception(dummy_request, exception, spider)
-
-    if none_returned_values_is_expected:
-        assert returned_value is None
-
-    else:
-        assert returned_value == dummy_request
+    returned_value = middleware.process_exception(dummy_request, exception, spider)  # type: ignore
+    assert returned_value is None