_utils._get_float_setting (#287)

Author: elacuesta

scrapy_playwright/_utils.py

@@ -7,6 +7,7 @@
 import scrapy
 from playwright.async_api import Error, Page, Request, Response
 from scrapy.http.headers import Headers
+from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from twisted.internet.defer import Deferred
 from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding
@@ -85,6 +86,13 @@ async def _get_page_content(
         raise
 
 
+def _get_float_setting(settings: Settings, key: str) -> Optional[float]:
+    try:
+        return float(settings[key])
+    except Exception:
+        return None
+
+
 async def _get_header_value(
     resource: Union[Request, Response],
     header_name: str,

scrapy_playwright/handler.py

@@ -36,6 +36,7 @@
     _ThreadedLoopAdapter,
     _deferred_from_coro,
     _encode_body,
+    _get_float_setting,
     _get_header_value,
     _get_page_content,
     _is_safe_close_error,
@@ -73,7 +74,7 @@ class Config:
     max_pages_per_context: int
     max_contexts: Optional[int]
     startup_context_kwargs: dict
-    navigation_timeout: Optional[float] = None
+    navigation_timeout: Optional[float]
 
     @classmethod
     def from_settings(cls, settings: Settings) -> "Config":
@@ -85,15 +86,15 @@ def from_settings(cls, settings: Settings) -> "Config":
             max_pages_per_context=settings.getint("PLAYWRIGHT_MAX_PAGES_PER_CONTEXT"),
             max_contexts=settings.getint("PLAYWRIGHT_MAX_CONTEXTS") or None,
             startup_context_kwargs=settings.getdict("PLAYWRIGHT_CONTEXTS"),
+            navigation_timeout=_get_float_setting(
+                settings, "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT"
+            ),
         )
         cfg.cdp_kwargs.pop("endpoint_url", None)
         if not cfg.max_pages_per_context:
             cfg.max_pages_per_context = settings.getint("CONCURRENT_REQUESTS")
         if cfg.cdp_url and cfg.launch_options:
             logger.warning("PLAYWRIGHT_CDP_URL is set, ignoring PLAYWRIGHT_LAUNCH_OPTIONS")
-        if "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT" in settings:
-            with suppress(TypeError, ValueError):
-                cfg.navigation_timeout = float(settings["PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT"])
         return cfg
 
 

tests/tests_asyncio/test_utils.py

@@ -1,15 +1,17 @@
 import logging
+from decimal import Decimal
 from unittest import IsolatedAsyncioTestCase
 from unittest.mock import AsyncMock
 
 import pytest
 from playwright.async_api import Error as PlaywrightError
 from scrapy import Spider
 from scrapy.http.headers import Headers
-
+from scrapy.settings import Settings
 from scrapy_playwright._utils import (
     _NAVIGATION_ERROR_MSG,
     _encode_body,
+    _get_float_setting,
     _get_header_value,
     _get_page_content,
     _maybe_await,
@@ -122,20 +124,19 @@ async def test_encode_mismatch(self):
 
 
 class TestHeaderValue(IsolatedAsyncioTestCase):
-    async def test_get_header_ok(self):
+    async def test_get_header_value(self):
         async def _identity(x):
             return x
 
-        resource = AsyncMock()
-        resource.header_value = _identity
-        assert "asdf" == await _get_header_value(resource, "asdf")
-        assert "qwerty" == await _get_header_value(resource, "qwerty")
+        res1 = AsyncMock()
+        res1.header_value = _identity
+        assert "asdf" == await _get_header_value(res1, "asdf")
+        assert "qwerty" == await _get_header_value(res1, "qwerty")
 
-    async def test_get_header_exception(self):
-        resource = AsyncMock()
-        resource.header_value.side_effect = Exception("nope")
-        assert await _get_header_value(resource, "asdf") is None
-        assert await _get_header_value(resource, "qwerty") is None
+        res2 = AsyncMock()
+        res2.header_value.side_effect = Exception("nope")
+        assert await _get_header_value(res2, "asdf") is None
+        assert await _get_header_value(res2, "qwerty") is None
 
 
 class TestMaybeAwait(IsolatedAsyncioTestCase):
@@ -149,3 +150,28 @@ async def _awaitable_identity(x):
         assert await _maybe_await("foo") == "foo"
         assert await _maybe_await("bar") == "bar"
         assert await _maybe_await(1234) == 1234
+
+
+class TestGetFloatSetting(IsolatedAsyncioTestCase):
+    async def test_get_float_setting(self):
+        settings = Settings(
+            {
+                "ZERO": 0,
+                "FLOAT": 1.5,
+                "DECIMAL": Decimal("2.5"),
+                "INT": 3,
+                "NUMERIC_STRING": "123",
+                "NON_NUMERIC_STRING": "asdf",
+                "NONE": None,
+                "LIST": [1, 2, 3],
+            }
+        )
+        assert _get_float_setting(settings, "ZERO") == 0.0
+        assert _get_float_setting(settings, "FLOAT") == 1.5
+        assert _get_float_setting(settings, "DECIMAL") == 2.5
+        assert _get_float_setting(settings, "INT") == 3.0
+        assert _get_float_setting(settings, "NUMERIC_STRING") == 123
+        assert _get_float_setting(settings, "NON_NUMERIC_STRING") is None
+        assert _get_float_setting(settings, "NONE") is None
+        assert _get_float_setting(settings, "LIST") is None
+        assert _get_float_setting(settings, "MISSING_KEY") is None