Merge pull request #115 from ogajduse/feat/add-http-headers-to-request

Author: ogajduse

.pre-commit-config.yaml

@@ -28,6 +28,7 @@ repos:
             voluptuous-stubs,
             types-python-dateutil,
             types-PyYAML,
+            types-requests,
           ]
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.

custom_components/feedparser/sensor.py

@@ -9,12 +9,14 @@
 
 import feedparser  # type: ignore[import]
 import homeassistant.helpers.config_validation as cv
+import requests
 import voluptuous as vol
 from dateutil import parser
 from feedparser import FeedParserDict
 from homeassistant.components.sensor import PLATFORM_SCHEMA, SensorEntity
 from homeassistant.const import CONF_NAME, CONF_SCAN_INTERVAL
 from homeassistant.util import dt
+from requests_file import FileAdapter
 
 if TYPE_CHECKING:
     from homeassistant.core import HomeAssistant
@@ -38,6 +40,7 @@
 DEFAULT_SCAN_INTERVAL = timedelta(hours=1)
 DEFAULT_THUMBNAIL = "https://www.home-assistant.io/images/favicon-192x192-full.png"
 DEFAULT_TOPN = 9999
+USER_AGENT = f"Home Assistant Feed-parser Integration {__version__}"
 
 PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
     {
@@ -124,9 +127,14 @@ def __repr__(self: FeedParserSensor) -> str:
     def update(self: FeedParserSensor) -> None:
         """Parse the feed and update the state of the sensor."""
         _LOGGER.debug("Feed %s: Polling feed data from %s", self.name, self._feed)
-        parsed_feed: FeedParserDict = feedparser.parse(self._feed)
-
-        if not parsed_feed:
+        s: requests.Session = requests.Session()
+        s.mount("file://", FileAdapter())
+        s.headers.update({"User-Agent": USER_AGENT})
+        res: requests.Response = s.get(self._feed)
+        res.raise_for_status()
+        parsed_feed: FeedParserDict = feedparser.parse(res.text)
+
+        if not parsed_feed.entries:
             self._attr_native_value = None
             _LOGGER.warning("Feed %s: No data received.", self.name)
             return

pyproject.toml

@@ -23,7 +23,13 @@ classifiers = [
   "Programming Language :: Python :: 3.11",
 ]
 requires-python = ">=3.11.0"
-dependencies = ["python-dateutil", "feedparser==6.0.10", "homeassistant"]
+dependencies = [
+  "feedparser==6.0.10",
+  "homeassistant",
+  "python-dateutil",
+  "requests-file",
+  "requests",
+]
 
 [project.optional-dependencies]
 dev = [
@@ -34,6 +40,7 @@ dev = [
   "ruff",
   "types-python-dateutil",
   "types-PyYAML",
+  "types-requests",
   "voluptuous-stubs",
   "pyyaml",
 
@@ -97,11 +104,11 @@ select = [
 
 # Q000,ANN,PT009,D,E501,
 ignore = [
-  "D107",    # Missing docstring in __init__
-  "FBT001",  # Boolean positional arg in function definition
-  "D203",    # 1 blank line required before class docstring
-  "D213",    # Multi-line docstring summary should start at the first line
-  "FBT001"   # Boolean positional argument in function definition
+  "D107",   # Missing docstring in __init__
+  "FBT001", # Boolean positional arg in function definition
+  "D203",   # 1 blank line required before class docstring
+  "D213",   # Multi-line docstring summary should start at the first line
+  "FBT001", # Boolean positional argument in function definition
 ]
 
 # Allow autofix for all enabled rules (when `--fix`) is provided.

tests/constants.py

@@ -93,3 +93,14 @@
 DEFAULT_EXCLUSIONS: list[str] = []
 DEFAULT_INCLUSIONS = ["image", "title", "link", "published"]
 DATE_FORMAT = "%a, %d %b %Y %H:%M:%S UTC%z"
+
+URLS_HEADERS_REQUIRED = [
+    {
+        "name": "elcomercio_gijon",
+        "url": "https://www.elcomercio.es/rss/2.0/?section=gijon",
+    },
+    {
+        "name": "nasdaq_options",
+        "url": "https://www.nasdaq.com/feed/rssoutbound?category=Options",
+    },
+]

tests/test_sensors.py

@@ -6,7 +6,7 @@
 
 import feedparser
 import pytest
-from constants import DATE_FORMAT
+from constants import DATE_FORMAT, URLS_HEADERS_REQUIRED
 from feedsource import FeedSource
 
 from custom_components.feedparser.sensor import (
@@ -157,3 +157,24 @@ def test_check_duplicates(feed_sensor: FeedParserSensor) -> None:
     feed_sensor.update()
     after_second_update = len(feed_sensor.feed_entries)
     assert after_first_update == after_second_update
+
+
+@pytest.mark.parametrize(
+    "online_feed",
+    URLS_HEADERS_REQUIRED,
+    ids=lambda feed_url: feed_url["name"],
+)
+def test_fetch_data_headers_required(online_feed: dict) -> None:
+    """Test fetching feed from remote server that requires request with headers."""
+    feed_sensor = FeedParserSensor(
+        feed=online_feed["url"],
+        name=online_feed["name"],
+        date_format=DATE_FORMAT,
+        local_time=False,
+        show_topn=9999,
+        inclusions=["image", "title", "link", "published"],
+        exclusions=[],
+        scan_interval=DEFAULT_SCAN_INTERVAL,
+    )
+    feed_sensor.update()
+    assert feed_sensor.feed_entries