Merge pull request #789 from flairNLP/generalize-sitemap-selectors

Generalize Sitemap Selectors

Author: addie9800

src/fundus/publishers/tz/__init__.py

@@ -15,8 +15,7 @@ class TZ(metaclass=PublisherGroup):
         sources=[
             Sitemap(
                 "https://dailynews.co.tz/wp-sitemap.xml",
-                sitemap_filter=inverse(regex_filter("sitemap-posts-post")),
-                reverse=True,
+                sitemap_filter=inverse(regex_filter("post-sitemap")),
                 languages={"en"},
             ),
         ],
@@ -27,9 +26,8 @@ class TZ(metaclass=PublisherGroup):
         parser=DailyNewsTZParser,
         sources=[
             Sitemap(
-                "https://habarileo.co.tz/wp-sitemap.xml",
-                sitemap_filter=inverse(regex_filter("sitemap-posts-post")),
-                reverse=True,
+                "https://habarileo.co.tz/sitemap.xml",
+                sitemap_filter=inverse(regex_filter("post-sitemap")),
             ),
         ],
     )

src/fundus/scraping/url.py

@@ -11,12 +11,10 @@
 import feedparser
 import lxml.html
 import validators
-from lxml.cssselect import CSSSelector
-from lxml.etree import XPath
+from lxml.etree import XMLParser, XPath
 from requests import ConnectionError, HTTPError, ReadTimeout
 
 from fundus.logging import create_logger
-from fundus.parser.utility import generic_nodes_to_text
 from fundus.scraping.filter import URLFilter, inverse
 from fundus.scraping.session import _default_header, session_handler
 
@@ -164,8 +162,9 @@ class Sitemap(URLSource):
     sitemap_filter: URLFilter = lambda url: not bool(url)
 
     _decompressor: ClassVar[_ArchiveDecompressor] = _ArchiveDecompressor()
-    _sitemap_selector: ClassVar[XPath] = CSSSelector("sitemap > loc")
-    _url_selector: ClassVar[XPath] = CSSSelector("url > loc")
+    _sitemap_selector: ClassVar[XPath] = XPath("//*[local-name()='sitemap']/*[local-name()='loc']")
+    _url_selector: ClassVar[XPath] = XPath("//*[local-name()='url']/*[local-name()='loc']")
+    _parser = XMLParser(strip_cdata=False)
 
     def __iter__(self) -> Iterator[str]:
         def yield_recursive(sitemap_url: str) -> Iterator[str]:
@@ -195,13 +194,13 @@ def yield_recursive(sitemap_url: str) -> Iterator[str]:
             if not content:
                 logger.warning(f"Warning! Empty sitemap at {sitemap_url!r}")
                 return
-            tree = lxml.html.fromstring(content)
-            urls = generic_nodes_to_text(self._url_selector(tree), normalize=True)
+            tree = lxml.etree.fromstring(content, parser=self._parser)
+            urls = [node.text for node in self._url_selector(tree)]
             if urls:
                 for new_url in reversed(urls) if self.reverse else urls:
                     yield clean_url(new_url)
             elif self.recursive:
-                sitemap_locs = [node.text_content() for node in self._sitemap_selector(tree)]
+                sitemap_locs = [node.text for node in self._sitemap_selector(tree)]
                 filtered_locs = list(filter(inverse(self.sitemap_filter), sitemap_locs))
                 for loc in reversed(filtered_locs) if self.reverse else filtered_locs:
                     yield from yield_recursive(loc)