handle malformed XML

Author: addie9800

src/fundus/scraping/url.py

@@ -11,7 +11,7 @@
 import feedparser
 import lxml.html
 import validators
-from lxml.etree import XMLParser, XPath
+from lxml.etree import XMLParser, XMLSyntaxError, XPath
 from requests import ConnectionError, HTTPError, ReadTimeout
 
 from fundus.logging import create_logger
@@ -164,7 +164,7 @@ class Sitemap(URLSource):
     _decompressor: ClassVar[_ArchiveDecompressor] = _ArchiveDecompressor()
     _sitemap_selector: ClassVar[XPath] = XPath("//*[local-name()='sitemap']/*[local-name()='loc']")
     _url_selector: ClassVar[XPath] = XPath("//*[local-name()='url']/*[local-name()='loc']")
-    _parser = XMLParser(strip_cdata=False)
+    _parser = XMLParser(strip_cdata=False, recover=True)
 
     def __iter__(self) -> Iterator[str]:
         def yield_recursive(sitemap_url: str) -> Iterator[str]:
@@ -194,7 +194,11 @@ def yield_recursive(sitemap_url: str) -> Iterator[str]:
             if not content:
                 logger.warning(f"Warning! Empty sitemap at {sitemap_url!r}")
                 return
-            tree = lxml.etree.fromstring(content, parser=self._parser)
+            try:
+                tree = lxml.etree.fromstring(content, parser=self._parser)
+            except XMLSyntaxError:
+                logger.warning(f"Warning! Couldn't parse sitemap {sitemap_url!r} because of invalid XML")
+                return
             urls = [node.text for node in self._url_selector(tree)]
             if urls:
                 for new_url in reversed(urls) if self.reverse else urls: