Merge pull request #801 from marten-ti/add-row-publisher

add Rest Of World Publisher

Author: addie9800

docs/supported_publishers.md

@@ -3433,6 +3433,25 @@
         <code>description</code>
       </td>
     </tr>
+    <tr>
+      <td>
+        <code>RestOfWorld</code>
+      </td>
+      <td>
+        <div>Rest of World</div>
+      </td>
+      <td>
+        <a href="https://restofworld.org/">
+          <span>restofworld.org</span>
+        </a>
+      </td>
+      <td>
+        <code>en</code>
+      </td>
+      <td>&#160;</td>
+      <td>&#160;</td>
+      <td>&#160;</td>
+    </tr>
     <tr>
       <td>
         <code>Reuters</code>

src/fundus/publishers/us/__init__.py

@@ -9,6 +9,7 @@
 from .free_beacon import FreeBeaconParser
 from .la_times import LATimesParser
 from .occupy_democrats import OccupyDemocratsParser
+from .rest_of_world import RestOfWorldParser
 from .reuters import ReutersParser
 from .rolling_stone import RollingStoneParser
 from .techcrunch import TechCrunchParser
@@ -274,3 +275,14 @@ class US(metaclass=PublisherGroup):
             Sitemap("https://www.wired.com/sitemap-archive-1.xml"),
         ],
     )
+
+    RestOfWorld = Publisher(
+        name="Rest of World",
+        domain="https://restofworld.org/",
+        parser=RestOfWorldParser,
+        url_filter=inverse(regex_filter(r"restofworld\.org\/20\d{2}\/")),
+        sources=[
+            RSSFeed("https://restofworld.org/feed/latest/"),
+            Sitemap("https://restofworld.org/sitemap.xml"),
+        ],
+    )

src/fundus/publishers/us/rest_of_world.py

@@ -0,0 +1,60 @@
+from datetime import date, datetime
+from typing import List, Optional
+
+from lxml.cssselect import CSSSelector
+from lxml.etree import XPath
+
+from fundus.parser import ArticleBody, BaseParser, Image, ParserProxy, attribute
+from fundus.parser.utility import (
+    extract_article_body_with_selector,
+    generic_author_parsing,
+    generic_date_parsing,
+    generic_topic_parsing,
+    image_extraction,
+)
+
+
+class RestOfWorldParser(ParserProxy):
+    class V1(BaseParser):
+        _summary_selector: CSSSelector = CSSSelector("div.post-subheader__summary li, p.post-header__text__dek")
+        _paragraph_selector: CSSSelector = CSSSelector("div.post-content > p")
+        _subheadline_selector: CSSSelector = CSSSelector("div.post-content > h2")
+
+        @attribute
+        def body(self) -> Optional[ArticleBody]:
+            return extract_article_body_with_selector(
+                self.precomputed.doc,
+                summary_selector=self._summary_selector,
+                subheadline_selector=self._subheadline_selector,
+                paragraph_selector=self._paragraph_selector,
+            )
+
+        @attribute
+        def authors(self) -> List[str]:
+            return generic_author_parsing(self.precomputed.ld.xpath_search("NewsArticle/author"))
+
+        @attribute
+        def publishing_date(self) -> Optional[datetime]:
+            return generic_date_parsing(self.precomputed.ld.xpath_search("NewsArticle/datePublished", scalar=True))
+
+        @attribute
+        def title(self) -> Optional[str]:
+            return self.precomputed.ld.xpath_search("NewsArticle/headline", scalar=True)
+
+        @attribute
+        def topics(self) -> List[str]:
+            return generic_topic_parsing(self.precomputed.ld.xpath_search("NewsArticle/keywords", scalar=True))
+
+        @attribute
+        def images(self) -> List[Image]:
+            return image_extraction(
+                doc=self.precomputed.doc,
+                paragraph_selector=self._paragraph_selector,
+                image_selector=XPath("//figure//img | //img[@src or @data-src]"),
+                caption_selector=XPath("./ancestor::figure[1]//*[contains(@class,'figcaption__caption')][1]"),
+                author_selector=XPath(
+                    "(./ancestor::figure[1]//*[(contains(@class,'figcaption__credit') "
+                    "or contains(@class,'credit') or contains(@class,'byline'))])[last()]"
+                ),
+                relative_urls=True,
+            )