Merge pull request #790 from flairNLP/update-sources-pt

Update sources ThePortugalNews

Author: addie9800

docs/supported_publishers.md

@@ -2661,7 +2661,22 @@
         </a>
       </td>
       <td>
+        <code>ar</code>
+        <code>br</code>
+        <code>de</code>
         <code>en</code>
+        <code>es</code>
+        <code>fi</code>
+        <code>fr</code>
+        <code>he</code>
+        <code>it</code>
+        <code>nl</code>
+        <code>pl</code>
+        <code>pt</code>
+        <code>ru</code>
+        <code>se</code>
+        <code>tr</code>
+        <code>zh</code>
       </td>
       <td>
         <code>topics</code>

src/fundus/publishers/pt/__init__.py

@@ -1,5 +1,5 @@
 from fundus.publishers.base_objects import Publisher, PublisherGroup
-from fundus.scraping.filter import inverse, lor, regex_filter
+from fundus.scraping.filter import inverse, regex_filter
 from fundus.scraping.url import Sitemap
 
 from .the_portugal_news import ThePortugalNewsParser
@@ -12,12 +12,85 @@ class PT(metaclass=PublisherGroup):
         name="The Portugal News",
         domain="https://www.theportugalnews.com/",
         parser=ThePortugalNewsParser,
-        # There are more languages un the sitemap that could be added in the future
         sources=[
             Sitemap(
-                "https://www.theportugalnews.com/sitemap.xml",
-                sitemap_filter=lor(regex_filter("category-pages"), inverse(regex_filter("/en/"))),
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-en.xml")),
                 languages={"en"},
             ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-de.xml")),
+                languages={"de"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-nl.xml")),
+                languages={"nl"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-fr.xml")),
+                languages={"fr"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-es.xml")),
+                languages={"es"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-it.xml")),
+                languages={"it"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-se.xml")),
+                languages={"se"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-ru.xml")),
+                languages={"ru"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-zh.xml")),
+                languages={"zh"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-tr.xml")),
+                languages={"tr"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-pt.xml")),
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-ar.xml")),
+                languages={"ar"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-he.xml")),
+                languages={"he"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-pl.xml")),
+                languages={"pl"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-fi.xml")),
+                languages={"fi"},
+            ),
+            Sitemap(
+                "https://www.theportugalnews.com/sitemap-news.xml",
+                sitemap_filter=inverse(regex_filter("news-br.xml")),
+                languages={"br"},
+            ),
         ],
     )