Progress3

Author: bucheben

pyproject.toml

@@ -82,3 +82,8 @@ filterwarnings = [
     "error"
 ]
 
+[tool.uv.workspace]
+members = [
+    "3-11",
+]
+

src/fundus/publishers/ua/__init__.py

@@ -1,5 +1,6 @@
 from fundus.publishers.base_objects import Publisher, PublisherGroup
-from fundus.scraping.url import NewsMap, RSSFeed, Sitemap
+from fundus.scraping.filter import inverse, regex_filter
+from fundus.scraping.url import NewsMap, Sitemap
 
 from .pravda import PravdaParser
 
@@ -12,6 +13,8 @@ class UA(metaclass=PublisherGroup):
         domain="https://www.pravda.com.ua",
         parser=PravdaParser,
         sources=[
-            NewsMap("https://www.pravda.com.ua/sitemap/sitemap-news.xml", languages={"uk", "en", "rus"}),
+            Sitemap("https://www.pravda.com.ua/sitemap/sitemap-archive.xml", languages={"uk", "en", "ru"}),
+            NewsMap("https://www.pravda.com.ua/sitemap/sitemap-news.xml", languages={"uk", "en", "ru"}),
         ],
+        url_filter=inverse(regex_filter("[^e]pravda.com.ua.*/news/")),
     )

src/fundus/publishers/ua/pravda.py

@@ -1,10 +1,9 @@
-from datetime import date, datetime
+from datetime import datetime
 from typing import List, Optional
 
 from lxml.cssselect import CSSSelector
-from lxml.etree import XPath
 
-from fundus.parser import ArticleBody, BaseParser, Image, ParserProxy, attribute
+from fundus.parser import ArticleBody, BaseParser, ParserProxy, attribute
 from fundus.parser.utility import (
     extract_article_body_with_selector,
     generic_author_parsing,
@@ -16,7 +15,6 @@
 class PravdaParser(ParserProxy):
     class V1(BaseParser):
         _paragraph_selector = CSSSelector("div.post_news_text > p")
-        # _summary_selector = CSSSelector("/html/head/meta[name='description']")
 
         @attribute
         def body(self) -> Optional[ArticleBody]:
@@ -31,8 +29,6 @@ def title(self) -> Optional[str]:
 
         @attribute
         def authors(self) -> List[str]:
-            # print(self.precomputed.ld.__dict__)
-            print(generic_author_parsing(self.precomputed.ld.xpath_search("ProfilePage/mainEntity/name")))
             return generic_author_parsing(self.precomputed.ld.xpath_search("ProfilePage/mainEntity/name"))
 
         @attribute

tests/test_publisher_collection.py

@@ -20,6 +20,7 @@ def get_two_letter_code() -> List[str]:
 
 
 language_codes = get_two_letter_code()
+print(language_codes)
 
 
 class TestPublisherCollection: