Merge branch 'main' into main

Author: meysam81

CHANGELOG.md

@@ -16,6 +16,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 
 -->
+
+## 1.14.0 - 2024-06-25
+
+### Bugs fixes 🐛
+
+* Update docs for `as_update` (boolean not supported anymore) by @brapoprod in <https://github.com/Guts/mkdocs-rss-plugin/pull/306>
+* improve(remote_images): handle connection error by @Guts in <https://github.com/Guts/mkdocs-rss-plugin/pull/307>
+* act depending on the used Mkdocs command and disable remote images length fetching to avoid HTTP errors by @Guts in <https://github.com/Guts/mkdocs-rss-plugin/pull/311>
+
+### Features and enhancements 🎉
+
+* Feature: add http caching mechanism by @Guts in <https://github.com/Guts/mkdocs-rss-plugin/pull/309>
+* improve(http): disable warnings for unverified requests by @Guts in <https://github.com/Guts/mkdocs-rss-plugin/pull/308>
+
+## New Contributors
+
+* @brapoprod made their first contribution in <https://github.com/Guts/mkdocs-rss-plugin/pull/306>
+
 ## 1.13.4 - 2024-06-24
 
 ### Bugs fixes 🐛

docs/configuration.md

@@ -196,6 +196,28 @@ Default: `<!-- more -->`
 
 ----
 
+### :material-recycle: `cache_dir`: folder where to store plugin's cached files { #cache_dir }
+
+The plugin implements a caching mechanism, ensuring that a remote media is only get once during its life-cycle on remote HTTP server (using [Cache Control](https://pypi.org/project/CacheControl/) under the hood). It is normally not necessary to specify this setting, except for when you want to change the path within your root directory where HTTP body and metadata files are cached.
+
+If you want to change it, use:
+
+``` yaml
+plugins:
+  - rss:
+      cache_dir: my/custom/dir
+```
+
+It's strongly recommended to add the path to your `.gitignore` file in the root of your project:
+
+``` title=".gitignore"
+.cache
+```
+
+Default: `.cache/plugins/rss`.
+
+----
+
 ### :material-tag-multiple: `categories`: item categories { #categories }
 
 `categories`: list of page metadata values to use as [RSS item categories](https://www.w3schools.com/xml/rss_tag_category_item.asp).

mkdocs_rss_plugin/__about__.py

@@ -40,7 +40,7 @@
 __title_clean__ = "".join(e for e in __title__ if e.isalnum())
 __uri__ = "https://github.com/Guts/mkdocs-rss-plugin/"
 
-__version__ = "1.13.4"
+__version__ = "1.14.0"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

mkdocs_rss_plugin/config.py

@@ -8,6 +8,9 @@
 from mkdocs.config import config_options
 from mkdocs.config.base import Config
 
+# package
+from mkdocs_rss_plugin.constants import DEFAULT_CACHE_FOLDER
+
 # ############################################################################
 # ########## Classes ###############
 # ##################################
@@ -16,7 +19,6 @@
 class _DateFromMeta(Config):
     """Sub configuration object for related date options."""
 
-    #  TODO: remove deprecated code in future version. Only str values will be accepted
     # for as_creation and as_update
     as_creation = config_options.Type(str, default="git")
     as_update = config_options.Type(str, default="git")
@@ -42,6 +44,7 @@ class RssPluginConfig(Config):
     categories = config_options.Optional(
         config_options.ListOfItems(config_options.Type(str))
     )
+    cache_dir = config_options.Type(str, default=f"{DEFAULT_CACHE_FOLDER.resolve()}")
     comments_path = config_options.Optional(config_options.Type(str))
     date_from_meta = config_options.SubConfig(_DateFromMeta)
     enabled = config_options.Type(bool, default=True)

mkdocs_rss_plugin/constants.py

@@ -14,6 +14,7 @@
 # ########## Globals #############
 # ################################
 
+DEFAULT_CACHE_FOLDER = Path(".cache/plugins/rss")
 DEFAULT_TEMPLATE_FOLDER = Path(__file__).parent / "templates"
 DEFAULT_TEMPLATE_FILENAME = DEFAULT_TEMPLATE_FOLDER / "rss.xml.jinja2"
 MKDOCS_LOGGER_NAME = "[RSS-plugin]"

mkdocs_rss_plugin/plugin.py

@@ -11,7 +11,7 @@
 from email.utils import formatdate
 from pathlib import Path
 from re import compile as re_compile
-from typing import List, Optional
+from typing import List, Literal, Optional
 
 # 3rd party
 from jinja2 import Environment, FileSystemLoader, select_autoescape
@@ -55,9 +55,30 @@ class GitRssPlugin(BasePlugin[RssPluginConfig]):
     # allow to set the plugin multiple times in the same mkdocs config
     supports_multiple_instances = True
 
-    def __init__(self):
+    def __init__(self, *args, **kwargs):
         """Instantiation."""
         # pages storage
+        super().__init__(*args, **kwargs)
+
+        self.cmd_is_serve: bool = False
+
+    def on_startup(
+        self, *, command: Literal["build", "gh-deploy", "serve"], dirty: bool
+    ) -> None:
+        """The `startup` event runs once at the very beginning of an `mkdocs` invocation.
+        Note that for initializing variables, the __init__ method is still preferred.
+        For initializing per-build variables (and whenever in doubt), use the
+        on_config event.
+
+        See: https://www.mkdocs.org/user-guide/plugins/#on_startup
+
+        Args:
+            command: the command that MkDocs was invoked with, e.g. "serve" for `mkdocs serve`.
+            dirty: whether `--dirty` flag was passed.
+        """
+        # flag used command to disable some actions if serve is used
+        self.cmd_is_serve = command == "serve"
+
         self.pages_to_filter: List[PageInformation] = []
         # prepare output feeds
         self.feed_created: dict = {}
@@ -92,6 +113,11 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig:
             self.config.enabled = False
             return config
 
+        # cache dir
+        self.cache_dir = Path(self.config.cache_dir)
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        logger.debug(f"Caching HTTP requests to: {self.cache_dir.resolve()}")
+
         # integrations - check if theme is Material and if social cards are enabled
         self.integration_material_social_cards = IntegrationMaterialSocialCards(
             mkdocs_config=config,
@@ -100,8 +126,10 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig:
 
         # instantiate plugin tooling
         self.util = Util(
+            cache_dir=self.cache_dir,
             use_git=self.config.use_git,
             integration_material_social_cards=self.integration_material_social_cards,
+            mkdocs_command_is_on_serve=self.cmd_is_serve,
         )
 
         # check template dirs
@@ -169,10 +197,6 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig:
             self.config.date_from_meta.default_time = datetime.strptime(
                 self.config.date_from_meta.default_time, "%H:%M"
             )
-            print(
-                self.config.date_from_meta.default_time,
-                type(self.config.date_from_meta.default_time),
-            )
         except (TypeError, ValueError) as err:
             logger.warning(
                 "Config error: `date_from_meta.default_time` value "

mkdocs_rss_plugin/util.py

@@ -19,6 +19,8 @@
 # 3rd party
 import markdown
 import urllib3
+from cachecontrol import CacheControl
+from cachecontrol.caches.file_cache import SeparateBodyFileCache
 from git import (
     GitCommandError,
     GitCommandNotFound,
@@ -34,7 +36,11 @@
 from requests.exceptions import ConnectionError, HTTPError
 
 # package
-from mkdocs_rss_plugin.constants import MKDOCS_LOGGER_NAME, REMOTE_REQUEST_HEADERS
+from mkdocs_rss_plugin.constants import (
+    DEFAULT_CACHE_FOLDER,
+    MKDOCS_LOGGER_NAME,
+    REMOTE_REQUEST_HEADERS,
+)
 from mkdocs_rss_plugin.git_manager.ci import CiHandler
 from mkdocs_rss_plugin.integrations.theme_material_social_plugin import (
     IntegrationMaterialSocialCards,
@@ -66,11 +72,13 @@ class Util:
 
     def __init__(
         self,
-        path: str = ".",
-        use_git: bool = True,
+        cache_dir: Path = DEFAULT_CACHE_FOLDER,
         integration_material_social_cards: Optional[
             IntegrationMaterialSocialCards
         ] = None,
+        mkdocs_command_is_on_serve: bool = False,
+        path: str = ".",
+        use_git: bool = True,
     ):
         """Class hosting the plugin logic.
 
@@ -80,6 +88,13 @@ def __init__(
             integration_material_social_cards (bool, optional): option to enable
                 integration with Social Cards plugin from Material theme. Defaults to True.
         """
+        self.mkdocs_command_is_on_serve = mkdocs_command_is_on_serve
+        if self.mkdocs_command_is_on_serve:
+            logger.debug(
+                "Mkdocs serve - Fetching remote images length is disabled to avoid "
+                "HTTP errors."
+            )
+
         if use_git:
             logger.debug("Git use is enabled.")
             try:
@@ -122,8 +137,13 @@ def __init__(
         self.social_cards = integration_material_social_cards
 
         # http/s session
-        self.req_session = Session()
-        self.req_session.headers.update(REMOTE_REQUEST_HEADERS)
+        session = Session()
+        session.headers.update(REMOTE_REQUEST_HEADERS)
+        self.req_session = CacheControl(
+            sess=session,
+            cache=SeparateBodyFileCache(directory=cache_dir),
+            cacheable_methods=("GET", "HEAD"),
+        )
 
     def build_url(
         self, base_url: str, path: str, args_dict: Optional[dict] = None
@@ -645,6 +665,9 @@ def get_remote_image_length(
         Returns:
             int | None: image length as int or None
         """
+        if self.mkdocs_command_is_on_serve:
+            return None
+
         # first, try HEAD request to avoid downloading the image
         try:
             attempt += 1

requirements/base.txt

@@ -1,7 +1,7 @@
 # Common requirements
 # -----------------------
 
-
+cachecontrol[filecache] >=0.14,<1
 GitPython>=3.1,<3.2
 mkdocs>=1.5,<2
 requests>=2.31,<3

tests/dev/dev_cached_http.py

@@ -0,0 +1,45 @@
+import http.client
+import logging
+from pathlib import Path
+
+import requests
+from cachecontrol import CacheControl
+from cachecontrol.caches.file_cache import FileCache
+
+http.client.HTTPConnection.debuglevel = 1
+logging.basicConfig()
+logging.getLogger().setLevel(logging.DEBUG)
+req_log = logging.getLogger("requests.packages.urllib3")
+req_log.setLevel(logging.DEBUG)
+req_log.propagate = True
+
+
+sess = CacheControl(
+    requests.Session(), cache=FileCache(".web_cache"), cacheable_methods=("HEAD", "GET")
+)
+
+
+# get requests
+resp = sess.get("https://geotribu.fr")
+resp_img = sess.get(
+    "https://cdn.geotribu.fr/img/articles-blog-rdp/capture-ecran/kevish_Air-Traffic.png"
+)
+
+# try again, cache hit expected
+resp = sess.get("https://geotribu.fr")
+resp_img = sess.get(
+    "https://cdn.geotribu.fr/img/articles-blog-rdp/capture-ecran/kevish_Air-Traffic.png"
+)
+
+# head requests
+resp_img = sess.head(
+    "https://cdn.geotribu.fr/img/articles-blog-rdp/capture-ecran/kevish_Air-Traffic.png"
+)
+
+
+# try again, cache hit expected
+resp_img = sess.head(
+    "https://cdn.geotribu.fr/img/articles-blog-rdp/capture-ecran/kevish_Air-Traffic.png"
+)
+
+print(list(Path(".web_cache").iterdir()))

tests/test_config.py

@@ -21,8 +21,10 @@
 # 3rd party
 from mkdocs.config.base import Config
 
-# plugin target
 from mkdocs_rss_plugin.config import RssPluginConfig
+
+# plugin target
+from mkdocs_rss_plugin.constants import DEFAULT_CACHE_FOLDER
 from mkdocs_rss_plugin.plugin import GitRssPlugin
 
 # test suite
@@ -62,6 +64,7 @@ def test_plugin_config_defaults(self):
             "abstract_chars_count": 160,
             "abstract_delimiter": "<!-- more -->",
             "categories": None,
+            "cache_dir": f"{DEFAULT_CACHE_FOLDER.resolve()}",
             "comments_path": None,
             "date_from_meta": {
                 "as_creation": "git",
@@ -105,6 +108,7 @@ def test_plugin_config_image(self):
         expected = {
             "abstract_chars_count": 160,
             "abstract_delimiter": "<!-- more -->",
+            "cache_dir": f"{DEFAULT_CACHE_FOLDER.resolve()}",
             "categories": None,
             "comments_path": None,
             "date_from_meta": {