Cache added to avoid recompiling unchanged notebooks (#267)

Author: gialmisi

README.md

@@ -258,6 +258,30 @@ you can create a `main.html` file like this:
 
 ![Download Notebook button](https://raw.githubusercontent.com/danielfrg/mkdocs-jupyter/master/docs/download-button.png)
 
+### Caching
+
+By default the plugin caches notebook conversion results so that unchanged
+notebooks are not re-converted on every build. The cache key is based on the
+notebook file content and all relevant plugin config options.
+
+To disable caching:
+
+```yaml
+plugins:
+    - mkdocs-jupyter:
+          cache: false
+```
+
+To change the cache directory (default is `.cache/mkdocs-jupyter`):
+
+```yaml
+plugins:
+    - mkdocs-jupyter:
+          cache_dir: .cache/custom-dir
+```
+
+Stale cache entries from previous builds are automatically cleaned up.
+
 ## Styles
 
 This extensions includes the Jupyter Lab nbconvert CSS styles and does some

src/mkdocs_jupyter/plugin.py

@@ -1,3 +1,5 @@
+import hashlib
+import json
 import logging
 import os
 import pathlib
@@ -54,6 +56,8 @@ class Plugin(mkdocs.plugins.BasePlugin):
         ("toc_depth", config_options.Type(int, default=6)),
         ("data_files", config_options.Type(dict, default={})),
         ("custom_mathjax_url", config_options.Type(str, default="")),
+        ("cache", config_options.Type(bool, default=True)),
+        ("cache_dir", config_options.Type(str, default=".cache/mkdocs-jupyter")),
     )
     _supported_extensions = [".ipynb", ".py", ".md"]
 
@@ -84,6 +88,9 @@ def should_include(self, file):
                 return True
         return False
 
+    def on_pre_build(self, config):
+        self._used_cache_paths = set()
+
     def on_files(self, files, config):
         ret = Files(
             [
@@ -117,8 +124,28 @@ def on_pre_page(self, page, config, files):
                         exec_nb = False
 
             theme = self.config["theme"]
+            cache_enabled = self.config["cache"]
+            cache_dir = self.config["cache_dir"]
+
+            if cache_enabled:
+                cache_key = _compute_cache_key(
+                    page.file.abs_src_path, self.config, exec_nb
+                )
+                cache_path = _get_cache_path(cache_dir, cache_key)
+                self._used_cache_paths.add(cache_path)
+            else:
+                cache_path = None
 
             def new_render(self, config, files):
+                if cache_path and cache_path.exists():
+                    logger.info("Cache hit: %s", page.file.abs_src_path)
+                    cached = json.loads(cache_path.read_text(encoding="utf-8"))
+                    self.content = cached["content"]
+                    self.toc = get_toc(cached["toc_tokens"])
+                    if cached.get("title") is not None and not ignore_h1_titles:
+                        self.title = cached["title"]
+                    return
+
                 body = convert.nb2html(
                     page.file.abs_src_path,
                     execute=exec_nb,
@@ -133,11 +160,27 @@ def new_render(self, config, files):
                     custom_mathjax_url=custom_mathjax_url,
                 )
                 self.content = body
-                toc, title = get_nb_toc(page.file.abs_src_path, toc_depth)
-                self.toc = toc
+                toc_tokens, title = _get_nb_toc_tokens(
+                    page.file.abs_src_path, toc_depth
+                )
+                self.toc = get_toc(toc_tokens)
                 if title is not None and not ignore_h1_titles:
                     self.title = title
 
+                if cache_path:
+                    logger.info("Cache miss, writing: %s", page.file.abs_src_path)
+                    cache_path.parent.mkdir(parents=True, exist_ok=True)
+                    cache_path.write_text(
+                        json.dumps(
+                            {
+                                "content": body,
+                                "toc_tokens": toc_tokens,
+                                "title": title,
+                            }
+                        ),
+                        encoding="utf-8",
+                    )
+
             # replace render with new_render for this object only
             page.render = new_render.__get__(page, Page)
 
@@ -182,22 +225,74 @@ def on_post_page(self, output_content, page, config):
                 copyfile(data_source, data_target)
             logger.info("Copied data files: %s to %s", data_files, data_target_dir)
 
+    def on_post_build(self, config):
+        if not self.config["cache"]:
+            return
+        cache_dir = pathlib.Path(self.config["cache_dir"])
+        if not cache_dir.is_dir():
+            return
+        for cache_file in cache_dir.glob("*.json"):
+            if cache_file not in self._used_cache_paths:
+                cache_file.unlink()
+                logger.info("Evicted stale cache: %s", cache_file)
+
 
 def _get_markdown_toc(markdown_source, toc_depth):
     md = markdown.Markdown(extensions=[TocExtension(toc_depth=toc_depth)])
     md.convert(markdown_source)
     return md.toc_tokens
 
 
-def get_nb_toc(fpath, toc_depth):
-    """Returns a TOC for the Notebook
-    It does that by converting first to MD
+def _get_nb_toc_tokens(fpath, toc_depth):
+    """Returns raw TOC tokens and title for the Notebook.
+
+    Converts to Markdown first, then extracts TOC tokens.
+    Returns (toc_tokens, title) where toc_tokens is a list of dicts.
     """
     body = convert.nb2md(fpath)
     md_toc_tokens = _get_markdown_toc(body, toc_depth)
-    toc = get_toc(md_toc_tokens)
     title = None
     for token in md_toc_tokens:
         if token["level"] == 1 and title is None:
             title = token["name"]
+    return md_toc_tokens, title
+
+
+def get_nb_toc(fpath, toc_depth):
+    """Returns a TOC for the Notebook
+    It does that by converting first to MD
+    """
+    md_toc_tokens, title = _get_nb_toc_tokens(fpath, toc_depth)
+    toc = get_toc(md_toc_tokens)
     return toc, title
+
+
+def _compute_cache_key(nb_path, config, exec_nb):
+    """Compute a SHA-256 hash from notebook content and relevant config options.
+
+    Uses the resolved exec_nb value (after execute_ignore processing) rather
+    than config["execute"], so that notebooks in execute_ignore get a distinct
+    cache key.
+    """
+    hasher = hashlib.sha256()
+    hasher.update(pathlib.Path(nb_path).read_bytes())
+    hasher.update(f"execute={exec_nb}".encode())
+    for key in (
+        "kernel_name",
+        "theme",
+        "allow_errors",
+        "show_input",
+        "no_input",
+        "remove_tag_config",
+        "highlight_extra_classes",
+        "include_requirejs",
+        "custom_mathjax_url",
+        "toc_depth",
+    ):
+        hasher.update(f"{key}={repr(config[key])}".encode())
+    return hasher.hexdigest()
+
+
+def _get_cache_path(cache_dir, cache_key):
+    """Return the Path for a given cache key."""
+    return pathlib.Path(cache_dir) / f"{cache_key}.json"

src/mkdocs_jupyter/tests/test_cache.py

@@ -0,0 +1,199 @@
+import json
+import os
+import pathlib
+import tempfile
+from unittest.mock import patch
+
+import pytest
+
+from mkdocs_jupyter.plugin import _compute_cache_key, _get_cache_path
+
+
+@pytest.fixture
+def sample_nb(tmp_path):
+    """Create a minimal .ipynb file for cache key tests."""
+    nb = {
+        "cells": [
+            {
+                "cell_type": "markdown",
+                "metadata": {},
+                "source": ["# Hello World"],
+            }
+        ],
+        "metadata": {
+            "kernelspec": {
+                "display_name": "Python 3",
+                "language": "python",
+                "name": "python3",
+            }
+        },
+        "nbformat": 4,
+        "nbformat_minor": 5,
+    }
+    nb_path = tmp_path / "test.ipynb"
+    nb_path.write_text(json.dumps(nb))
+    return nb_path
+
+
+@pytest.fixture
+def base_config():
+    return {
+        "kernel_name": "",
+        "theme": "",
+        "allow_errors": True,
+        "show_input": True,
+        "no_input": False,
+        "remove_tag_config": {},
+        "highlight_extra_classes": "",
+        "include_requirejs": False,
+        "custom_mathjax_url": "",
+        "toc_depth": 6,
+    }
+
+
+class TestComputeCacheKey:
+    def test_same_file_same_config_same_key(self, sample_nb, base_config):
+        key1 = _compute_cache_key(str(sample_nb), base_config, False)
+        key2 = _compute_cache_key(str(sample_nb), base_config, False)
+        assert key1 == key2
+
+    def test_different_file_content_different_key(self, tmp_path, base_config):
+        nb1 = tmp_path / "nb1.ipynb"
+        nb2 = tmp_path / "nb2.ipynb"
+        nb1.write_text(json.dumps({"cells": [], "metadata": {}, "nbformat": 4}))
+        nb2.write_text(
+            json.dumps(
+                {
+                    "cells": [{"cell_type": "code"}],
+                    "metadata": {},
+                    "nbformat": 4,
+                }
+            )
+        )
+
+        key1 = _compute_cache_key(str(nb1), base_config, False)
+        key2 = _compute_cache_key(str(nb2), base_config, False)
+        assert key1 != key2
+
+    def test_different_config_different_key(self, sample_nb, base_config):
+        key1 = _compute_cache_key(str(sample_nb), base_config, False)
+
+        config2 = {**base_config, "allow_errors": False}
+        key2 = _compute_cache_key(str(sample_nb), config2, False)
+        assert key1 != key2
+
+    def test_different_exec_nb_different_key(self, sample_nb, base_config):
+        """Resolved exec_nb (after execute_ignore) affects the key."""
+        key1 = _compute_cache_key(str(sample_nb), base_config, True)
+        key2 = _compute_cache_key(str(sample_nb), base_config, False)
+        assert key1 != key2
+
+    def test_modified_file_different_key(self, sample_nb, base_config):
+        key1 = _compute_cache_key(str(sample_nb), base_config, False)
+
+        # Modify the file
+        nb = json.loads(sample_nb.read_text())
+        nb["cells"].append({"cell_type": "code", "metadata": {}, "source": ["x = 1"]})
+        sample_nb.write_text(json.dumps(nb))
+
+        key2 = _compute_cache_key(str(sample_nb), base_config, False)
+        assert key1 != key2
+
+
+class TestGetCachePath:
+    def test_returns_json_path(self):
+        path = _get_cache_path("/tmp/cache", "abc123")
+        assert path == pathlib.Path("/tmp/cache/abc123.json")
+
+    def test_path_under_cache_dir(self):
+        path = _get_cache_path(".cache/mkdocs-jupyter", "deadbeef")
+        assert str(path) == ".cache/mkdocs-jupyter/deadbeef.json"
+
+
+class TestCacheIntegration:
+    """Integration tests using full mkdocs build."""
+
+    def test_cache_populated_on_first_build(self):
+        """First build should create cache files."""
+        from mkdocs.commands.build import build
+        from mkdocs.config import load_config
+
+        this_dir = os.path.dirname(os.path.realpath(__file__))
+        config_file = os.path.join(this_dir, "mkdocs/base-with-nbs.yml")
+
+        with tempfile.TemporaryDirectory() as cache_dir:
+            cfg = load_config(config_file)
+            cfg["plugins"]["mkdocs-jupyter"].config["cache"] = True
+            cfg["plugins"]["mkdocs-jupyter"].config["cache_dir"] = cache_dir
+
+            build(cfg)
+
+            cache_files = list(pathlib.Path(cache_dir).glob("*.json"))
+            assert len(cache_files) > 0, "Cache files should be created on first build"
+
+    def test_cache_hit_on_second_build(self):
+        """Second build should use cached results (nb2html not called again)."""
+        from mkdocs.commands.build import build
+        from mkdocs.config import load_config
+
+        this_dir = os.path.dirname(os.path.realpath(__file__))
+        config_file = os.path.join(this_dir, "mkdocs/base-with-nbs.yml")
+
+        with tempfile.TemporaryDirectory() as cache_dir:
+            cfg = load_config(config_file)
+            cfg["plugins"]["mkdocs-jupyter"].config["cache"] = True
+            cfg["plugins"]["mkdocs-jupyter"].config["cache_dir"] = cache_dir
+
+            # First build populates cache
+            build(cfg)
+
+            # Second build should hit cache — nb2html should not be called
+            cfg2 = load_config(config_file)
+            cfg2["plugins"]["mkdocs-jupyter"].config["cache"] = True
+            cfg2["plugins"]["mkdocs-jupyter"].config["cache_dir"] = cache_dir
+
+            with patch("mkdocs_jupyter.convert.nb2html") as mock_nb2html:
+                build(cfg2)
+                mock_nb2html.assert_not_called()
+
+    def test_cache_disabled(self):
+        """When cache=False, no cache files should be created."""
+        from mkdocs.commands.build import build
+        from mkdocs.config import load_config
+
+        this_dir = os.path.dirname(os.path.realpath(__file__))
+        config_file = os.path.join(this_dir, "mkdocs/base-with-nbs.yml")
+
+        with tempfile.TemporaryDirectory() as cache_dir:
+            cfg = load_config(config_file)
+            cfg["plugins"]["mkdocs-jupyter"].config["cache"] = False
+            cfg["plugins"]["mkdocs-jupyter"].config["cache_dir"] = cache_dir
+
+            build(cfg)
+
+            cache_files = list(pathlib.Path(cache_dir).glob("*.json"))
+            assert len(cache_files) == 0, "No cache files when cache is disabled"
+
+    def test_stale_cache_evicted(self):
+        """Stale cache files from previous builds are cleaned up."""
+        from mkdocs.commands.build import build
+        from mkdocs.config import load_config
+
+        this_dir = os.path.dirname(os.path.realpath(__file__))
+        config_file = os.path.join(this_dir, "mkdocs/base-with-nbs.yml")
+
+        with tempfile.TemporaryDirectory() as cache_dir:
+            # Plant a stale cache file
+            stale = pathlib.Path(cache_dir) / "stale_old_entry.json"
+            stale.write_text("{}")
+
+            cfg = load_config(config_file)
+            cfg["plugins"]["mkdocs-jupyter"].config["cache"] = True
+            cfg["plugins"]["mkdocs-jupyter"].config["cache_dir"] = cache_dir
+
+            build(cfg)
+
+            assert not stale.exists(), "Stale cache file should be evicted"
+            # But valid cache files should remain
+            cache_files = list(pathlib.Path(cache_dir).glob("*.json"))
+            assert len(cache_files) > 0