✨ NEW: Add create_toc_dict

Author: chrisjsewell

README.md

@@ -198,7 +198,7 @@ meta:
   create_append:
     intro: |
       This is some
-      extra text
+      appended text
   create_files:
   - doc1
   - doc2
@@ -211,9 +211,9 @@ The ToC file is parsed to a `SiteMap`, which is a `MutableMapping` subclass, wit
 
 ```python
 import yaml
-from sphinx_external_toc.api import parse_toc_file
+from sphinx_external_toc.api import parse_toc_yaml
 path = "path/to/_toc.yml"
-site_map = parse_toc_file(path)
+site_map = parse_toc_yaml(path)
 yaml.dump(site_map.as_json())
 ```
 
@@ -257,7 +257,7 @@ Questions / TODOs:
 
 - Should `titlesonly` default to `True` (as in jupyter-book)?
 - nested numbered toctree not allowed (logs warning), so should be handled if `numbered: true` is in defaults
-- Add additional top-level keys, e.g. `appendices` and `bibliography`
+- Add additional top-level keys, e.g. `appendices` (see https://github.com/sphinx-doc/sphinx/issues/2502) and `bibliography`
 - Add tests for "bad" toc files
 - Using `external_toc_exclude_missing` to exclude a certain file suffix:
   currently if you had files `doc.md` and `doc.rst`, and put `doc.md` in your ToC,

sphinx_external_toc/api.py

@@ -7,6 +7,10 @@
 from attr.validators import instance_of, deep_iterable, optional
 import yaml
 
+FILE_KEY = "file"
+GLOB_KEY = "glob"
+URL_KEY = "url"
+
 
 class FileItem(str):
     """A document path in a toctree list.
@@ -147,7 +151,7 @@ class MalformedError(Exception):
     """Raised if toc file is malformed."""
 
 
-def parse_toc_file(path: Union[str, Path], encoding: str = "utf8") -> SiteMap:
+def parse_toc_yaml(path: Union[str, Path], encoding: str = "utf8") -> SiteMap:
     """Parse the ToC file."""
     with Path(path).open(encoding=encoding) as handle:
         data = yaml.safe_load(handle)
@@ -168,7 +172,7 @@ def parse_toc_data(data: Dict[str, Any]) -> SiteMap:
 
 
 def _parse_doc_item(
-    data: Dict[str, Any], defaults: Dict[str, Any], path: str, file_key: str = "file"
+    data: Dict[str, Any], defaults: Dict[str, Any], path: str, file_key: str = FILE_KEY
 ) -> Tuple[DocItem, Sequence[Dict[str, Any]]]:
     """Parse a single doc item."""
     if file_key not in data:
@@ -184,7 +188,7 @@ def _parse_doc_item(
     if not isinstance(parts_data, Sequence):
         raise MalformedError(f"'parts' not a sequence: '{path}'")
 
-    _known_link_keys = {"file", "glob", "url"}
+    _known_link_keys = {FILE_KEY, GLOB_KEY, URL_KEY}
 
     parts = []
     for part_idx, part in enumerate(parts_data):
@@ -203,12 +207,12 @@ def _parse_doc_item(
                     "toctree section contains incompatible keys "
                     f"{link_keys!r}: {path}{part_idx}/{sect_idx}"
                 )
-            if link_keys == {"file"}:
-                sections.append(FileItem(section["file"]))
-            elif link_keys == {"glob"}:
-                sections.append(GlobItem(section["glob"]))
-            elif link_keys == {"url"}:
-                sections.append(UrlItem(section["url"], section.get("title")))
+            if link_keys == {FILE_KEY}:
+                sections.append(FileItem(section[FILE_KEY]))
+            elif link_keys == {GLOB_KEY}:
+                sections.append(GlobItem(section[GLOB_KEY]))
+            elif link_keys == {URL_KEY}:
+                sections.append(UrlItem(section[URL_KEY], section.get("title")))
 
         # generate toc key-word arguments
         keywords = {}
@@ -239,7 +243,7 @@ def _parse_doc_item(
         section
         for part in parts_data
         for section in part["sections"]
-        if "file" in section
+        if FILE_KEY in section
     ]
 
     return (
@@ -264,3 +268,72 @@ def _parse_docs_list(
         site_map[docname] = child_item
 
         _parse_docs_list(child_docs_list, site_map, defaults, child_path)
+
+
+def create_toc_dict(site_map: SiteMap, *, skip_defaults: bool = True) -> Dict[str, Any]:
+    """Create the Toc dictionary from a site-map."""
+    data = _docitem_to_dict(
+        site_map.root, site_map, skip_defaults=skip_defaults, file_key="root"
+    )
+    if site_map.meta:
+        data["meta"] = site_map.meta.copy()
+    return data
+
+
+def _docitem_to_dict(
+    doc_item: DocItem,
+    site_map: SiteMap,
+    *,
+    skip_defaults: bool = True,
+    file_key: str = FILE_KEY,
+    parsed_docnames: Optional[Set[str]] = None,
+) -> Dict[str, Any]:
+
+    # protect against infinite recursion
+    parsed_docnames = parsed_docnames or set()
+    if doc_item.docname in parsed_docnames:
+        raise RecursionError(f"{doc_item.docname!r} in site-map multiple times")
+    parsed_docnames.add(doc_item.docname)
+
+    data: Dict[str, Any] = {}
+
+    data[file_key] = doc_item.docname
+    if doc_item.title is not None:
+        data["title"] = doc_item.title
+
+    if not doc_item.parts:
+        return data
+
+    def _parse_section(item):
+        if isinstance(item, FileItem):
+            return _docitem_to_dict(
+                site_map[item],
+                site_map,
+                skip_defaults=skip_defaults,
+                parsed_docnames=parsed_docnames,
+            )
+        if isinstance(item, GlobItem):
+            return {GLOB_KEY: str(item)}
+        if isinstance(item, UrlItem):
+            if item.title is not None:
+                return {URL_KEY: item.url, "title": item.title}
+            return {URL_KEY: item.url}
+        raise TypeError(item)
+
+    data["parts"] = []
+    fields = attr.fields_dict(TocItem)
+    for part in doc_item.parts:
+        # only add these keys if their value is not the default
+        part_data = {
+            key: getattr(part, key)
+            for key in ("caption", "numbered", "reversed", "titlesonly")
+            if (not skip_defaults) or getattr(part, key) != fields[key].default
+        }
+        part_data["sections"] = [_parse_section(s) for s in part.sections]
+        data["parts"].append(part_data)
+
+    # apply shorthand if possible
+    if len(data["parts"]) == 1 and list(data["parts"][0]) == ["sections"]:
+        data["sections"] = data.pop("parts")[0]["sections"]
+
+    return data

sphinx_external_toc/cli.py

@@ -2,7 +2,7 @@
 import yaml
 
 from sphinx_external_toc import __version__
-from sphinx_external_toc.api import parse_toc_file
+from sphinx_external_toc.api import parse_toc_yaml
 from sphinx_external_toc.tools import create_site_from_toc
 
 
@@ -16,7 +16,7 @@ def main():
 @click.argument("toc_file", type=click.Path(exists=True, file_okay=True))
 def parse_toc(toc_file):
     """Parse a ToC file to a site-map YAML."""
-    site_map = parse_toc_file(toc_file)
+    site_map = parse_toc_yaml(toc_file)
     click.echo(yaml.dump(site_map.as_json()))
 
 

sphinx_external_toc/events.py

@@ -14,7 +14,7 @@
 from sphinx.util.docutils import SphinxDirective
 from sphinx.util.matching import Matcher, patfilter, patmatch
 
-from .api import DocItem, GlobItem, FileItem, SiteMap, UrlItem, parse_toc_file
+from .api import DocItem, GlobItem, FileItem, SiteMap, UrlItem, parse_toc_yaml
 
 logger = logging.getLogger(__name__)
 
@@ -51,7 +51,7 @@ def parse_toc_to_env(app: Sphinx, config: Config) -> None:
     Also, change the ``master_doc`` and add to ``exclude_patterns`` if necessary.
     """
     try:
-        site_map = parse_toc_file(Path(app.srcdir) / app.config["external_toc_path"])
+        site_map = parse_toc_yaml(Path(app.srcdir) / app.config["external_toc_path"])
     except Exception as exc:
         raise ExtensionError(f"[etoc] {exc}") from exc
     config.external_site_map = site_map
@@ -120,13 +120,15 @@ def add_changed_toctrees(
 
 
 class TableOfContentsNode(nodes.Element):
-    """A placeholder for the insertion of a toctree (in ``append_toctrees``)."""
+    """A placeholder for the insertion of a toctree (in ``insert_toctrees``)."""
 
     def __init__(self, **attributes: Any) -> None:
         super().__init__(rawsource="", **attributes)
 
 
 class TableofContents(SphinxDirective):
+    """Insert a placeholder for toctree insertion."""
+
     # TODO allow for name option of tableofcontents (to reference it)
     def run(self) -> List[TableOfContentsNode]:
         """Insert a ``TableOfContentsNode`` node."""

sphinx_external_toc/tools.py

@@ -3,7 +3,7 @@
 import shutil
 from typing import Mapping, Optional, Sequence, Union
 
-from .api import parse_toc_file, SiteMap
+from .api import parse_toc_yaml, SiteMap
 
 
 def create_site_from_toc(
@@ -31,7 +31,7 @@ def create_site_from_toc(
 
     """
     assert default_ext in {".rst", ".md"}
-    site_map = parse_toc_file(toc_path)
+    site_map = parse_toc_yaml(toc_path)
 
     root_path = Path(toc_path).parent if root_path is None else Path(root_path)
     root_path.mkdir(parents=True, exist_ok=True)

tests/test_api.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from sphinx_external_toc.api import parse_toc_file
+from sphinx_external_toc.api import create_toc_dict, parse_toc_yaml
 
 TOC_FILES = list(Path(__file__).parent.joinpath("_toc_files").glob("*.yml"))
 
@@ -11,5 +11,14 @@
     "path", TOC_FILES, ids=[path.name.rsplit(".", 1)[0] for path in TOC_FILES]
 )
 def test_file_to_sitemap(path: Path, data_regression):
-    site_map = parse_toc_file(path)
+    site_map = parse_toc_yaml(path)
     data_regression.check(site_map.as_json())
+
+
+@pytest.mark.parametrize(
+    "path", TOC_FILES, ids=[path.name.rsplit(".", 1)[0] for path in TOC_FILES]
+)
+def test_create_toc_dict(path: Path, data_regression):
+    site_map = parse_toc_yaml(path)
+    data = create_toc_dict(site_map)
+    data_regression.check(data)

tests/test_api/test_create_toc_dict_basic_.yml

@@ -0,0 +1,13 @@
+meta:
+  regress: intro
+parts:
+- caption: Part Caption
+  numbered: true
+  sections:
+  - file: doc1
+  - file: doc2
+  - file: doc3
+    sections:
+    - file: subfolder/doc4
+    - url: https://example.com
+root: intro

tests/test_api/test_create_toc_dict_basic_compressed_.yml

@@ -0,0 +1,12 @@
+meta:
+  regress: intro
+parts:
+- numbered: true
+  sections:
+  - file: doc1
+  - file: doc2
+  - file: doc3
+    sections:
+    - file: doc4
+    - url: https://example.com
+root: intro

tests/test_api/test_create_toc_dict_exclude_missing_.yml

@@ -0,0 +1,9 @@
+meta:
+  create_files:
+  - doc2
+  - subfolder/other1
+  exclude_missing: true
+root: intro
+sections:
+- file: doc1
+- glob: subfolder/other*

tests/test_api/test_create_toc_dict_glob_.yml

@@ -0,0 +1,8 @@
+meta:
+  create_files:
+  - doc1
+  - doc2
+  - doc3
+root: intro
+sections:
+- glob: doc*