♻️ REFACTOR: Move parsing to separate module

add add more parsing validation

Author: chrisjsewell

README.md

@@ -283,7 +283,7 @@ The ToC file is parsed to a `SiteMap`, which is a `MutableMapping` subclass, wit
 
 ```python
 import yaml
-from sphinx_external_toc.api import parse_toc_yaml
+from sphinx_external_toc.parsing import parse_toc_yaml
 path = "path/to/_toc.yml"
 site_map = parse_toc_yaml(path)
 yaml.dump(site_map.as_json())

sphinx_external_toc/api.py

@@ -1,16 +1,10 @@
-""" """
-from collections.abc import Mapping, MutableMapping
-from pathlib import Path
-from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Tuple, Union
+"""Defines the `SiteMap` object, for storing the parsed ToC."""
+from collections.abc import MutableMapping
+from typing import Any, Dict, Iterator, List, Optional, Set, Union
 
 import attr
-import yaml
 from attr.validators import deep_iterable, instance_of, optional
 
-FILE_KEY = "file"
-GLOB_KEY = "glob"
-URL_KEY = "url"
-
 
 class FileItem(str):
     """A document path in a toctree list.
@@ -145,211 +139,3 @@ def as_json(
             assert meta_key not in dct
             dct[meta_key] = self.meta
         return dct
-
-
-class MalformedError(Exception):
-    """Raised if toc file is malformed."""
-
-
-def parse_toc_yaml(path: Union[str, Path], encoding: str = "utf8") -> SiteMap:
-    """Parse the ToC file."""
-    with Path(path).open(encoding=encoding) as handle:
-        data = yaml.safe_load(handle)
-    return parse_toc_data(data)
-
-
-def parse_toc_data(data: Dict[str, Any]) -> SiteMap:
-    """Parse a dictionary of the ToC."""
-    if not isinstance(data, Mapping):
-        raise MalformedError(f"toc is not a mapping: {type(data)}")
-
-    defaults: Dict[str, Any] = data.get("defaults", {})
-
-    doc_item, docs_list = _parse_doc_item(data, defaults, "/", file_key="root")
-
-    site_map = SiteMap(root=doc_item, meta=data.get("meta"))
-
-    _parse_docs_list(docs_list, site_map, defaults, "/")
-
-    return site_map
-
-
-def _parse_doc_item(
-    data: Dict[str, Any], defaults: Dict[str, Any], path: str, file_key: str = FILE_KEY
-) -> Tuple[DocItem, Sequence[Dict[str, Any]]]:
-    """Parse a single doc item."""
-    if file_key not in data:
-        raise MalformedError(f"'{file_key}' key not found: '{path}'")
-    if "sections" in data:
-        # this is a shorthand for defining a single part
-        if "parts" in data:
-            raise MalformedError(f"Both 'sections' and 'parts' found: '{path}'")
-        parts_data = [{"sections": data["sections"]}]
-    else:
-        parts_data = data.get("parts", [])
-
-    if not isinstance(parts_data, Sequence):
-        raise MalformedError(f"'parts' not a sequence: '{path}'")
-
-    _known_link_keys = {FILE_KEY, GLOB_KEY, URL_KEY}
-
-    parts = []
-    for part_idx, part in enumerate(parts_data):
-
-        # generate sections list
-        sections: List[Union[GlobItem, FileItem, UrlItem]] = []
-        for sect_idx, section in enumerate(part["sections"]):
-            link_keys = _known_link_keys.intersection(section)
-            if not link_keys:
-                raise MalformedError(
-                    "toctree section does not contain one of "
-                    f"{_known_link_keys!r}: {path}{part_idx}/{sect_idx}"
-                )
-            if not len(link_keys) == 1:
-                raise MalformedError(
-                    "toctree section contains incompatible keys "
-                    f"{link_keys!r}: {path}{part_idx}/{sect_idx}"
-                )
-
-            if link_keys == {FILE_KEY}:
-                sections.append(FileItem(section[FILE_KEY]))
-            elif link_keys == {GLOB_KEY}:
-                if "sections" in section or "parts" in section:
-                    raise MalformedError(
-                        "toctree section contains incompatible keys "
-                        f"{GLOB_KEY} and parts/sections: {path}{part_idx}/{sect_idx}"
-                    )
-                sections.append(GlobItem(section[GLOB_KEY]))
-            elif link_keys == {URL_KEY}:
-                if "sections" in section or "parts" in section:
-                    raise MalformedError(
-                        "toctree section contains incompatible keys "
-                        f"{URL_KEY} and parts/sections: {path}{part_idx}/{sect_idx}"
-                    )
-                sections.append(UrlItem(section[URL_KEY], section.get("title")))
-
-        # generate toc key-word arguments
-        keywords = {}
-        for key in ("caption", "numbered", "titlesonly", "reversed"):
-            if key in part:
-                keywords[key] = part[key]
-            elif key in defaults:
-                keywords[key] = defaults[key]
-
-        # TODO this is a hacky fix for the fact that sphinx logs a warning
-        # for nested toctrees, see:
-        # sphinx/environment/collectors/toctree.py::TocTreeCollector::assign_section_numbers::_walk_toctree
-        if keywords.get("numbered") and path != "/":
-            keywords.pop("numbered")
-
-        try:
-            toc_item = TocItem(sections=sections, **keywords)
-        except TypeError as exc:
-            raise MalformedError(f"toctree validation: {path}{part_idx}") from exc
-        parts.append(toc_item)
-
-    try:
-        doc_item = DocItem(docname=data[file_key], title=data.get("title"), parts=parts)
-    except TypeError as exc:
-        raise MalformedError(f"doc validation: {path}") from exc
-
-    docs_data = [
-        section
-        for part in parts_data
-        for section in part["sections"]
-        if FILE_KEY in section
-    ]
-
-    return (
-        doc_item,
-        docs_data,
-    )
-
-
-def _parse_docs_list(
-    docs_list: Sequence[Dict[str, Any]],
-    site_map: SiteMap,
-    defaults: Dict[str, Any],
-    path: str,
-):
-    """Parse a list of docs."""
-    for doc_data in docs_list:
-        docname = doc_data["file"]
-        if docname in site_map:
-            raise MalformedError(f"document file used multiple times: {docname}")
-        child_path = f"{path}{docname}/"
-        child_item, child_docs_list = _parse_doc_item(doc_data, defaults, child_path)
-        site_map[docname] = child_item
-
-        _parse_docs_list(child_docs_list, site_map, defaults, child_path)
-
-
-def create_toc_dict(site_map: SiteMap, *, skip_defaults: bool = True) -> Dict[str, Any]:
-    """Create the Toc dictionary from a site-map."""
-    data = _docitem_to_dict(
-        site_map.root, site_map, skip_defaults=skip_defaults, file_key="root"
-    )
-    if site_map.meta:
-        data["meta"] = site_map.meta.copy()
-    return data
-
-
-def _docitem_to_dict(
-    doc_item: DocItem,
-    site_map: SiteMap,
-    *,
-    skip_defaults: bool = True,
-    file_key: str = FILE_KEY,
-    parsed_docnames: Optional[Set[str]] = None,
-) -> Dict[str, Any]:
-
-    # protect against infinite recursion
-    parsed_docnames = parsed_docnames or set()
-    if doc_item.docname in parsed_docnames:
-        raise RecursionError(f"{doc_item.docname!r} in site-map multiple times")
-    parsed_docnames.add(doc_item.docname)
-
-    data: Dict[str, Any] = {}
-
-    data[file_key] = doc_item.docname
-    if doc_item.title is not None:
-        data["title"] = doc_item.title
-
-    if not doc_item.parts:
-        return data
-
-    def _parse_section(item):
-        if isinstance(item, FileItem):
-            if item in site_map:
-                return _docitem_to_dict(
-                    site_map[item],
-                    site_map,
-                    skip_defaults=skip_defaults,
-                    parsed_docnames=parsed_docnames,
-                )
-            return {FILE_KEY: str(item)}
-        if isinstance(item, GlobItem):
-            return {GLOB_KEY: str(item)}
-        if isinstance(item, UrlItem):
-            if item.title is not None:
-                return {URL_KEY: item.url, "title": item.title}
-            return {URL_KEY: item.url}
-        raise TypeError(item)
-
-    data["parts"] = []
-    fields = attr.fields_dict(TocItem)
-    for part in doc_item.parts:
-        # only add these keys if their value is not the default
-        part_data = {
-            key: getattr(part, key)
-            for key in ("caption", "numbered", "reversed", "titlesonly")
-            if (not skip_defaults) or getattr(part, key) != fields[key].default
-        }
-        part_data["sections"] = [_parse_section(s) for s in part.sections]
-        data["parts"].append(part_data)
-
-    # apply shorthand if possible
-    if len(data["parts"]) == 1 and list(data["parts"][0]) == ["sections"]:
-        data["sections"] = data.pop("parts")[0]["sections"]
-
-    return data

sphinx_external_toc/cli.py

@@ -4,7 +4,7 @@
 import yaml
 
 from sphinx_external_toc import __version__
-from sphinx_external_toc.api import create_toc_dict, parse_toc_yaml
+from sphinx_external_toc.parsing import create_toc_dict, parse_toc_yaml
 from sphinx_external_toc.tools import create_site_from_toc, create_site_map_from_path
 
 

sphinx_external_toc/events.py

@@ -14,7 +14,8 @@
 from sphinx.util.docutils import SphinxDirective
 from sphinx.util.matching import Matcher, patfilter, patmatch
 
-from .api import DocItem, FileItem, GlobItem, SiteMap, UrlItem, parse_toc_yaml
+from .api import DocItem, FileItem, GlobItem, SiteMap, UrlItem
+from .parsing import parse_toc_yaml
 
 logger = logging.getLogger(__name__)