DOP-3200 (#429)

* handle piped url for toctree directive

* add validation for toc node entries. add test. TODO. add error with duplicate external nodes

* throw error from post process if external node is duplicated

* add unit test for duplicate external toc

* remove testing code

* format + cleanup

* revert log level

* update diagnostic messages. add diagnostic case for missing title with ref project

* address comments. use set of tuples instead of untyped list

Author: seungpark

snooty/diagnostics.py

@@ -411,6 +411,40 @@ def did_you_mean(self) -> List[str]:
         return [".. ia::"]
 
 
+class MissingAssociatedToc(Diagnostic):
+    severity = Diagnostic.Level.warning
+
+    def __init__(
+        self,
+        expected_project: str,
+        start: Union[int, Tuple[int, int]],
+        end: Union[None, int, Tuple[int, int]] = None,
+    ) -> None:
+        super().__init__(
+            f"""Detected an associated toctree entry at {expected_project} 
+            which does not exist in an associated_products entry within the snooty.toml. 
+            Removing this toctree entry.""",
+            start,
+            end,
+        )
+
+
+class DuplicatedExternalToc(Diagnostic):
+    severity = Diagnostic.Level.error
+
+    def __init__(
+        self,
+        duplicated_toc: str,
+        start: Union[int, Tuple[int, int]],
+        end: Union[None, int, Tuple[int, int]] = None,
+    ) -> None:
+        super().__init__(
+            f"Detected a duplicated associated toctree entry at {duplicated_toc}. Removing this toctree entry.",
+            start,
+            end,
+        )
+
+
 class InvalidIAEntry(Diagnostic):
     severity = Diagnostic.Level.error
 

snooty/n.py

@@ -359,6 +359,7 @@ class TocTreeDirectiveEntry(NamedTuple):
     title: Optional[str]
     url: Optional[str]
     slug: Optional[str]
+    ref_project: Optional[str]
 
     def serialize(self) -> SerializedNode:
         result: SerializedNode = {}
@@ -368,6 +369,8 @@ def serialize(self) -> SerializedNode:
             result["url"] = self.url
         if self.slug:
             result["slug"] = self.slug
+        if self.ref_project:
+            result["ref_project"] = self.ref_project
         return result
 
 

snooty/parser.py

@@ -57,6 +57,7 @@
     InvalidURL,
     MalformedGlossary,
     MalformedRelativePath,
+    MissingAssociatedToc,
     MissingChild,
     RemovedLiteralBlockSyntax,
     TabMustBeDirective,
@@ -67,7 +68,7 @@
 )
 from .gizaparser.nodes import GizaCategory
 from .icon_names import ICON_SET
-from .n import FileId, SerializableType
+from .n import FileId, SerializableType, TocTreeDirectiveEntry
 from .openapi import OpenAPI
 from .page import Page, PendingTask
 from .postprocess import DevhubPostprocessor, Postprocessor, PostprocessorResult
@@ -576,6 +577,11 @@ def handle_directive(
         options = node["options"] or {}
 
         if name == "toctree":
+            self.diagnostics.extend(
+                validate_toc_entries(
+                    node["entries"], self.project_config.associated_products, line
+                )
+            )
             doc: n.Directive = n.TocTreeDirective(
                 (line,), [], domain, name, [], options, node["entries"]
             )
@@ -1070,6 +1076,28 @@ def __make_child_visitor(self) -> "JSONVisitor":
         return visitor
 
 
+def validate_toc_entries(
+    node_entries: List[TocTreeDirectiveEntry],
+    associated_products: List[Dict[str, object]],
+    line: int,
+) -> List[Diagnostic]:
+    """
+    validates that external toc node exists as one of the associated products
+    if not found, removes this node and emits a warning
+    associated_products come in form of {name: str, versions: List[str]}
+    """
+    diagnostics: List[Diagnostic] = []
+    associated_product_names = [product["name"] for product in associated_products]
+    for toc_entry in node_entries:
+        if (
+            toc_entry.ref_project
+            and toc_entry.ref_project not in associated_product_names
+        ):
+            diagnostics.append(MissingAssociatedToc(toc_entry.ref_project, line))
+            node_entries.remove(toc_entry)
+    return diagnostics
+
+
 def _validate_io_code_block_children(node: n.Directive) -> List[Diagnostic]:
     """Validates that a given io-code-block directive has 1 input and 1 output
     child nodes, and copies the io-code-block's options into the options of the

snooty/postprocess.py

@@ -35,6 +35,7 @@
     CannotOpenFile,
     ChapterAlreadyExists,
     Diagnostic,
+    DuplicatedExternalToc,
     DuplicateDirective,
     ExpectedPathArg,
     ExpectedTabs,
@@ -1703,8 +1704,15 @@ def build_toctree(cls, context: Context) -> Dict[str, SerializableType]:
         toc_landing_pages = [
             clean_slug(slug) for slug in context[ProjectConfig].toc_landing_pages
         ]
+        ref_project_set: Set[Tuple[Optional[str], Optional[str]]] = set()
         cls.find_toctree_nodes(
-            context, starting_fileid, ast, root, toc_landing_pages, {starting_fileid}
+            context,
+            starting_fileid,
+            ast,
+            root,
+            toc_landing_pages,
+            ref_project_set,
+            {starting_fileid},
         )
 
         return root
@@ -1717,6 +1725,7 @@ def find_toctree_nodes(
         ast: n.Node,
         node: Dict[str, Any],
         toc_landing_pages: List[str],
+        external_nodes: Set[Tuple[Optional[str], Optional[str]]],
         visited_file_ids: Set[FileId] = set(),
     ) -> None:
         """Iterate over AST to find toctree directives and construct their nodes for the unified toctree"""
@@ -1729,6 +1738,19 @@ def find_toctree_nodes(
             # Recursively build the tree for each toctree node in this entries list
             for entry in ast.entries:
                 toctree_node: Dict[str, object] = {}
+                if entry.ref_project:
+                    toctree_node = {
+                        "title": [n.Text((0,), entry.title).serialize()]
+                        if entry.title
+                        else None,
+                        "options": {"project": entry.ref_project},
+                    }
+                    ref_project_pair = (entry.title, entry.ref_project)
+                    if ref_project_pair in external_nodes:
+                        context.diagnostics[fileid].append(
+                            DuplicatedExternalToc(entry.ref_project, ast.span[0])
+                        )
+                    external_nodes.add(ref_project_pair)
                 if entry.url:
                     toctree_node = {
                         "title": [n.Text((0,), entry.title).serialize()]
@@ -1795,6 +1817,7 @@ def find_toctree_nodes(
                             new_ast,
                             toctree_node,
                             toc_landing_pages,
+                            external_nodes,
                             visited_file_ids.union({slug_fileid}),
                         )
 
@@ -1804,7 +1827,13 @@ def find_toctree_nodes(
         # Locate the correct directive object containing the toctree within this AST
         for child_ast in ast.children:
             cls.find_toctree_nodes(
-                context, fileid, child_ast, node, toc_landing_pages, visited_file_ids
+                context,
+                fileid,
+                child_ast,
+                node,
+                toc_landing_pages,
+                external_nodes,
+                visited_file_ids,
             )
 
     @staticmethod

snooty/rstparser.py

@@ -944,14 +944,19 @@ def make_toc_entry(
         title: Optional[str] = None
         url: Optional[str] = None
         slug: Optional[str] = None
+        ref_project: Optional[str] = None
         if match:
             title, target = match["label"], match["target"]
+            # pipelines denote project reference
+            if target.startswith("|") and target.endswith("|"):
+                ref_project = target[1:-1]
+                target = None
         else:
             target = child
 
-        if not title and util.PAT_URI.match(target):
+        if not title and (util.PAT_URI.match(target) or ref_project):
             # If entry is surrounded by <> tags, assume it is a URL and log an error.
-            err = "toctree nodes with URLs must include titles"
+            err = "toctree nodes with URLs or project references must include titles"
             error_node = self.state.document.reporter.error(err, line=self.lineno)
             return None, [error_node]
 
@@ -960,7 +965,7 @@ def make_toc_entry(
             url = target
         else:
             slug = target
-        return n.TocTreeDirectiveEntry(title, url, slug), []
+        return n.TocTreeDirectiveEntry(title, url, slug, ref_project), []
 
 
 class NoTransformRstParser(docutils.parsers.rst.Parser):

snooty/test_parser.py

@@ -2624,8 +2624,9 @@ def test_footnote_reference() -> None:
 
 
 def test_toctree() -> None:
-    path = ROOT_PATH.joinpath(Path("test.rst"))
-    project_config = ProjectConfig(ROOT_PATH, "", source="./")
+    project_root = ROOT_PATH.joinpath("test_project")
+    path = project_root.joinpath(Path("source/test.rst")).resolve()
+    [project_config, project_config_diagnostics] = ProjectConfig.open(project_root)
     parser = rstparser.Parser(project_config, JSONVisitor)
 
     page, diagnostics = parse_rst(
@@ -2638,17 +2639,21 @@ def test_toctree() -> None:
    Title here </test1>
    /test2/faq
    URL with title <https://www.mongodb.com/docs/atlas>
+   Associated Product <|non-existent-associated|>
+   Associated Product <|test-name|>
    <https://www.mongodb.com/docs/stitch>
 """,
     )
     page.finish(diagnostics)
-
-    # MESSAGE need to create a toctree diagnostic
-    assert len(diagnostics) == 1 and "toctree" in diagnostics[0].message
+    assert (
+        len(diagnostics) == 2
+        and "toctree" in diagnostics[0].message
+        and "toctree" in diagnostics[1].message
+    )
     check_ast_testing_string(
         page.ast,
         """<root fileid="test.rst">
-        <directive name="toctree" titlesonly="True" entries="[{'title': 'Title here', 'slug': '/test1'}, {'slug': '/test2/faq'}, {'title': 'URL with title', 'url': 'https://www.mongodb.com/docs/atlas'}]" />
+        <directive name="toctree" titlesonly="True" entries="[{'title': 'Title here', 'slug': '/test1'}, {'slug': '/test2/faq'}, {'title': 'URL with title', 'url': 'https://www.mongodb.com/docs/atlas'}, {'title': 'Associated Product', 'ref_project': 'test-name'}]" />
         </root>""",
     )
 

snooty/test_postprocess.py

@@ -8,6 +8,7 @@
 from .diagnostics import (
     ChapterAlreadyExists,
     DocUtilsParseError,
+    DuplicatedExternalToc,
     DuplicateDirective,
     ExpectedPathArg,
     ExpectedTabs,
@@ -559,6 +560,45 @@ def test_toctree_self_add() -> None:
         )
 
 
+def test_toctree_duplicate_node() -> None:
+    with make_test(
+        {
+            Path(
+                "snooty.toml"
+            ): """
+name = "test_name"
+title = "MongoDB title"
+
+[[associated_products]]
+name = "test_associated_product"
+versions = ["v1", "v2"]
+            """,
+            Path(
+                "source/index.txt"
+            ): """
+.. toctree::
+
+    /page1
+    Duplicate Toc <|test_associated_product|>
+            """,
+            Path(
+                "source/page1.txt"
+            ): """
+==================
+Page 1
+==================
+
+.. toctree::
+
+   Duplicate Toc <|test_associated_product|>
+            """,
+        }
+    ) as result:
+        diagnostics = result.diagnostics[FileId("index.txt")]
+        assert len(diagnostics) == 1
+        assert isinstance(diagnostics[0], DuplicatedExternalToc)
+
+
 def test_case_sensitive_labels() -> None:
     with make_test(
         {