DOP-5507: allow merging of rstspec and snooty configs for composables (#660)

* init commit. merge spec and snooty config for composables

* add diagnostics

* make spec parser intake rst spec as well

* log error from snooty toml

* allow writers to dictate dependencies in total

* return tuple

* fix keyerror

* address comments

* merge fix

Author: seungpark

snooty/main.py

@@ -40,7 +40,13 @@
 from .page import Page
 from .parser import Project, ProjectBackend, ProjectLoadError
 from .types import BuildIdentifierSet, ProjectConfig
-from .util import EXT_FOR_PAGE, SOURCE_FILE_EXTENSIONS, HTTPCache, PerformanceLogger
+from .util import (
+    EXT_FOR_PAGE,
+    SNOOTY_TOML,
+    SOURCE_FILE_EXTENSIONS,
+    HTTPCache,
+    PerformanceLogger,
+)
 
 PARANOID_MODE = os.environ.get("SNOOTY_PARANOID", "0") == "1"
 PATTERNS = ["*" + ext for ext in SOURCE_FILE_EXTENSIONS]
@@ -256,15 +262,6 @@ def main() -> None:
 
     logger.info(f"Snooty {__version__} starting")
 
-    if args["--rstspec"]:
-        rstspec_path = args["--rstspec"]
-        if rstspec_path.startswith("https://") or rstspec_path.startswith("http://"):
-            rstspec_bytes = HTTPCache.singleton().get(args["--rstspec"])
-            rstspec_text = str(rstspec_bytes, "utf-8")
-        else:
-            rstspec_text = Path(rstspec_path).expanduser().read_text(encoding="utf-8")
-        specparser.Spec.initialize(rstspec_text)
-
     if PARANOID_MODE:
         logger.info("Paranoid mode on")
 
@@ -283,6 +280,17 @@ def main() -> None:
     assert args["<source-path>"] is not None
     root_path = Path(args["<source-path>"])
 
+    if args["--rstspec"]:
+        rstspec_path = args["--rstspec"]
+        if rstspec_path.startswith("https://") or rstspec_path.startswith("http://"):
+            rstspec_bytes = HTTPCache.singleton().get(args["--rstspec"])
+            rstspec_text = str(rstspec_bytes, "utf-8")
+        else:
+            rstspec_text = Path(rstspec_path).expanduser().read_text(encoding="utf-8")
+        specparser.Spec.initialize(
+            rstspec_text, Path.joinpath(root_path, Path(SNOOTY_TOML))
+        )
+
     branch = args["--branch"]
 
     try:

snooty/parser.py

@@ -615,7 +615,9 @@ def handle_tabset(self, node: n.Directive) -> None:
         line = node.start[0]
         # retrieve dictionary associated with this specific tabset
         try:
-            tab_definitions_list = specparser.Spec.get().tabs[tabset]
+            tab_definitions_list = specparser.Spec.get(
+                self.project_config.config_path
+            ).tabs[tabset]
         except KeyError:
             self.diagnostics.append(UnknownTabset(tabset, line))
             return
@@ -675,7 +677,9 @@ def handle_tabset(self, node: n.Directive) -> None:
         )
 
     def handle_wayfinding(self, node: n.Directive) -> None:
-        expected_options = specparser.Spec.get().wayfinding["options"]
+        expected_options = specparser.Spec.get(
+            self.project_config.config_path
+        ).wayfinding["options"]
         expected_options_dict = {option.id: option for option in expected_options}
         expected_child_opt_name = "wayfinding-option"
         expected_child_desc_name = "wayfinding-description"
@@ -797,7 +801,9 @@ def check_valid_option_id(
             raise ChildValidationError()
 
     def handle_method_selector(self, node: n.Directive) -> None:
-        expected_options = specparser.Spec.get().method_selector["options"]
+        expected_options = specparser.Spec.get(
+            self.project_config.config_path
+        ).method_selector["options"]
         expected_options_dict = {option.id: option for option in expected_options}
         expected_child_name = "method-option"
 
@@ -885,7 +891,9 @@ def handle_composable(self, node: n.ComposableDirective) -> None:
             )
 
         # get the expected composable options from the spec
-        spec_composables = specparser.Spec.get().composables
+        spec_composables = specparser.Spec.get(
+            self.project_config.config_path
+        ).composables
         spec_composables_dict = {
             expected_option.id: expected_option for expected_option in spec_composables
         }

snooty/postprocess.py

@@ -1697,7 +1697,7 @@ def __init__(self, context: Context) -> None:
         super().__init__(context)
         self.project_config = context[ProjectConfig]
         self.targets = context[TargetDatabase]
-        self.spec = specparser.Spec.get()
+        self.spec = specparser.Spec.get(self.project_config.config_path)
 
     def enter_node(self, fileid_stack: FileIdStack, node: n.Node) -> None:
         """When a node of type ref_role is encountered, ensure that it references a valid target.

snooty/specparser.py

@@ -1,12 +1,14 @@
 """Parser for a TOML spec file containing definitions of all supported reStructuredText
-   directives and roles, and what types of data each should expect."""
+directives and roles, and what types of data each should expect."""
 
 from __future__ import annotations
 
 import dataclasses
+import logging
 from dataclasses import dataclass, field
 from datetime import datetime
 from enum import Enum
+from pathlib import Path
 from typing import (
     Any,
     Callable,
@@ -18,13 +20,16 @@
     Optional,
     Sequence,
     Set,
+    Tuple,
     TypeVar,
     Union,
 )
 
 import tomli
 from typing_extensions import Protocol
 
+from snooty.diagnostics import Diagnostic, UnknownOptionId
+
 from . import tinydocutils, util
 from .flutter import check_type, checked
 
@@ -70,6 +75,8 @@
     PrimitiveType.linenos: util.option_string,
 }
 
+logger = logging.getLogger(__name__)
+
 #: Option types can be a primitive type (PrimitiveType), an enum
 #: defined in the spec, or a union of those.
 ArgumentType = Union[List[Union[PrimitiveType, str]], PrimitiveType, str, None]
@@ -308,6 +315,7 @@ class Spec:
     wayfinding: Dict[str, List[WayfindingOption]] = field(default_factory=dict)
     data_fields: List[str] = field(default_factory=list)
     composables: List[Composable] = field(default_factory=list)
+    merged: bool = False
 
     SPEC: ClassVar[Optional[Spec]] = None
 
@@ -439,15 +447,131 @@ def resolve_value(key: str, inheritable: _T) -> _T:
             resolve_value(key, inheritable)
 
     @classmethod
-    def initialize(cls, text: str) -> None:
+    def _merge_composables(
+        cls, spec: Spec, custom_composables: List[Dict[str, Any]]
+    ) -> Tuple[Spec, List[Diagnostic]]:
+        res: List[Composable] = []
+        diagnostics: List[Diagnostic] = []
+
+        custom_composable_by_id = {
+            composable["id"]: composable for composable in custom_composables
+        }
+
+        for defined_composable in spec.composables:
+            custom_composable = custom_composable_by_id.pop(defined_composable.id, None)
+            if not custom_composable:
+                res.append(defined_composable)
+                continue
+            merged_title = custom_composable["title"]
+
+            # merge all the options
+            defined_options = {
+                option.id: option for option in defined_composable.options
+            }
+            custom_options = {
+                option["id"]: option for option in custom_composable["options"]
+            }
+
+            merged_options = []
+            for option_id in set(defined_options.keys()) | set(custom_options.keys()):
+                if option_id in custom_options:
+                    custom_option = custom_options[option_id]
+                    merged_options.append(
+                        TabDefinition(custom_option["id"], custom_option["title"])
+                    )
+                else:
+                    merged_options.append(defined_options[option_id])
+
+            merged_dependencies = (
+                custom_composable["dependencies"]
+                if "dependencies" in custom_composable
+                else defined_composable.dependencies
+            )
+
+            merged_default = (
+                custom_composable["default"]
+                if "default" in custom_composable
+                else defined_composable.default
+            )
+            default_option = next(
+                (
+                    option
+                    for option in merged_options
+                    if merged_default and option.id == merged_default
+                ),
+                None,
+            )
+            if merged_default and not default_option:
+                diagnostics.append(
+                    UnknownOptionId(
+                        "Spec composables default",
+                        merged_default,
+                        [option.title for option in merged_options],
+                        0,
+                    )
+                )
+            res.append(
+                Composable(
+                    defined_composable.id,
+                    merged_title,
+                    merged_default,
+                    merged_dependencies,
+                    merged_options,
+                )
+            )
+
+        for composable_obj in custom_composable_by_id.values():
+            res.append(
+                Composable(
+                    composable_obj["id"],
+                    composable_obj["title"],
+                    composable_obj["default"] if "default" in composable_obj else None,
+                    (
+                        composable_obj["dependencies"]
+                        if "dependencies" in composable_obj
+                        else None
+                    ),
+                    list(
+                        map(
+                            lambda option: TabDefinition(option["id"], option["title"]),
+                            composable_obj["options"],
+                        )
+                    ),
+                )
+            )
+
+        spec.composables = res
+        return (spec, diagnostics)
+
+    @classmethod
+    def initialize(cls, text: str, configPath: Optional[Path]) -> "Spec":
         cls.SPEC = Spec.loads(text)
+        if configPath:
+            project_config = tomli.loads(configPath.read_text(encoding="utf-8"))
+            # NOTE: would like to check_type but circular imports
+            # this is already verified earlier in the process
+            spec, diagnostics = cls._merge_composables(
+                cls.SPEC,
+                (
+                    project_config["composables"]
+                    if "composables" in project_config
+                    else []
+                ),
+            )
+            spec.merged = True
+            cls.SPEC = spec
+            for diagnostic in diagnostics:
+                logger.error(diagnostic)
+        return cls.SPEC
 
     @classmethod
-    def get(cls) -> "Spec":
-        if cls.SPEC is None:
-            path = util.PACKAGE_ROOT.joinpath("rstspec.toml")
-            cls.initialize(path.read_text(encoding="utf-8"))
-
-        spec = cls.SPEC
-        assert spec is not None
-        return spec
+    def get(cls, configPath: Optional[Path] = None) -> "Spec":
+        if cls.SPEC and cls.SPEC.merged:
+            return cls.SPEC
+
+        path = util.PACKAGE_ROOT.joinpath("rstspec.toml")
+        spec = cls.initialize(path.read_text(encoding="utf-8"), configPath)
+
+        cls.SPEC = spec
+        assert cls.SPEC is not None
+        return cls.SPEC

snooty/types.py

@@ -218,6 +218,7 @@ class ProjectConfig:
     bundle: BundleConfig = field(default_factory=BundleConfig)
     data: Dict[str, object] = field(default_factory=dict)
     associated_products: List[AssociatedProduct] = field(default_factory=list)
+    composables: List[specparser.Composable] = field(default_factory=list)
 
     # banner_nodes contains parsed banner nodes with target data
     banner_nodes: List[ParsedBannerConfig] = field(