DocGen: add complex categories (#126)

Author: Laren-AWS

aws_doc_sdk_examples_tools/categories.py

@@ -0,0 +1,115 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+from dataclasses import dataclass, field
+
+from aws_doc_sdk_examples_tools import metadata_errors
+from .metadata_errors import (
+    MetadataErrors,
+)
+
+
+@dataclass
+class TitleInfo:
+    title: Optional[str] = field(default=None)
+    title_abbrev: Optional[str] = field(default=None)
+    synopsis: Optional[str] = field(default=None)
+    title_suffixes: str | Dict[str, str] = field(default_factory=dict)
+
+    @classmethod
+    def from_yaml(cls, yaml: Dict[str, str] | None) -> Optional[TitleInfo]:
+        if yaml is None:
+            return None
+
+        title = yaml.get("title")
+        title_suffixes: str | Dict[str, str] = yaml.get("title_suffixes", {})
+        title_abbrev = yaml.get("title_abbrev")
+        synopsis = yaml.get("synopsis")
+
+        return cls(
+            title=title,
+            title_suffixes=title_suffixes,
+            title_abbrev=title_abbrev,
+            synopsis=synopsis,
+        )
+
+
+@dataclass
+class CategoryWithNoDisplayError(metadata_errors.MetadataError):
+    def message(self):
+        return "Category has no display value"
+
+
+@dataclass
+class Category:
+    key: str
+    display: str
+    defaults: Optional[TitleInfo] = field(default=None)
+    overrides: Optional[TitleInfo] = field(default=None)
+    description: Optional[str] = field(default=None)
+
+    def validate(self, errors: MetadataErrors):
+        if not self.display:
+            errors.append(CategoryWithNoDisplayError(id=self.key))
+
+    @classmethod
+    def from_yaml(
+        cls, key: str, yaml: Dict[str, Any]
+    ) -> tuple[Category, MetadataErrors]:
+        errors = MetadataErrors()
+        display = str(yaml.get("display"))
+        defaults = TitleInfo.from_yaml(yaml.get("defaults"))
+        overrides = TitleInfo.from_yaml(yaml.get("overrides"))
+        description = yaml.get("description")
+
+        return (
+            cls(
+                key=key,
+                display=display,
+                defaults=defaults,
+                overrides=overrides,
+                description=description,
+            ),
+            errors,
+        )
+
+
+def parse(
+    file: Path, yaml: Dict[str, Any]
+) -> tuple[List[str], Dict[str, Category], MetadataErrors]:
+    categories: Dict[str, Category] = {}
+    errors = MetadataErrors()
+
+    standard_cats = yaml.get("standard_categories", [])
+    # Work around inconsistency where some tools use 'Actions' and DocGen uses 'Api' to refer to single-action examples.
+    for i in range(len(standard_cats)):
+        if standard_cats[i] == "Actions":
+            standard_cats[i] = "Api"
+    for key, yaml_cat in yaml.get("categories", {}).items():
+        if yaml_cat is None:
+            errors.append(metadata_errors.MissingCategoryBody(id=key, file=file))
+        else:
+            category, cat_errs = Category.from_yaml(key, yaml_cat)
+            categories[key] = category
+            for error in cat_errs:
+                error.file = file
+                error.id = key
+            errors.extend(cat_errs)
+
+    return standard_cats, categories, errors
+
+
+if __name__ == "__main__":
+    from pprint import pp
+    import yaml
+
+    path = Path(__file__).parent / "config" / "categories.yaml"
+    with open(path) as file:
+        meta = yaml.safe_load(file)
+    standard_cats, cats, errs = parse(path, meta)
+    pp(standard_cats)
+    pp(cats)

aws_doc_sdk_examples_tools/categories_test.py

@@ -0,0 +1,73 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from pathlib import Path
+from typing import Dict, List, Tuple
+import pytest
+import yaml
+
+from aws_doc_sdk_examples_tools import metadata_errors
+from .categories import (
+    parse,
+    Category,
+    TitleInfo,
+)
+
+
+def load(
+    path: str,
+) -> Tuple[List[str], Dict[str, Category], metadata_errors.MetadataErrors]:
+    root = Path(__file__).parent
+    filename = root / "test_resources" / path
+    with open(filename) as file:
+        meta = yaml.safe_load(file)
+    return parse(filename, meta)
+
+
+def test_empty_categories():
+    _, _, errs = load("empty_categories.yaml")
+    assert [*errs] == [
+        metadata_errors.MissingCategoryBody(
+            file=Path(__file__).parent / "test_resources/empty_categories.yaml",
+            id="EmptyCat",
+        )
+    ]
+
+
+def test_categories():
+    _, categories, _ = load("categories.yaml")
+    assert categories == {
+        "Actions": Category(
+            key="Actions",
+            display="Actions test",
+            overrides=TitleInfo(
+                title="Title override",
+                title_suffixes={
+                    "cli": " with a CLI",
+                    "sdk": " with an &AWS; SDK",
+                    "sdk_cli": " with an &AWS; SDK or CLI",
+                },
+                title_abbrev="Title abbrev override",
+                synopsis="synopsis test.",
+            ),
+            description="test description.",
+        ),
+        "Basics": Category(
+            key="Basics",
+            display="Basics",
+            defaults=TitleInfo(
+                title="Title default",
+                title_abbrev="Title abbrev default",
+            ),
+            description="default description.",
+        ),
+        "TributaryLite": Category(
+            key="TributaryLite",
+            display="Tea light",
+            description="light your way.",
+        ),
+    }
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-vv"])

aws_doc_sdk_examples_tools/config/categories.yaml

@@ -0,0 +1,34 @@
+standard_categories: ["Hello", "Basics", "Actions", "Scenarios"]
+categories:
+  Hello:
+    display: "Hello"
+    overrides:
+      title: "Hello {{.ServiceEntity.Short}}"
+      title_abbrev: "Hello {{.ServiceEntity.Short}}"
+      synopsis: "get started using {{.ServiceEntity.Short}}."
+  Actions:
+    display: "Actions"
+    overrides:
+      title: "Use <code>{{.Action}}</code>"
+      title_suffixes:
+        cli: " with a CLI"
+        sdk: " with an &AWS; SDK"
+        sdk_cli: " with an &AWS; SDK or CLI"
+      title_abbrev: "<code>{{.Action}}</code>"
+      synopsis: "use <code>{{.Action}}</code>."
+    description: "are code excerpts from larger programs and must be run in context. While actions
+      show you how to call individual service functions, you can see actions in context in their related scenarios."
+  Basics:
+    display: "Basics"
+    defaults:
+      title: "Learn the basics of {{.ServiceEntity.Short}} with an &AWS; SDK"
+      title_abbrev: "Learn the basics"
+    description: "are code examples that show you how to perform the essential operations within a service."
+  Scenarios:
+    display: "Scenarios"
+    description: "are code examples that show you how to accomplish specific tasks by
+      calling multiple functions within a service or combined with other &AWS-services;."
+  TributaryLite:
+    display: "&AWS; community contributions"
+    description: "are examples that were created and are maintained by multiple teams across &AWS;. 
+      To provide feedback, use the mechanism provided in the linked repositories."

aws_doc_sdk_examples_tools/doc_gen.py

@@ -12,6 +12,7 @@
 
 # from os import glob
 
+from .categories import Category, parse as parse_categories
 from .metadata import (
     Example,
     DocFilenames,
@@ -55,6 +56,8 @@ class DocGen:
     validation: ValidationConfig = field(default_factory=ValidationConfig)
     sdks: Dict[str, Sdk] = field(default_factory=dict)
     services: Dict[str, Service] = field(default_factory=dict)
+    standard_categories: List[str] = field(default_factory=list)
+    categories: Dict[str, Category] = field(default_factory=dict)
     snippets: Dict[str, Snippet] = field(default_factory=dict)
     snippet_files: Set[str] = field(default_factory=set)
     examples: Dict[str, Example] = field(default_factory=dict)
@@ -201,6 +204,19 @@ def for_root(
         except Exception:
             pass
 
+        try:
+            categories_path = config / "categories.yaml"
+            with categories_path.open(encoding="utf-8") as file:
+                meta = yaml.safe_load(file)
+                standard_categories, categories, errs = parse_categories(
+                    categories_path, meta
+                )
+                self.standard_categories = standard_categories
+                self.categories = categories
+                self.errors.extend(errs)
+        except Exception:
+            pass
+
         try:
             entities_config_path = config / "entities.yaml"
             with entities_config_path.open(encoding="utf-8") as file:
@@ -236,6 +252,7 @@ def process_metadata(self, path: Path) -> "DocGen":
                 yaml.safe_load(file),
                 self.sdks,
                 self.services,
+                self.standard_categories,
                 self.cross_blocks,
                 self.validation,
                 self.root,
@@ -268,6 +285,8 @@ def validate(self):
             sdk.validate(self.errors)
         for service in self.services.values():
             service.validate(self.errors)
+        for category in self.categories.values():
+            category.validate(self.errors)
         for example in self.examples.values():
             example.validate(self.errors, self.root)
         validate_metadata(self.root, self.validation.strict_titles, self.errors)
@@ -339,6 +358,7 @@ def parse_examples(
     yaml: Dict[str, Any],
     sdks: Dict[str, Sdk],
     services: Dict[str, Service],
+    standard_categories: List[str],
     blocks: Set[str],
     validation: Optional[ValidationConfig],
     root: Optional[Path] = None,
@@ -350,12 +370,13 @@ def parse_examples(
         example, example_errors = example_from_yaml(
             yaml[id], sdks, services, blocks, validation, root or file.parent
         )
-        check_id_format(
-            id,
-            example.services,
-            validation.strict_titles and example.category == "Api",
-            example_errors,
-        )
+        if example.category in standard_categories:
+            check_id_format(
+                id,
+                example.services,
+                validation.strict_titles and example.category == "Api",
+                example_errors,
+            )
         for error in example_errors:
             error.file = file
             error.id = id
@@ -411,7 +432,7 @@ def get_doc_filenames(example_id: str, example: Example) -> Optional[DocFilename
                     )
                 )
             else:
-                anchor = "actions" if example.category == "Actions" else "scenarios"
+                anchor = "actions" if example.category == "Api" else "scenarios"
                 sdk_pages[language.property][version.sdk_version] = SDKPageVersion(
                     actions_scenarios={
                         service_id: f"{base_url}/{language.property}_{version.sdk_version}_{service_id}_code_examples.html#{anchor}"

aws_doc_sdk_examples_tools/metadata_errors.py

@@ -370,6 +370,12 @@ def message(self):
         return f"URL {self.url} is missing a title"
 
 
+@dataclass
+class MissingCategoryBody(MetadataParseError):
+    def message(self):
+        return "category definition missing body"
+
+
 @dataclass
 class ExampleMergeMismatchedId(MetadataError):
     other_id: str = ""