Load aws-doc-sdk-code-examples with 1:1 fidelity.

DavidSouther · DavidSouther · commit 758be943aab9 · 2024-12-09T17:05:05.000-05:00
diff --git a/aws_doc_sdk_examples_tools/labels/from_doc_gen.py b/aws_doc_sdk_examples_tools/labels/from_doc_gen.py
@@ -1,12 +1,12 @@
-from typing import Dict, Iterable, List, Set
+from typing import Dict, Iterable, List, Set, Optional
 
 from aws_doc_sdk_examples_tools.doc_gen import DocGen
 from aws_doc_sdk_examples_tools.metadata import Example as DocGenExample
 from aws_doc_sdk_examples_tools.snippets import Snippet as DocGenSnippet
 from aws_doc_sdk_examples_tools.sdks import Sdk as DocGenSdk
 from aws_doc_sdk_examples_tools.services import Service as DocGenService
 
-from .labels import Sdk, Service, Example, Snippet, Expanded, Label, Excerpt
+from .labels import Sdk, Service, Example, Snippet, Expanded, Label, Excerpt, Link, ApiRef
 from . import known_labels
 
 
@@ -37,11 +37,26 @@ def _sdk(id: str, doc_gen_sdk: DocGenSdk) -> Iterable[Sdk]:
         labels: List[Label] = []
         if v.caveat:
             labels.append(Label(name="caveat", value=v.caveat))
+        title_override = None
+        if v.title_override:
+            title_override = Expanded(
+                long=v.title_override.title or "",
+                short=v.title_override.title_abbrev or "",
+            )
+        api_ref = None
+        if v.api_ref:
+            api_ref = ApiRef(
+                uid=v.api_ref.uid,
+                name=v.api_ref.name,
+                link_template=v.api_ref.link_template,
+            )
         sdk = Sdk(
             language=doc_gen_sdk.property,
             version=str(v.version),
             name=Expanded(long=v.long, short=v.short),
-            labels=labels
+            labels=labels,
+            title_override=title_override,
+            api_ref=api_ref
         )
         yield sdk
 
@@ -59,13 +74,21 @@ def _service(id: str, doc_gen_service: DocGenService) -> Service :
     if doc_gen_service.expanded:
         expanded = Expanded(long=doc_gen_service.expanded.long, short=doc_gen_service.expanded.short)
 
+    guide: Optional[Link] = None
+    if doc_gen_service.guide:
+        guide = Link(
+            title=doc_gen_service.guide.subtitle,
+            url=doc_gen_service.guide.url
+        )
+
     service = Service(
         id=id,
         sort=doc_gen_service.sort,
         name=Expanded(long=doc_gen_service.long, short=doc_gen_service.short),
         expanded=expanded,
         version=str(doc_gen_service.version),
         labels=labels,
+        guide=guide,
     )
     return service
 
@@ -76,14 +99,27 @@ def _examples(examples: Dict[str, DocGenExample]) -> Set[Example]:
 
 def _example(id: str, doc_gen_example: DocGenExample) -> Example:
     labels = doc_gen_example_labels(doc_gen_example)
+    synopsis = []
+    if doc_gen_example.synopsis:
+        synopsis.append(doc_gen_example.synopsis)
+    if doc_gen_example.synopsis_list:
+        synopsis.extend(doc_gen_example.synopsis_list)
+    guide_topic: Optional[Link] = None
+    if doc_gen_example.guide_topic:
+        guide_topic = Link(
+            title=doc_gen_example.guide_topic.title,
+            url=doc_gen_example.guide_topic.url or doc_gen_example.guide_topic.title,
+        )
 
     example = Example(
         id=id,
-        title=doc_gen_example.title,
-        title_abbrev=doc_gen_example.title_abbrev,
-        synopsis=doc_gen_example.synopsis or "",
-        synopsis_list=doc_gen_example.synopsis_list,
+        title=Expanded(
+            long=doc_gen_example.title or "",
+            short=doc_gen_example.title_abbrev or "",
+        ),
+        synopsis=synopsis,
         labels=labels,
+        guide_topic=guide_topic
     )
 
     return example
@@ -110,6 +146,7 @@ def _snippets(doc_gen_examples: Iterable[DocGenExample], doc_gen_snippets: Dict[
             lang = lang.lower()
             for version in language.versions:
                 excerpts: List[Excerpt] = []
+
                 if version.block_content:
                     excerpts.append(Excerpt(description=version.block_content))
                 for excerpt in version.excerpts:
@@ -119,6 +156,7 @@ def _snippets(doc_gen_examples: Iterable[DocGenExample], doc_gen_snippets: Dict[
                         doc_gen_snippet = doc_gen_snippets.get(tag, None)
                         if doc_gen_snippet:
                             excerpts.append(Excerpt(path=doc_gen_snippet.file, range=(doc_gen_snippet.line_start, doc_gen_snippet.line_end), content=doc_gen_snippet.code))
+
                 labels: List[Label] = [
                     *example_labels,
                     Label(name=known_labels.SDK, value=f"{lang}:{version.sdk_version}"),
diff --git a/aws_doc_sdk_examples_tools/labels/from_doc_gen_test.py b/aws_doc_sdk_examples_tools/labels/from_doc_gen_test.py
@@ -6,7 +6,7 @@
 
 from . import from_doc_gen
 from . import known_labels
-from .labels import Sdk, Service, Example, Snippet, Expanded, Label, Excerpt, Context
+from .labels import Sdk, Service, Example, Snippet, Expanded, Label, Excerpt, Context, Link, ApiRef
 
 
 def test_from_doc_gen():
@@ -94,10 +94,11 @@ def test_from_doc_gen():
                 version="3",
                 language="javascript",
                 name=Expanded(long="&JSBlong; V3", short="&JSB; V3"),
-                #   api_ref:
-                #     uid: "AWSJavaScriptSDK",
-                #     name: "&guide-jsb-api;",
-                #     link_template: "AWSJavaScriptSDK/v3/latest/clients/client-{{.Service}}/classes/{{.OperationLower}}command.html",
+                api_ref=ApiRef(
+                    uid="AWSJavaScriptSDK",
+                    name="&guide-jsb-api;",
+                    link_template="AWSJavaScriptSDK/v3/latest/clients/client-{{.Service}}/classes/{{.OperationLower}}command.html",
+                ),
                 guide="&guide-jsb-dev;",
             ),
             Sdk(
@@ -118,9 +119,10 @@ def test_from_doc_gen():
                 version="3",
                 language="csharp",
                 name=Expanded(long="&NETlong;", short="&NET;"),
-                #   title_override:
-                #     title: "Additional &NET; code examples",
-                #     title_abbrev: "Additional code examples",
+                title_override=Expanded(
+                    long="Additional &NET; code examples",
+                    short="Additional code examples",
+                ),
                 guide="&guide-net-dev;",
             ),
             Sdk(
@@ -166,9 +168,10 @@ def test_from_doc_gen():
                     ),
                 ],
                 api_ref="AmazonS3/latest/API/Welcome.html",
-                #   guide:
-                #     subtitle: User Guide
-                #     url: AmazonS3/latest/userguide/Welcome.html
+                guide=Link(
+                    title="User Guide",
+                    url="AmazonS3/latest/userguide/Welcome.html"
+                )
             ),
             Service(
                 id="medical-imaging",
@@ -206,16 +209,19 @@ def test_from_doc_gen():
         [
             Example(
                 id="medical-imaging_TestExample",
-                title="Check whether a phone number is opted out using an &AWS; SDK",
-                title_abbrev="Check whether a phone number is opted out",
-                synopsis="check whether a phone number is opted out using some of the &AWS; SDKs that are available.",
-                synopsis_list=[
+                title=Expanded(
+                    long="Check whether a phone number is opted out using an &AWS; SDK",
+                    short="Check whether a phone number is opted out",
+                ),
+                synopsis=[
+                    "check whether a phone number is opted out using some of the &AWS; SDKs that are available.",
                     "Check the one thing.",
                     "Do some other thing.",
                 ],
-                # guide_topic:
-                #     title: Test guide topic title
-                #     url: test-guide/url
+                guide_topic=Link(
+                    title="Test guide topic title",
+                    url="test-guide/url"
+                ),
                 labels=[
                     Label(name=known_labels.CATEGORY, value="Usage"),
                     Label(name=known_labels.SERVICE, value="medical-imaging"),
diff --git a/aws_doc_sdk_examples_tools/labels/labels.py b/aws_doc_sdk_examples_tools/labels/labels.py
@@ -46,6 +46,12 @@ def covers(self, other: "Labeled"):
         return self._label_set.covers(other._label_set)
 
 
+@dataclass
+class Link:
+    title: str
+    url: str
+
+
 @dataclass
 class Context:
     ...
@@ -79,8 +85,8 @@ def __hash__(self):
 
 @dataclass
 class Expanded:
-    long: str
-    short: str
+    long: str = ""
+    short: str = ""
 
     def __eq__(self, other: object) -> bool:
         return isinstance(other, Expanded) and self.long == other.long and self.short == other.short
@@ -89,11 +95,19 @@ def __hash__(self):
         return hash((self.long, self.short))
 
 
+@dataclass
+class ApiRef:
+    uid: str
+    name: str
+    link_template: Optional[str] = None
+
 @dataclass
 class Sdk(Labeled):
     language: str = ""
     version: str = ""
-    name: Optional[Expanded] = None
+    name: Expanded = field(default_factory=Expanded)
+    title_override: Optional[Expanded] = None
+    api_ref: Optional[ApiRef] = None
     guide: str = ""
 
     def __post_init__(self):
@@ -110,12 +124,12 @@ def __hash__(self):
 @dataclass
 class Service(Labeled):
     id: str = ""
-    name: Optional[Expanded] = None
-    expanded: Optional[Expanded] = None
+    name: Expanded = field(default_factory=Expanded)
+    expanded: Expanded = field(default_factory=Expanded)
     sort: str = ""
     version: str = ""
     api_ref: Optional[str] = None
-    # guide: Optional[ServiceGuide] = None
+    guide: Optional[Link] = None
 
     def __post_init__(self):
         self.labels = [*self.labels, Label(name=known_labels.SERVICE, value=f"{self.id}")]
@@ -131,13 +145,11 @@ def __hash__(self):
 @dataclass
 class Example(Labeled):
     id: str = ""
-    title: Optional[str] = ""
-    title_abbrev: Optional[str] = ""
-    synopsis: Optional[str] = ""
+    title: Expanded = field(default_factory=Expanded)
     category: Optional[str] = None
-    # guide_topic: Optional[Url] = None
+    guide_topic: Optional[Link] = None
     # doc_filenames: Optional[DocFilenames] = None
-    synopsis_list: List[str] = field(default_factory=list)
+    synopsis: List[str] = field(default_factory=list)
 
     def __eq__(self, other):
         return isinstance(other, Example) and self.id == other.id
@@ -152,4 +164,3 @@ def select(items: Iterable[Labeled], for_labels_in: Labeled):
         if item.covers(for_labels_in):
             return_set.add(item)
     return frozenset(return_set)
-    # return frozenset(item for item in items if item.covers(for_labels_in))
diff --git a/aws_doc_sdk_examples_tools/labels/load_project.py b/aws_doc_sdk_examples_tools/labels/load_project.py
@@ -0,0 +1,57 @@
+import argparse
+from ast import literal_eval
+from pathlib import Path
+from time import time
+
+
+from aws_doc_sdk_examples_tools.doc_gen import DocGen
+from .from_doc_gen import from_doc_gen
+from .labels import select
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--root",
+        default=f"{Path(__file__).parent.parent.parent}",
+        help="The root path from which to search for files to check. The default is the root of the git repo (two up from this file).",
+    )
+    parser.add_argument(
+        "--doc_gen_only",
+        type=literal_eval,
+        default=True,
+        help="Only perform extended validation on snippet contents",
+        required=False,
+    )
+    parser.add_argument(
+        "--strict_titles",
+        type=literal_eval,
+        default=False,
+        help="Strict title requirements: Action examples must not have title/title_abbrev; non-Action examples "
+        "must have them.",
+        required=False,
+    )
+    args = parser.parse_args()
+    root_path = Path(args.root).resolve()
+
+    doc_gen = DocGen.from_root(root=root_path)
+    doc_gen.collect_snippets()
+
+    labeled = from_doc_gen(doc_gen)
+
+    # print(labeled)
+    print('Services: ', len(labeled['services']))
+    print('SDKs: ', len(labeled['sdks']))
+    print('Examples: ', len(labeled['examples']))
+    print('Snippets: ', len(labeled['snippets']))
+
+    for sdk in labeled['sdks']:
+        before = time()
+        snippets = select(labeled['snippets'], sdk)
+        after = time()
+        elapsed = after - before
+        print(f"{sdk.language}:{sdk.version} has {len(snippets)} snippets (took {(elapsed * 1000):.2} ms)")
+
+
+if __name__ == "__main__":
+    exit(main())
