Improve service detection during example loading.

aws_doc_sdk_examples_tools/categories.py

@@ -88,7 +88,7 @@ def evaluate(
         defaults = field(self.defaults or empty_title_info)
         if defaults:
             return fake_gotmpl(defaults, service, action)
-        return ""
+        return f"{service} {action}"
 
     def validate(self, errors: MetadataErrors):
         if not self.display:

aws_doc_sdk_examples_tools/doc_gen.py

@@ -121,6 +121,7 @@ def merge(self, other: "DocGen") -> MetadataErrors:
         for name, service in other.services.items():
             if name not in self.services:
                 self.services[name] = service
+            else:
                 warnings.append(
                     DocGenMergeWarning(
                         file=other.root, id=f"conflict in service {name}"
@@ -129,6 +130,7 @@ def merge(self, other: "DocGen") -> MetadataErrors:
         for name, snippet in other.snippets.items():
             if name not in self.snippets:
                 self.snippets[name] = snippet
+            else:
                 warnings.append(
                     DocGenMergeWarning(
                         file=other.root, id=f"conflict in snippet {name}"
@@ -177,7 +179,7 @@ def clone(self) -> "DocGen":
             root=self.root,
             validation=self.validation.clone(),
             sdks={**self.sdks},
-            entities=self.entities,
+            entities={**self.entities},
             services={**self.services},
             errors=MetadataErrors(),
             snippets={},
@@ -277,7 +279,6 @@ def process_metadata(self, path: Path) -> "DocGen":
                 self.standard_categories,
                 self.cross_blocks,
                 self.validation,
-                self.root,
             )
             self.extend_examples(examples, self.errors)
             self.errors.extend(errs)
@@ -310,7 +311,7 @@ def validate(self):
         for category in self.categories.values():
             category.validate(self.errors)
         for example in self.examples.values():
-            example.validate(self.errors, self.root)
+            example.validate(self.errors, self.services, self.root)
         validate_metadata(self.root, self.validation.strict_titles, self.errors)
         validate_no_duplicate_api_examples(self.examples.values(), self.errors)
         validate_snippets(
@@ -323,22 +324,25 @@ def validate(self):
 
     def fill_missing_fields(self):
         for example in self.examples.values():
+            id_service, id_action = example.id.split("_", 1)
             service_id = example.service_main or next(
                 (k for (k, _) in example.services.items()), None
             )
             if service_id is None:
                 # TODO Log and find which tributaries this effects, as it was supposed to be caught by validations.
-                continue
+                service_id = id_service
             action = (
                 next((k for k in example.services.get(service_id, [])), None)
                 or example.id.split("_", 1)[1]
             )
             if action is None:
                 # TODO Log and find which tributaries this effects, as it was supposed to be caught by validations.
-                continue
-            example.fill_display_fields(
-                self.categories, self.services[service_id].short, action
-            )
+                action = id_action
+            if service_id in self.services:
+                service_name = self.services[service_id].short
+            else:
+                service_name = service_id
+            example.fill_display_fields(self.categories, service_name, action)
 
     def stats(self):
         values = self.examples.values()
@@ -402,14 +406,13 @@ def parse_examples(
     standard_categories: List[str],
     blocks: Set[str],
     validation: Optional[ValidationConfig],
-    root: Optional[Path] = None,
 ) -> Tuple[List[Example], MetadataErrors]:
     examples: List[Example] = []
     errors = MetadataErrors()
     validation = validation or ValidationConfig()
     for id in yaml:
         example, example_errors = example_from_yaml(
-            yaml[id], sdks, services, blocks, validation, root or file.parent
+            yaml[id], sdks, services, blocks, validation
         )
         if example.category in standard_categories:
             check_id_format(

aws_doc_sdk_examples_tools/metadata.py

@@ -6,7 +6,7 @@
 
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Dict, Literal, List, Optional, Set, Iterable
+from typing import Container, Dict, Literal, List, Optional, Set, Iterable
 from os.path import splitext
 from pathlib import Path
 
@@ -55,7 +55,6 @@ class Version:
     excerpts: List[Excerpt] = field(default_factory=list)
     # Link to the source code for this example. TODO rename.
     github: Optional[str] = field(default=None)
-    add_services: Dict[str, Set[str]] = field(default_factory=dict)
     # Deprecated. Replace with guide_topic list.
     sdkguide: Optional[str] = field(default=None)
     # Link to additional topic places.
@@ -209,8 +208,17 @@ def merge(self, other: Example, errors: MetadataErrors):
                         err.other_file = other.file  # type: ignore
                 errors.extend(merge_errs)
 
-    def validate(self, errors: MetadataErrors, root: Path):
+    def validate(
+        self, errors: MetadataErrors, known_services: Container[str], root: Path
+    ):
         errs = MetadataErrors()
+        for service in self.services.keys():
+            if service not in known_services:
+                errors.append(
+                    metadata_errors.UnknownService(
+                        id=self.id, file=self.file, service=service
+                    )
+                )
         for language in self.languages.values():
             language.validate(errs, root)
         for error in errs:

aws_doc_sdk_examples_tools/metadata_errors.py

@@ -6,7 +6,7 @@
 import re
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Optional, Iterator, Iterable, List, TypeVar, Generic
+from typing import Optional, Iterator, Iterable, List, TypeVar, Generic, Dict, Set
 
 
 ErrorT = TypeVar("ErrorT")
@@ -270,10 +270,18 @@ def message(self):
 class APIExampleCannotAddService(SdkVersionError):
     def message(self):
         return (
-            "is an API example but lists additional services in the add_service field."
+            "is an API example but lists additional services in the add_services field."
         )
 
 
+@dataclass
+class AddServicesHasBeenDeprecated(SdkVersionError):
+    add_services: Dict[str, Set[str]] = field(default_factory=dict)
+
+    def message(self):
+        return "lists additional services in add_services, which has been deprecated."
+
+
 @dataclass
 class APIMustHaveOneServiceOneAction(MetadataParseError):
     svc_actions: str = ""

aws_doc_sdk_examples_tools/metadata_test.py

@@ -484,7 +484,6 @@ def test_verify_load_successful():
                 github="test_path",
                 block_content="test block",
                 excerpts=[],
-                add_services={},
                 sdkguide=None,
                 more_info=[],
             ),
@@ -498,7 +497,6 @@ def test_verify_load_successful():
             Version(
                 sdk_version=3,
                 block_content=None,
-                add_services={"s3": set()},
                 excerpts=[
                     Excerpt(
                         description="Descriptive",
@@ -533,7 +531,6 @@ def test_verify_load_successful():
                         snippet_files=["snippet_file.txt"],
                     )
                 ],
-                add_services={},
                 more_info=[],
             )
         ],
@@ -646,11 +643,12 @@ def test_verify_load_successful():
                     language="Perl",
                     sdk_version=1,
                 ),
-                metadata_errors.APIExampleCannotAddService(
+                metadata_errors.AddServicesHasBeenDeprecated(
                     file=ERRORS_METADATA_PATH,
                     id="sqs_WrongServiceSlug",
                     language="Perl",
                     sdk_version=1,
+                    add_services={"sqs": set()},
                 ),
                 metadata_errors.ServiceNameFormat(
                     file=ERRORS_METADATA_PATH,
@@ -677,11 +675,6 @@ def test_verify_load_successful():
                 #     sdk_version=2,
                 #     tag="this.snippet.does.not.exist",
                 # ),
-                metadata_errors.UnknownService(
-                    file=ERRORS_METADATA_PATH,
-                    id="medical-imaging_TestExample2",
-                    service="garbled",
-                ),
                 metadata_errors.FieldError(
                     file=ERRORS_METADATA_PATH,
                     id="medical-imaging_TestExample2",
@@ -696,6 +689,13 @@ def test_verify_load_successful():
                     language="Java",
                     sdk_version=2,
                 ),
+                metadata_errors.AddServicesHasBeenDeprecated(
+                    file=ERRORS_METADATA_PATH,
+                    id="cross_TestExample_Versions",
+                    language="Java",
+                    sdk_version=2,
+                    add_services={"sqs": set()},
+                ),
                 metadata_errors.MissingCrossContent(
                     file=ERRORS_METADATA_PATH,
                     id="cross_TestExample_Missing",
@@ -723,6 +723,11 @@ def test_verify_load_successful():
                     link="perl/example_code/medical-imaging",
                     root=TEST_RESOURCES_PATH,
                 ),
+                metadata_errors.UnknownService(
+                    file=ERRORS_METADATA_PATH,
+                    id="medical-imaging_TestExample2",
+                    service="garbled",
+                ),
                 metadata_errors.InvalidGithubLink(
                     file=ERRORS_METADATA_PATH,
                     id="medical-imaging_TestExample2",
@@ -739,12 +744,12 @@ def test_verify_load_successful():
                     file=FORMATTER_METADATA_PATH,
                     id="WrongNameFormat",
                 ),
-                metadata_errors.UnknownService(
+                metadata_errors.AddServicesHasBeenDeprecated(
                     file=FORMATTER_METADATA_PATH,
                     id="cross_TestExample",
                     language="Java",
                     sdk_version=2,
-                    service="garbage",
+                    add_services={"garbage": set()},
                 ),
             ],
             [],
@@ -761,7 +766,7 @@ def test_common_errors(
     assert expected_errors == [*actual]
     validations = MetadataErrors()
     for example in examples:
-        example.validate(validations, root.parent)
+        example.validate(validations, DOC_GEN.services, root.parent)
     assert validation_errors == [*validations]
 
 
@@ -976,6 +981,10 @@ def test_no_duplicate_title_abbrev():
                 services={"svc": set(), "cvs": set()},
             ),
         },
+        services={
+            "svc": Service(long="Service", short="svc", version="1", sort="svc"),
+            "cvs": Service(long="CVS", short="cvs", version="2", sort="cvs"),
+        },
     )
     doc_gen.validate()
 

aws_doc_sdk_examples_tools/test_resources/valid_metadata.yaml

@@ -24,8 +24,6 @@ medical-imaging_TestExample:
               genai: some
               snippet_tags:
                 - medical-imaging.JavaScript.datastore.createDatastoreV3
-          add_services:
-            s3:
     PHP:
       versions:
         - sdk_version: 3