Add DocGen method to replace entities. (#56)

* Add DocGen method to replace entities.

* Change entities import to relative to package.

* Update doc gen cli to expand entities in additional fields and add a flag to disable entity expansion.

Author: cpyle0819

aws_doc_sdk_examples_tools/config/entities.yaml

@@ -0,0 +1,17 @@
+expanded_override:
+  "&AWS;": "AWS"
+  "&aws_sec_sdk_use-federation-warning;": ""
+  "&ASH;": "Security Hub"
+  "&DAX;": "DynamoDB Accelerator"
+  "&EC2long;": "Amazon Elastic Compute Cloud"
+  "&ELB;": "Elastic Load Balancing"
+  "&ELBlong;": "Elastic Load Balancing"
+  "&GLUDCLong;": "AWS Glue Data Catalog"
+  "&GLUDC;": "Data Catalog"
+  "&IAM-user;": "IAM user"
+  "&IAM-users;": "IAM users"
+  "&kms-key;": "KMS key"
+  "&kms-keys;": "KMS keys"
+  "&kms-keys-long;": "AWS KMS keys"
+  "&S3long;": "Amazon Simple Storage Service"
+  "&SLN;": "Amazon States Language"

aws_doc_sdk_examples_tools/doc_gen.py

@@ -8,7 +8,7 @@
 from dataclasses import dataclass, field, is_dataclass, asdict
 from functools import reduce
 from pathlib import Path
-from typing import Dict, Iterable, Optional, Set
+from typing import Dict, Iterable, Optional, Set, Tuple
 
 # from os import glob
 
@@ -17,6 +17,7 @@
     parse as parse_examples,
     validate_no_duplicate_api_examples,
 )
+from .entities import expand_all_entities, EntityErrors
 from .metadata_errors import MetadataErrors, MetadataError
 from .metadata_validator import validate_metadata
 from .project_validator import ValidationConfig
@@ -39,6 +40,7 @@ class DocGenMergeWarning(MetadataError):
 class DocGen:
     root: Path
     errors: MetadataErrors
+    entities: Dict[str, str] = field(default_factory=dict)
     prefix: Optional[str] = None
     validation: ValidationConfig = field(default_factory=ValidationConfig)
     sdks: Dict[str, Sdk] = field(default_factory=dict)
@@ -72,6 +74,9 @@ def languages(self) -> Set[str]:
                 languages.add(f"{sdk_name}:{version.version}")
         return languages
 
+    def expand_entities(self, text: str) -> Tuple[str, EntityErrors]:
+        return expand_all_entities(text, self.entities)
+
     def merge(self, other: "DocGen") -> MetadataErrors:
         """Merge fields from other into self, prioritizing self fields."""
         warnings = MetadataErrors()
@@ -99,6 +104,16 @@ def merge(self, other: "DocGen") -> MetadataErrors:
                     )
                 )
 
+        for entity, expanded in other.entities.items():
+            if entity not in self.entities:
+                self.entities[entity] = expanded
+            else:
+                warnings.append(
+                    DocGenMergeWarning(
+                        file=other.root, id=f"conflict in entity {entity}"
+                    )
+                )
+
         self.snippet_files.update(other.snippet_files)
         self.cross_blocks.update(other.cross_blocks)
         self.extend_examples(other.examples.values(), warnings)
@@ -166,10 +181,23 @@ def for_root(
                 meta = yaml.safe_load(file)
                 services, service_errors = parse_services(services_path, meta)
                 self.services = services
+                for service in self.services.values():
+                    if service.expanded:
+                        self.entities[service.long] = service.expanded.long
+                        self.entities[service.short] = service.expanded.short
                 self.errors.extend(service_errors)
         except Exception:
             pass
 
+        try:
+            entities_config_path = config / "entities.yaml"
+            with entities_config_path.open(encoding="utf-8") as file:
+                entities_config = yaml.safe_load(file)
+            for entity, expanded in entities_config["expanded_override"].items():
+                self.entities[entity] = expanded
+        except Exception as e:
+            pass
+
         metadata = root / ".doc_gen/metadata"
         try:
             self.cross_blocks = set(
@@ -280,6 +308,11 @@ def default(self, obj):
         if isinstance(obj, MetadataErrors):
             return {"__metadata_errors__": [asdict(error) for error in obj]}
 
+        if isinstance(obj, EntityErrors):
+            return {
+                "__entity_errors__": [{error.entity: error.message()} for error in obj]
+            }
+
         if isinstance(obj, set):
             return {"__set__": list(obj)}
 

aws_doc_sdk_examples_tools/doc_gen_cli.py

@@ -7,6 +7,7 @@
 import logging
 
 from .doc_gen import DocGen, DocGenEncoder
+from .entities import EntityErrors
 
 logging.basicConfig(level=logging.INFO)
 
@@ -34,6 +35,12 @@ def main():
         help="Exit with non-zero code if errors are present. By default errors are written to the output.",
     )
 
+    parser.add_argument(
+        "--skip-entity-expansion",
+        action="store_true",
+        help="Do not expand entities. Entities are expanded by default.",
+    )
+
     args = parser.parse_args()
 
     merged_doc_gen = DocGen.empty()
@@ -45,6 +52,41 @@ def main():
         logging.error("Errors found in metadata: %s", merged_doc_gen.errors)
         exit(1)
 
+    if not args.skip_entity_expansion:
+        # Replace entities
+        for example in merged_doc_gen.examples.values():
+            errors = EntityErrors()
+            print(example)
+            title, title_errors = merged_doc_gen.expand_entities(example.title)
+            errors.extend(title_errors)
+
+            title_abbrev, title_abbrev_errors = merged_doc_gen.expand_entities(
+                example.title_abbrev
+            )
+            errors.extend(title_abbrev_errors)
+
+            synopsis, synopsis_errors = merged_doc_gen.expand_entities(example.synopsis)
+            errors.extend(synopsis_errors)
+
+            synopsis_list = []
+            for synopsis in example.synopsis_list:
+                expanded_synopsis, synopsis_errors = merged_doc_gen.expand_entities(
+                    synopsis
+                )
+                synopsis_list.append(expanded_synopsis)
+                errors.extend(synopsis_errors)
+
+            if errors:
+                logging.error(
+                    f"Errors expanding entities for example: {example}. {errors}"
+                )
+                exit(1)
+
+            example.title = title
+            example.title_abbrev = title_abbrev
+            example.synopsis = synopsis
+            example.synopsis_list = synopsis_list
+
     serialized = json.dumps(merged_doc_gen, cls=DocGenEncoder)
 
     with open(args.write_json, "w") as out:

aws_doc_sdk_examples_tools/doc_gen_cli_test.py

@@ -1,18 +1,60 @@
 import pytest
 from unittest.mock import patch, mock_open
 from argparse import Namespace
-import logging
-from .doc_gen import DocGen, MetadataError
+from pathlib import Path
+
+from .doc_gen import DocGen, MetadataError, Example
 from .doc_gen_cli import main
+from .metadata import DocFilenames, Sdk, Language, SDKPageVersion, Version
+from .sdks import SdkVersion
+
+
+@pytest.fixture
+def mock_example():
+    return Example(
+        file=Path("test_cpp.yaml"),
+        id="medical-imaging_GoodScenario",
+        title="Scenario title",
+        title_abbrev="Scenario title abbrev",
+        synopsis="scenario synopsis.",
+        category="Scenarios",
+        doc_filenames=DocFilenames(
+            service_pages={
+                "medical-imaging": "link",
+            },
+            sdk_pages={
+                "cpp": {
+                    1: SDKPageVersion(actions_scenarios={"medical-imaging": f"link"})
+                }
+            },
+        ),
+        services={
+            "medical-imaging": {"GoodOne"},
+        },
+        languages={
+            "C++": Language(
+                name="C++", property="cpp", versions=[Version(sdk_version=1)]
+            )
+        },
+    )
 
 
 @pytest.fixture
-def mock_doc_gen():
+def mock_doc_gen(mock_example):
     doc_gen = DocGen.empty()
     doc_gen.errors._errors = [
         MetadataError(file="a.yaml", id="Error 1"),
         MetadataError(file="b.yaml", id="Error 2"),
     ]
+    doc_gen.sdks = {
+        "JavaScript": Sdk(
+            name="JavaScript",
+            versions=[SdkVersion(version=3, long="&JS;", short="&JSlong")],
+            guide="",
+            property="javascript",
+        )
+    }
+    doc_gen.examples = {"ex": mock_example}
     return doc_gen
 
 
@@ -32,7 +74,10 @@ def patched_environment(mock_doc_gen):
 def test_doc_gen_strict_option(strict, should_raise, patched_environment):
     mock_parse_args, mock_json_dump = patched_environment
     mock_args = Namespace(
-        from_root=["/mock/path"], write_json="mock_output.json", strict=strict
+        from_root=["/mock/path"],
+        write_json="mock_output.json",
+        strict=strict,
+        skip_entity_expansion=False,
     )
     mock_parse_args.return_value = mock_args
 
@@ -42,3 +87,38 @@ def test_doc_gen_strict_option(strict, should_raise, patched_environment):
         assert exc_info.value.code == 1
     else:
         main()
+
+
+def test_skip_entity_expansion(patched_environment):
+    mock_parse_args, mock_json_dump = patched_environment
+    mock_args = Namespace(
+        from_root=["/mock/path"],
+        write_json="mock_output.json",
+        strict=False,
+        skip_entity_expansion=True,
+    )
+    mock_parse_args.return_value = mock_args
+
+    with patch(
+        "aws_doc_sdk_examples_tools.doc_gen.DocGen.expand_entities"
+    ) as mock_expand_entities:
+        main()
+        assert not mock_expand_entities.called
+
+
+def test_default_entity_expansion(patched_environment):
+    mock_parse_args, mock_json_dump = patched_environment
+    mock_args = Namespace(
+        from_root=["/mock/path"],
+        write_json="mock_output.json",
+        strict=False,
+        skip_entity_expansion=False,
+    )
+    mock_parse_args.return_value = mock_args
+
+    with patch(
+        "aws_doc_sdk_examples_tools.doc_gen.DocGen.expand_entities"
+    ) as mock_expand_entities:
+        mock_expand_entities.return_value = None, []
+        main()
+        assert mock_expand_entities.called

aws_doc_sdk_examples_tools/doc_gen_test.py

@@ -12,7 +12,7 @@
 from .metadata_errors import MetadataErrors, MetadataError
 from .doc_gen import DocGen, DocGenEncoder
 from .sdks import Sdk, SdkVersion
-from .services import Service
+from .services import Service, ServiceExpanded
 from .snippets import Snippet
 
 
@@ -80,6 +80,7 @@ def sample_doc_gen() -> DocGen:
         root=Path("/test/root"),
         errors=metadata_errors,
         prefix="test_prefix",
+        entities={"&S3long;": "Amazon Simple Storage Service", "&S3;": "Amazon S3"},
         sdks={
             "python": Sdk(
                 name="python",
@@ -92,8 +93,11 @@ def sample_doc_gen() -> DocGen:
         },
         services={
             "s3": Service(
-                long="Amazon S3",
-                short="S3",
+                long="&S3long;",
+                short="&S3;",
+                expanded=ServiceExpanded(
+                    long="Amazon Simple Storage Service", short="Amazon S3"
+                ),
                 sort="Amazon S3",
                 version="2006-03-01",
             )
@@ -113,6 +117,12 @@ def sample_doc_gen() -> DocGen:
     )
 
 
+def test_expand_entities(sample_doc_gen: DocGen):
+    expanded, errors = sample_doc_gen.expand_entities("Hello &S3;")
+    assert expanded == "Hello Amazon S3"
+    assert not errors
+
+
 def test_doc_gen_encoder(sample_doc_gen: DocGen):
     encoded = json.dumps(sample_doc_gen, cls=DocGenEncoder)
     decoded = json.loads(encoded)
@@ -132,8 +142,8 @@ def test_doc_gen_encoder(sample_doc_gen: DocGen):
     # Verify service information
     assert "services" in decoded
     assert "s3" in decoded["services"]
-    assert decoded["services"]["s3"]["long"] == "Amazon S3"
-    assert decoded["services"]["s3"]["short"] == "S3"
+    assert decoded["services"]["s3"]["long"] == "&S3long;"
+    assert decoded["services"]["s3"]["short"] == "&S3;"
 
     # Verify snippet information
     assert "snippets" in decoded