DOCSP-5545: Implement release specifications

* Add a ReleaseSpecificationsGizaCategory class
* Merge source_constants into the giza substitutions system
* Add tests
* Raise a diagnostic upon use of undefined giza substitutions

Author: i80and

snooty/gizaparser/__init__.py

@@ -1 +1 @@
-from . import steps, extracts  # NoQA
+from . import steps, extracts, release  # NoQA

snooty/gizaparser/extracts.py

@@ -57,9 +57,12 @@ def to_pages(self,
                  extracts: Sequence[Extract]) -> List[Page]:
         pages: List[Page] = []
         for extract in extracts:
+            assert extract.ref is not None
+            if extract.ref.startswith('_'):
+                continue
+
             page, rst_parser = page_factory()
             page.category = 'extracts'
-            assert extract.ref is not None
             page.output_filename = extract.ref
             page.ast = extract_to_page(page, extract, rst_parser)
             pages.append(page)

snooty/gizaparser/nodes.py

@@ -5,7 +5,8 @@
 from dataclasses import dataclass, field
 from pathlib import Path, PurePath
 from typing import cast, Callable, Dict, Set, Generic, Optional, \
-                   TypeVar, Tuple, Iterator, Sequence, List, Union
+                   TypeVar, Tuple, Iterator, Sequence, List, Union, \
+                   Match
 from ..flutter import checked
 from ..types import Diagnostic, Page, EmbeddedRstParser, SerializableType, ProjectConfig
 
@@ -14,13 +15,26 @@
 logger = logging.getLogger(__name__)
 
 
-def substitute_text(text: str, replacements: Dict[str, str]) -> str:
-    return PAT_SUBSTITUTION.sub(lambda match: replacements.get(match.group(1), ''), text)
+def substitute_text(text: str,
+                    replacements: Dict[str, str],
+                    diagnostics: List[Diagnostic]) -> str:
+    def substitute(match: Match[str]) -> str:
+        try:
+            return replacements[match.group(1)]
+        except KeyError:
+            diagnostics.append(Diagnostic.warning(
+                f'Unknown substitution: "{match.group(1)}". ' +
+                'You may intend this substitution to be empty', 1))
+            return ''
 
+    return PAT_SUBSTITUTION.sub(substitute, text)
 
-def substitute(obj: _T, replacements: Dict[str, str]) -> _T:
+
+def substitute(obj: _T,
+               replacements: Dict[str, str],
+               diagnostics: List[Diagnostic]) -> _T:
     if isinstance(obj, str):
-        return substitute_text(obj, replacements)
+        return substitute_text(obj, replacements, diagnostics)
 
     if not dataclasses.is_dataclass(obj):
         return obj
@@ -29,11 +43,11 @@ def substitute(obj: _T, replacements: Dict[str, str]) -> _T:
     for obj_field in dataclasses.fields(obj):
         value = getattr(obj, obj_field.name)
         if isinstance(value, str):
-            new_str = substitute_text(value, replacements)
+            new_str = substitute_text(value, replacements, diagnostics)
             if new_str is not value:
                 changes[obj_field.name] = new_str
         elif dataclasses.is_dataclass(value):
-            new_value = substitute(value, replacements)
+            new_value = substitute(value, replacements, diagnostics)
             if new_value is not value:
                 changes[obj_field.name] = new_value
 
@@ -65,7 +79,10 @@ class Inheritable(Node):
 _I = TypeVar('_I', bound=Inheritable)
 
 
-def inherit(obj: _I, parent: Optional[_I]) -> _I:
+def inherit(project_config: ProjectConfig,
+            obj: _I,
+            parent: Optional[_I],
+            diagnostics: List[Diagnostic]) -> _I:
     logger.debug('Inheriting %s', obj.ref)
     changes: Dict[str, object] = {}
 
@@ -77,6 +94,11 @@ def inherit(obj: _I, parent: Optional[_I]) -> _I:
             if src not in replacement:
                 replacement[src] = dest
 
+    # Merge in project-wide constants into the giza substitutions system
+    new_replacement = {k: str(v) for k, v in project_config.constants.items()}
+    new_replacement.update(replacement)
+    replacement = new_replacement
+
     # Inherit root-level keys
     for field_name in (field.name for field in dataclasses.fields(obj)
                        if field.name not in {'replacement', 'ref', 'source', 'inherit'}):
@@ -87,8 +109,8 @@ def inherit(obj: _I, parent: Optional[_I]) -> _I:
                 changes[field_name] = new_value
                 value = new_value
 
-        if replacement and value is not None:
-            changes[field_name] = substitute(value, replacement)
+        if value is not None:
+            changes[field_name] = substitute(value, replacement, diagnostics)
 
     return dataclasses.replace(obj, **changes) if changes else obj
 
@@ -187,7 +209,7 @@ def reify(self, obj: _I, diagnostics: List[Diagnostic]) -> _I:
         if obj.ref is None:
             obj.ref = ''
 
-        obj = inherit(obj, parent)
+        obj = inherit(self.project_config, obj, parent, diagnostics)
         return obj
 
     def reify_file_id(self,

snooty/gizaparser/release.py

@@ -0,0 +1,74 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable, Optional, List, Tuple, Sequence
+from ..flutter import checked
+from ..types import Diagnostic, EmbeddedRstParser, SerializableType, Page
+from .parse import parse
+from .nodes import Inheritable, GizaCategory
+
+
+@checked
+@dataclass
+class ReleaseSpecification(Inheritable):
+    copyable: Optional[bool]
+    language: Optional[str]
+    code: Optional[str]
+
+    def render(self, page: Page, parse_rst: EmbeddedRstParser) -> SerializableType:
+        children: List[SerializableType] = []
+        if self.code:
+            children.append({
+                'type': 'code',
+                'lang': self.language,
+                'copyable': True if self.copyable is None else self.copyable,
+                'position': {'start': {'line': self.line}},
+                'value': self.code
+            })
+        return children
+
+
+def release_specification_to_page(page: Page,
+                                  node: ReleaseSpecification,
+                                  rst_parser: EmbeddedRstParser) -> SerializableType:
+    rendered = node.render(page, rst_parser)
+    return {
+        'type': 'directive',
+        'name': 'release_specification',
+        'position': {'start': {'line': node.line}},
+        'children': rendered
+    }
+
+
+class GizaReleaseSpecificationCategory(GizaCategory[ReleaseSpecification]):
+    def parse(self,
+              path: Path,
+              text: Optional[str] = None) -> Tuple[
+                  Sequence[ReleaseSpecification], str, List[Diagnostic]]:
+        nodes, text, diagnostics = parse(ReleaseSpecification, path, self.project_config, text)
+
+        def report_missing_ref(node: ReleaseSpecification) -> bool:
+            diagnostics.append(
+                Diagnostic.error(
+                    'Missing ref; all release specifications must define a ref', node.line))
+            return False
+
+        # All nodes must have an explicitly-defined ref ID
+        release_specifications = [node for node in nodes if node.ref or report_missing_ref(node)]
+        return release_specifications, text, diagnostics
+
+    def to_pages(self,
+                 page_factory: Callable[[], Tuple[Page, EmbeddedRstParser]],
+                 nodes: Sequence[ReleaseSpecification]) -> List[Page]:
+        pages: List[Page] = []
+        for node in nodes:
+            assert node.ref is not None
+            if node.ref.startswith('_'):
+                continue
+
+            page, rst_parser = page_factory()
+            page.category = 'release'
+            page.output_filename = node.ref
+            page.ast = release_specification_to_page(page, node, rst_parser)
+            pages.append(page)
+
+        return pages

snooty/gizaparser/test_extracts.py

@@ -18,7 +18,7 @@ def add_main_file() -> List[Diagnostic]:
         assert len(parse_diagnostics) == 1
         assert parse_diagnostics[0].severity == Diagnostic.Level.error
         assert parse_diagnostics[0].start == (21, 0)
-        assert len(extracts) == 4
+        assert len(extracts) == 5
         return parse_diagnostics
 
     def add_parent_file() -> List[Diagnostic]:
@@ -41,22 +41,21 @@ def create_page() -> Tuple[Page, EmbeddedRstParser]:
 
     pages = category.to_pages(create_page, giza_node.data)
     assert [str(page.get_id()) for page in pages] == [
-        'test_data/extracts/_base',
         'test_data/extracts/installation-directory-rhel',
         'test_data/extracts/broken-inherit',
-        'test_data/extracts/another-file']
-    assert ast_to_testing_string(pages[0].ast) == ''.join((
-        '<directive name="extract"><paragraph><text>By default, MongoDB stores its data files in ',
-        '{{mongodDatadir}} and its\nlog files in </text><literal><text>/var/log/mongodb</text>',
-        '</literal><text>.</text></paragraph></directive>'
-    ))
+        'test_data/extracts/another-file',
+        'test_data/extracts/missing-substitution']
 
-    assert ast_to_testing_string(pages[1].ast) == ''.join((
+    assert ast_to_testing_string(pages[0].ast) == ''.join((
         '<directive name="extract"><paragraph><text>By default, MongoDB stores its data files in ',
         '</text><literal><text>/var/lib/mongo</text></literal><text> and its\nlog files in </text>',
         '<literal><text>/var/log/mongodb</text></literal><text>.</text></paragraph></directive>'
     ))
 
+    assert ast_to_testing_string(pages[3].ast) == ''.join((
+        '<directive name="extract"><paragraph><text>Substitute</text></paragraph></directive>'
+    ))
+
     # XXX: We need to track source file information for each property.
     # Line number 1 here should correspond to parent_path, not path.
     assert set(d.start[0] for d in all_diagnostics[path]) == set((21, 13, 1))

snooty/gizaparser/test_nodes.py

@@ -1,5 +1,8 @@
 from dataclasses import dataclass
+from pathlib import Path
+from typing import List
 from . import nodes
+from ..types import Diagnostic, ProjectConfig
 
 
 @dataclass
@@ -30,44 +33,51 @@ def test_dependency_graph() -> None:
 
 
 def test_substitution() -> None:
+    diagnostics: List[Diagnostic] = []
     replacements = {
         'verb': 'test',
         'noun': 'substitution'
     }
     test_string = r'{{verb}}ing {{noun}}. {{verb}}.'
     substituted_string = 'testing substitution. test.'
-    assert nodes.substitute_text(test_string, replacements) == substituted_string
+    assert nodes.substitute_text(test_string, replacements, diagnostics) == substituted_string
 
     obj = object()
-    assert nodes.substitute(obj, replacements) is obj
+    assert nodes.substitute(obj, replacements, diagnostics) is obj
 
     # Test complex substitution
     node = SubstitutionTest(
         foo=test_string,
         foo2=test_string,
         child=Child(test_string))
-    substituted_node = nodes.substitute(node, replacements)
+    substituted_node = nodes.substitute(node, replacements, diagnostics)
     assert substituted_node == SubstitutionTest(
         foo=substituted_string,
         foo2=substituted_string,
         child=Child(substituted_string))
 
     # Make sure that no substitution == ''
+    assert diagnostics == []
     del replacements['noun']
-    assert nodes.substitute_text(test_string, replacements) == 'testing . test.'
+    assert nodes.substitute_text(test_string, replacements, diagnostics) == 'testing . test.'
+    assert len(diagnostics) == 1
 
     # Ensure the identity of the zero-substitutions case remains the same
+    diagnostics = []
     test_string = 'foo'
-    assert nodes.substitute_text(test_string, {}) is test_string
+    assert nodes.substitute_text(test_string, {}, diagnostics) is test_string
+    assert not diagnostics
 
 
 def test_inheritance() -> None:
+    project_config, diagnostics = ProjectConfig.open(Path('test_data'))
     parent = nodes.Inheritable('parent', {'foo': 'bar', 'old': ''}, source=None, inherit=None)
     child = nodes.Inheritable(
         'child',
         {'bar': 'baz', 'old': 'new'},
         source=nodes.Inherit('self.yaml', 'parent'),
         inherit=None)
-    child = nodes.inherit(child, parent)
+    child = nodes.inherit(project_config, child, parent, diagnostics)
 
     assert child.replacement == {'foo': 'bar', 'bar': 'baz', 'old': 'new'}
+    assert not diagnostics

snooty/gizaparser/test_release.py

@@ -0,0 +1,65 @@
+from pathlib import Path, PurePath
+from typing import Dict, Tuple, List
+from .release import GizaReleaseSpecificationCategory
+from ..types import Diagnostic, Page, EmbeddedRstParser, ProjectConfig
+from ..parser import make_embedded_rst_parser
+from ..util import ast_to_testing_string
+
+
+def test_release_specification() -> None:
+    project_config = ProjectConfig(Path('test_data'), '')
+    project_config.constants['version'] = '3.4'
+    category = GizaReleaseSpecificationCategory(project_config)
+    path = Path('test_data/release-specifications.yaml')
+    parent_path = Path('test_data/release-base.yaml')
+
+    def add_main_file() -> List[Diagnostic]:
+        extracts, text, parse_diagnostics = category.parse(path)
+        category.add(path, text, extracts)
+        assert len(parse_diagnostics) == 0
+        assert len(extracts) == 2
+        return parse_diagnostics
+
+    def add_parent_file() -> List[Diagnostic]:
+        extracts, text, parse_diagnostics = category.parse(parent_path)
+        category.add(parent_path, text, extracts)
+        assert len(parse_diagnostics) == 0
+        assert len(extracts) == 2
+        return parse_diagnostics
+
+    all_diagnostics: Dict[PurePath, List[Diagnostic]] = {}
+    all_diagnostics[path] = add_main_file()
+    all_diagnostics[parent_path] = add_parent_file()
+
+    assert len(category) == 2
+    file_id, giza_node = next(category.reify_all_files(all_diagnostics))
+
+    def create_page() -> Tuple[Page, EmbeddedRstParser]:
+        page = Page(path, '', {})
+        return page, make_embedded_rst_parser(project_config, page, all_diagnostics[path])
+
+    pages = category.to_pages(create_page, giza_node.data)
+    assert [str(page.get_id()) for page in pages] == [
+        'test_data/release/untar-release-osx-x86_64',
+        'test_data/release/install-ent-windows-default']
+
+    assert all((not diagnostics for diagnostics in all_diagnostics.values()))
+
+    print(ast_to_testing_string(pages[0].ast))
+    assert ast_to_testing_string(pages[0].ast) == ''.join((
+        '<directive name="release_specification">',
+        '<code lang="sh" copyable="True">',
+        'tar -zxvf mongodb-macos-x86_64-3.4.tgz\n'
+        '</code>',
+        '</directive>'
+    ))
+
+    print(ast_to_testing_string(pages[1].ast))
+    assert ast_to_testing_string(pages[1].ast) == ''.join((
+        '<directive name="release_specification">',
+        '<code lang="bat" copyable="True">',
+        'msiexec.exe /l*v mdbinstall.log  /qb /i ',
+        'mongodb-win32-x86_64-enterprise-windows-64-3.4-signed.msi\n',
+        '</code>',
+        '</directive>'
+    ))

snooty/parser.py

@@ -344,7 +344,8 @@ def __init__(self,
 
         self.yaml_mapping: Dict[str, GizaCategory[Any]] = {
             'steps': gizaparser.steps.GizaStepsCategory(self.config),
-            'extracts': gizaparser.extracts.GizaExtractsCategory(self.config)
+            'extracts': gizaparser.extracts.GizaExtractsCategory(self.config),
+            'release': gizaparser.release.GizaReleaseSpecificationCategory(self.config),
         }
 
         username = pwd.getpwuid(os.getuid()).pw_name

test_data/extracts-test.yaml

@@ -25,4 +25,8 @@ ref: another-file
 inherit:
   file: extracts-test-parent.yaml
   ref: a-parent-ref
+---
+ref: missing-substitution
+content: |
+  Substitute {{mongodDatadir}}
 ...

test_data/release-base.yaml

@@ -0,0 +1,15 @@
+ref: _untar-release
+copyable: true
+language: 'sh'
+code: |
+  tar -zxvf mongodb-{{platform}}-{{builder}}-{{version}}.tgz
+---
+ref: _install-windows-ent
+copyable: true
+language: bat
+code: |
+   msiexec.exe /l*v mdbinstall.log  /qb /i mongodb-win32-x86_64-enterprise-windows-64-{{version}}-signed.msi
+
+# Adding SHOULD_INSTALL_COMPASS="0" since the install page states that this installs just what's specified in ADDLOCAL
+
+...