DOCSP-4337: parse code-block into a 'code' node

Author: i80and

snooty.idl

@@ -117,7 +117,8 @@ interface HTML <: Literal {
 interface Code <: Literal {
   type: "code";
   lang: string?;
-  meta: string?;
+  copyable: boolean;
+  emphasize_lines: [(int, int)];
 }
 
 interface Definition <: Node {

snooty/gizaparser/steps.py

@@ -41,7 +41,7 @@ def render(self, parse_rst: EmbeddedRstParser) -> List[SerializableType]:
             nodes_to_append_children.append({
                 'type': 'code',
                 'lang': self.language,
-                'copyable': False if self.copyable is None else self.copyable,
+                'copyable': True if self.copyable is None else self.copyable,
                 'position': {'start': {'line': self.line}},
                 'value': self.code
             })

snooty/gizaparser/test_steps.py

@@ -67,7 +67,8 @@ def create_page() -> Tuple[Page, EmbeddedRstParser]:
         '</heading><paragraph><text>hi</text></paragraph>',
         '<paragraph><text>You can install either the latest stable version of MongoDB ',
         'or a\nspecific version of MongoDB.</text></paragraph>',
-        '<directive name="code-block">',
-        '<text>sh</text><literal></literal></directive><paragraph><text>bye</text></paragraph>',
+        '<code lang="sh" copyable="True">',
+        'echo "mongodb-org hold" | sudo dpkg --set-selections',
+        '</code><paragraph><text>bye</text></paragraph>',
         '</section></directive></directive>'
     ))

snooty/parser.py

@@ -67,6 +67,15 @@ def dispatch_visit(self, node: docutils.nodes.Node) -> None:
             value = node.children[1].astext()
             self.state[-1].setdefault('options', {})[key] = value
             raise docutils.nodes.SkipNode()
+        elif node_name == 'code':
+            doc['type'] = 'code'
+            doc['lang'] = node['lang']
+            doc['copyable'] = node['copyable']
+            if node['emphasize_lines']:
+                doc['emphasize_lines'] = node['emphasize_lines']
+            doc['value'] = node.astext()
+            self.state[-1]['children'].append(doc)
+            raise docutils.nodes.SkipNode()
 
         self.state.append(doc)
 
@@ -75,9 +84,6 @@ def dispatch_visit(self, node: docutils.nodes.Node) -> None:
             doc['value'] = str(node)
             return
 
-        # Most, but not all, nodes have children nodes
-        make_children = True
-
         if node_name == 'directive':
             self.handle_directive(node, doc)
         elif node_name == 'role':
@@ -103,13 +109,8 @@ def dispatch_visit(self, node: docutils.nodes.Node) -> None:
             doc['type'] = 'listItem'
         elif node_name == 'title':
             doc['type'] = 'heading'
-        elif node_name == 'FixedTextElement':
-            doc['type'] = 'literal'
-            doc['value'] = node.astext()
-            make_children = False
 
-        if make_children:
-            doc['children'] = []
+        doc['children'] = []
 
     def dispatch_departure(self, node: docutils.nodes.Node) -> None:
         node_name = node.__class__.__name__

snooty/rstparser.py

@@ -20,7 +20,6 @@
 PAT_EXPLICIT_TILE = re.compile(r'^(?P<label>.+?)\s*(?<!\x00)<(?P<target>.*?)>$', re.DOTALL)
 PAT_WHITESPACE = re.compile(r'^\x20*')
 PAT_BLOCK_HAS_ARGUMENT = re.compile(r'^\x20*\.\.\x20[^\s]+::\s*\S+')
-SPECIAL_DIRECTIVES = {'code-block', 'include', 'only'}
 
 
 @checked
@@ -38,6 +37,10 @@ class LegacyTabsDefinition(nodes.Node):
     tabs: List[LegacyTabDefinition]
 
 
+def option_bool(argument: str) -> Any:
+    return docutils.parsers.rst.directives.choice(argument, ('true', 'false', None))
+
+
 class directive_argument(docutils.nodes.General, docutils.nodes.TextElement):
     pass
 
@@ -48,6 +51,10 @@ def __init__(self, name: str) -> None:
         self['name'] = name
 
 
+class code(docutils.nodes.General, docutils.nodes.FixedTextElement):
+    pass
+
+
 class role(docutils.nodes.General, docutils.nodes.Inline, docutils.nodes.Element):
     def __init__(self, name: str, rawtext: str, text: str, lineno: int) -> None:
         super(role, self).__init__()
@@ -131,6 +138,22 @@ def parse_options(block_text: str) -> Dict[str, str]:
     return kv
 
 
+def parse_linenos(term: str, max_val: int) -> List[Tuple[int, int]]:
+    """Parse a comma-delimited list of line numbers and ranges."""
+    results: List[Tuple[int, int]] = []
+    for term in (term for term in term.split(',') if term.strip()):
+        parts = term.split('-', 1)
+        lower = int(parts[0])
+        higher = int(parts[1]) if len(parts) == 2 else lower
+
+        if lower < 0 or lower > max_val or higher < 0 or higher > max_val or lower > higher:
+            raise ValueError(f'Invalid line number specification: {term}')
+
+        results.append((lower, higher))
+
+    return results
+
+
 class Directive(docutils.parsers.rst.Directive):
     optional_arguments = 1
     final_argument_whitespace = True
@@ -163,7 +186,7 @@ def run(self) -> List[docutils.nodes.Node]:
                 node.append(argument)
 
         # Parse the content
-        if self.name in SPECIAL_DIRECTIVES:
+        if self.name in {'include', 'only'}:
             raw = docutils.nodes.FixedTextElement()
             raw.document = self.state.document
             raw.source, raw.line = source, line
@@ -195,9 +218,8 @@ def prepare_viewlist(text: str, ignore: int = 1) -> List[str]:
 class TabsDirective(Directive):
     option_spec = {
         'tabset': str,
-        'hidden': bool
+        'hidden': option_bool
     }
-    has_content = True
 
     def run(self) -> List[docutils.nodes.Node]:
         # Transform the old YAML-based syntax into the new pure-rst syntax.
@@ -262,6 +284,38 @@ def make_tab_node(self, source: str, child: LegacyTabDefinition) -> docutils.nod
         return node
 
 
+class CodeDirective(Directive):
+    required_arguments = 1
+    optional_arguments = 0
+    option_spec = {
+        'copyable': option_bool,
+        'emphasize-lines': str
+    }
+
+    def run(self) -> List[docutils.nodes.Node]:
+        source, line = self.state_machine.get_source_and_line(self.lineno)
+        copyable = 'copyable' not in self.options or self.options['copyable'] == 'true'
+
+        try:
+            n_lines = len(self.content)
+            emphasize_lines = parse_linenos(self.options.get('emphasize-lines', ''), n_lines)
+        except ValueError as err:
+            error_node = self.state.document.reporter.error(
+                str(err),
+                line=self.lineno)
+            return [error_node]
+
+        value = '\n'.join(self.content)
+        node = code(value, value)
+        node['name'] = 'code'
+        node['lang'] = self.arguments[0]
+        node['copyable'] = copyable
+        node['emphasize_lines'] = emphasize_lines
+        node.document = self.state.document
+        node.source, node.line = source, line
+        return [node]
+
+
 def handle_role(typ: str, rawtext: str, text: str,
                 lineno: int, inliner: object,
                 options: Dict[str, object] = {},
@@ -275,6 +329,9 @@ def lookup_directive(directive_name: str, language_module: object,
     if directive_name.startswith('tabs'):
         return TabsDirective, []
 
+    if directive_name in {'code-block', 'sourcecode'}:
+        return CodeDirective, []
+
     return Directive, []
 
 

snooty/test_parser.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from . import rstparser
 from .util import ast_to_testing_string
-from .types import ProjectConfig
+from .types import Diagnostic, ProjectConfig
 from .parser import parse_rst, JSONVisitor
 
 
@@ -36,3 +36,48 @@ def test_tabs() -> None:
     assert len(diagnostics) == 1 and \
         diagnostics[0].message.startswith('Unexpected field') and \
         diagnostics[0].start[0] == 36
+
+
+def test_codeblock() -> None:
+    root = Path('test_data')
+    tabs_path = Path(root).joinpath(Path('test.rst'))
+    project_config = ProjectConfig(root, '')
+    parser = rstparser.Parser(project_config, JSONVisitor)
+
+    # Test a simple code-block
+    page, diagnostics = parse_rst(parser, tabs_path, '''
+.. code-block:: sh
+
+   foo bar
+     indented
+   end''')
+    assert diagnostics == []
+    assert ast_to_testing_string(page.ast) == ''.join((
+        '<root>',
+        '<code lang="sh" copyable="True">foo bar\n  indented\nend</code>'
+        '</root>'
+    ))
+
+    # Test parsing of emphasize-lines
+    page, diagnostics = parse_rst(parser, tabs_path, '''
+.. code-block:: sh
+   :copyable: false
+   :emphasize-lines: 1, 2-3
+
+   foo
+   bar
+   baz''')
+    assert diagnostics == []
+    assert ast_to_testing_string(page.ast) == ''.join((
+        '<root>',
+        '<code lang="sh" emphasize_lines="[(1, 1), (2, 3)]">foo\nbar\nbaz</code>'
+        '</root>'
+    ))
+
+    # Test handling of out-of-range lines
+    page, diagnostics = parse_rst(parser, tabs_path, '''
+.. code-block:: sh
+   :emphasize-lines: 10
+
+   foo''')
+    assert diagnostics[0].severity == Diagnostic.Level.warning

stubs/docutils/nodes.pyi

@@ -63,3 +63,7 @@ class SkipNode(TreePruningException): ...
 
 
 class system_message(Element): ...
+
+
+class Text(Node):
+    def __init__(self, data: object, rawsource: str=''): ...

stubs/docutils/parsers/rst/directives.pyi

@@ -1,5 +1,7 @@
-from typing import Any, List, Type, Tuple
+from typing import Any, Collection, List, Type, Tuple
 import docutils.nodes
 
 
 def directive(directive_name: str, language_module: object, document: docutils.nodes.document) -> Tuple[Type[Any], List[object]]: ...
+
+def choice(argument: str, values: Collection[object]) -> Any: ...

stubs/docutils/statemachine.pyi

@@ -9,7 +9,8 @@ class ViewList:
     def __iter__(self) -> Iterator[str]: ...
 
 
-class StringList(ViewList): ...
+class StringList(ViewList):
+    def __len__(self) -> int: ...
 
 
 class State: