Remove productionlist hard-coding in translators

Author: AA-Turner

doc/usage/restructuredtext/directives.rst

@@ -1647,7 +1647,7 @@ derived forms), but provides enough to allow context-free grammars to be
 displayed in a way that causes uses of a symbol to be rendered as hyperlinks to
 the definition of the symbol.  There is this directive:
 
-.. rst:directive:: .. productionlist:: [productionGroup]
+.. rst:directive:: .. productionlist:: [production_group]
 
    This directive is used to enclose a group of productions.  Each production
    is given on a single line and consists of a name, separated by a colon from
@@ -1662,26 +1662,26 @@ the definition of the symbol.  There is this directive:
    production list, you can reference to token productions using
    :rst:role:`token`.
 
-   The *productionGroup* argument to :rst:dir:`productionlist` serves to
+   The *production_group* argument to :rst:dir:`productionlist` serves to
    distinguish different sets of production lists that belong to different
-   grammars.  Multiple production lists with the same *productionGroup* thus
+   grammars.  Multiple production lists with the same *production_group* thus
    define rules in the same scope.
 
    Inside of the production list, tokens implicitly refer to productions
    from the current group. You can refer to the production of another
    grammar by prefixing the token with its group name and a colon, e.g,
-   "``otherGroup:sum``". If the group of the token should not be shown in
+   "``other-group:sum``". If the group of the token should not be shown in
    the production, it can be prefixed by a tilde, e.g.,
-   "``~otherGroup:sum``". To refer to a production from an unnamed
+   "``~other-group:sum``". To refer to a production from an unnamed
    grammar, the token should be prefixed by a colon, e.g., "``:sum``".
 
    Outside of the production list,
-   if you have given a *productionGroup* argument you must prefix the
+   if you have given a *production_group* argument you must prefix the
    token name in the cross-reference with the group name and a colon,
-   e.g., "``myGroup:sum``" instead of just "``sum``".
+   e.g., "``my_group:sum``" instead of just "``sum``".
    If the group should not be shown in the title of the link either
-   an explicit title can be given (e.g., "``myTitle <myGroup:sum>``"),
-   or the target can be prefixed with a tilde (e.g., "``~myGroup:sum``").
+   an explicit title can be given (e.g., "``myTitle <my_group:sum>``"),
+   or the target can be prefixed with a tilde (e.g., "``~my_group:sum``").
 
    Note that no further reStructuredText parsing is done in the production,
    so that you don't have to escape ``*`` or ``|`` characters.

sphinx/domains/std/__init__.py

@@ -22,7 +22,7 @@
 from sphinx.util.parsing import nested_parse_to_nodes
 
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterable, Iterator, Set
+    from collections.abc import Callable, Iterable, Iterator, Sequence, Set
     from typing import Any, ClassVar, Final
 
     from docutils.nodes import Element, Node, system_message
@@ -597,41 +597,91 @@ class ProductionList(SphinxDirective):
     option_spec: ClassVar[OptionSpec] = {}
 
     def run(self) -> list[Node]:
-        domain = self.env.domains.standard_domain
-        node: Element = addnodes.productionlist()
+        node = addnodes.productionlist()
         self.set_source_info(node)
         # The backslash handling is from ObjectDescription.get_signatures
         nl_escape_re = re.compile(r'\\\n')
         lines = nl_escape_re.sub('', self.arguments[0]).split('\n')
+        production_group = self.production_group(lines, self.options)
+        production_lines = list(self.production_definitions(lines))
+        max_len = max(len(name) for _, name, _ in production_lines)
+        node_location = self.get_location()
+        node += [
+            self.make_production(
+                rawsource=rule,
+                name=name,
+                tokens=tokens,
+                production_group=production_group,
+                max_len=max_len,
+                location=node_location,
+            )
+            for rule, name, tokens in production_lines
+        ]
+        return [node]
 
-        production_group = ''
-        first_rule_seen = False
-        for rule in lines:
-            if not first_rule_seen and ':' not in rule:
-                production_group = rule.strip()
-                continue
-            first_rule_seen = True
-            try:
-                name, tokens = rule.split(':', 1)
-            except ValueError:
+    @staticmethod
+    def production_group(lines: Sequence[str], options: dict[str, Any]) -> str:  # NoQA: ARG004
+        # get production_group
+        if not lines or ':' in lines[0]:
+            return ''
+        production_group = lines[0].strip()
+        lines[:] = lines[1:]
+        return production_group
+
+    @staticmethod
+    def production_definitions(lines: Iterable[str]) -> Iterator[tuple[str, str, str]]:
+        """Yield triples of rawsource, name, definition."""
+        for line in lines:
+            if ':' not in line:
                 break
-            subnode = addnodes.production(rule)
-            name = name.strip()
-            subnode['tokenname'] = name
-            if subnode['tokenname']:
-                prefix = 'grammar-token-%s' % production_group
-                node_id = make_id(self.env, self.state.document, prefix, name)
-                subnode['ids'].append(node_id)
-                self.state.document.note_implicit_target(subnode, subnode)
-
-                if len(production_group) != 0:
-                    obj_name = f'{production_group}:{name}'
-                else:
-                    obj_name = name
-                domain.note_object('token', obj_name, node_id, location=node)
-            subnode.extend(token_xrefs(tokens, production_group=production_group))
-            node.append(subnode)
-        return [node]
+            name, _, tokens = line.partition(':')
+            yield line, name.strip(), tokens.strip()
+
+    def make_production(
+        self,
+        rawsource: str,
+        name: str,
+        tokens: str,
+        production_group: str,
+        max_len: int,
+        location: str,
+    ) -> addnodes.production:
+        production_node = addnodes.production(rawsource, tokenname=name)
+        if name:
+            production_node += self.make_target(name, production_group, location)
+        else:
+            production_node += self.continuation_padding(max_len)
+        production_node.append(self.production_separator(name, max_len))
+        production_node += token_xrefs(tokens, production_group=production_group)
+        production_node.append(nodes.Text('\n'))
+        return production_node
+
+    def make_target(
+        self,
+        name: str,
+        production_group: str,
+        location: str,
+    ) -> addnodes.literal_strong:
+        """Make a link target for the given production."""
+        name_node = addnodes.literal_strong(name, name)
+        prefix = f'grammar-token-{production_group}'
+        node_id = make_id(self.env, self.state.document, prefix, name)
+        name_node['ids'].append(node_id)
+        self.state.document.note_implicit_target(name_node, name_node)
+        obj_name = f'{production_group}:{name}' if production_group else name
+        std = self.env.domains.standard_domain
+        std.note_object('token', obj_name, node_id, location=location)
+        return name_node
+
+    @staticmethod
+    def continuation_padding(max_len: int) -> nodes.Text:
+        return nodes.Text(' ' * max_len)
+
+    @staticmethod
+    def production_separator(name: str, max_len: int) -> nodes.Text:
+        if name:
+            return nodes.Text(' ::= '.rjust(max_len - len(name) + 5))
+        return nodes.Text('     ')
 
 
 class TokenXRefRole(XRefRole):

sphinx/writers/html5.py

@@ -5,7 +5,7 @@
 import posixpath
 import re
 import urllib.parse
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING
 
 from docutils import nodes
 from docutils.writers.html5_polyglot import HTMLTranslator as BaseTranslator
@@ -17,8 +17,6 @@
 from sphinx.util.images import get_image_size
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable
-
     from docutils.nodes import Element, Node, Text
 
     from sphinx.builders import Builder
@@ -695,23 +693,9 @@ def depart_literal(self, node: Element) -> None:
 
     def visit_productionlist(self, node: Element) -> None:
         self.body.append(self.starttag(node, 'pre'))
-        productionlist = cast('Iterable[addnodes.production]', node)
-        maxlen = max(len(production['tokenname']) for production in productionlist)
-        lastname = None
-        for production in productionlist:
-            if production['tokenname']:
-                lastname = production['tokenname'].ljust(maxlen)
-                self.body.append(self.starttag(production, 'strong', ''))
-                self.body.append(lastname + '</strong> ::= ')
-            elif lastname is not None:
-                self.body.append(' ' * (maxlen + 5))
-            production.walkabout(self)
-            self.body.append('\n')
-        self.body.append('</pre>\n')
-        raise nodes.SkipNode
 
     def depart_productionlist(self, node: Element) -> None:
-        pass
+        self.body.append('</pre>\n')
 
     def visit_production(self, node: Element) -> None:
         pass

sphinx/writers/latex.py

@@ -323,7 +323,7 @@ def __init__(
 
         # flags
         self.in_title = 0
-        self.in_production_list = 0
+        self.in_production_list = False
         self.in_footnote = 0
         self.in_caption = 0
         self.in_term = 0
@@ -671,20 +671,25 @@ def depart_glossary(self, node: Element) -> None:
     def visit_productionlist(self, node: Element) -> None:
         self.body.append(BLANKLINE)
         self.body.append(r'\begin{productionlist}' + CR)
-        self.in_production_list = 1
+        self.in_production_list = True
 
     def depart_productionlist(self, node: Element) -> None:
         self.body.append(r'\end{productionlist}' + BLANKLINE)
-        self.in_production_list = 0
+        self.in_production_list = False
 
     def visit_production(self, node: Element) -> None:
         if node['tokenname']:
             tn = node['tokenname']
-            self.body.append(self.hypertarget('grammar-token-' + tn))
+            self.body.append(self.hypertarget(f'grammar-token-{tn}'))
             self.body.append(r'\production{%s}{' % self.encode(tn))
         else:
             self.body.append(r'\productioncont{')
 
+        # remove name/padding and seperator child nodes,
+        # these are handled by '\production' and '\productioncont'
+        # TODO: remove special LaTeX handling of production nodes
+        node[:] = node[2:]
+
     def depart_production(self, node: Element) -> None:
         self.body.append('}' + CR)
 
@@ -2070,9 +2075,13 @@ def depart_strong(self, node: Element) -> None:
         self.body.append('}')
 
     def visit_literal_strong(self, node: Element) -> None:
+        if self.in_production_list:
+            return
         self.body.append(r'\sphinxstyleliteralstrong{\sphinxupquote{')
 
     def depart_literal_strong(self, node: Element) -> None:
+        if self.in_production_list:
+            return
         self.body.append('}}')
 
     def visit_abbreviation(self, node: Element) -> None:

sphinx/writers/manpage.py

@@ -79,8 +79,6 @@ class ManualPageTranslator(SphinxTranslator, BaseTranslator):  # type: ignore[mi
     def __init__(self, document: nodes.document, builder: Builder) -> None:
         super().__init__(document, builder)
 
-        self.in_productionlist = 0
-
         # first title is the manpage title
         self.section_level = -1
 
@@ -274,25 +272,10 @@ def depart_seealso(self, node: Element) -> None:
 
     def visit_productionlist(self, node: Element) -> None:
         self.ensure_eol()
-        self.in_productionlist += 1
         self.body.append('.sp\n.nf\n')
-        productionlist = cast('Iterable[addnodes.production]', node)
-        maxlen = max(len(production['tokenname']) for production in productionlist)
-        lastname = None
-        for production in productionlist:
-            if production['tokenname']:
-                lastname = production['tokenname'].ljust(maxlen)
-                self.body.append(self.defs['strong'][0])
-                self.body.append(self.deunicode(lastname))
-                self.body.append(self.defs['strong'][1])
-                self.body.append(' ::= ')
-            elif lastname is not None:
-                self.body.append(' ' * (maxlen + 5))
-            production.walkabout(self)
-            self.body.append('\n')
+
+    def depart_productionlist(self, node: Element) -> None:
         self.body.append('\n.fi\n')
-        self.in_productionlist -= 1
-        raise nodes.SkipNode
 
     def visit_production(self, node: Element) -> None:
         pass

sphinx/writers/texinfo.py

@@ -189,6 +189,7 @@ def __init__(self, document: nodes.document, builder: TexinfoBuilder) -> None:
         self.escape_hyphens = 0
         self.curfilestack: list[str] = []
         self.footnotestack: list[dict[str, list[collected_footnote | bool]]] = []
+        self.in_production_list = False
         self.in_footnote = 0
         self.in_samp = 0
         self.handled_abbrs: set[str] = set()
@@ -1308,20 +1309,11 @@ def unknown_departure(self, node: Node) -> None:
 
     def visit_productionlist(self, node: Element) -> None:
         self.visit_literal_block(None)
-        productionlist = cast('Iterable[addnodes.production]', node)
-        maxlen = max(len(production['tokenname']) for production in productionlist)
-
-        for production in productionlist:
-            if production['tokenname']:
-                for id in production.get('ids'):
-                    self.add_anchor(id, production)
-                s = production['tokenname'].ljust(maxlen) + ' ::='
-            else:
-                s = ' ' * (maxlen + 4)
-            self.body.append(self.escape(s))
-            self.body.append(self.escape(production.astext() + '\n'))
+        self.in_production_list = True
+
+    def depart_productionlist(self, node: Element) -> None:
+        self.in_production_list = False
         self.depart_literal_block(None)
-        raise nodes.SkipNode
 
     def visit_production(self, node: Element) -> None:
         pass
@@ -1336,9 +1328,15 @@ def depart_literal_emphasis(self, node: Element) -> None:
         self.body.append('}')
 
     def visit_literal_strong(self, node: Element) -> None:
+        if self.in_production_list:
+            for id_ in node['ids']:
+                self.add_anchor(id_, node)
+            return
         self.body.append('@code{')
 
     def depart_literal_strong(self, node: Element) -> None:
+        if self.in_production_list:
+            return
         self.body.append('}')
 
     def visit_index(self, node: Element) -> None: