Add partial support for PEP 695 and PEP 696 syntax (#11438)

Author: picnixz

CHANGES

@@ -18,6 +18,10 @@ Deprecated
 Features added
 --------------
 
+* #11438: Add support for the :rst:dir:`py:class` and :rst:dir:`py:function`
+  directives for PEP 695 (generic classes and functions declarations) and
+  PEP 696 (default type parameters).
+  Patch by Bénédikt Tran.
 * #11415: Add a checksum to JavaScript and CSS asset URIs included within
   generated HTML, using the CRC32 algorithm.
 * :meth:`~sphinx.application.Sphinx.require_sphinx` now allows the version

sphinx/addnodes.py

@@ -253,9 +253,17 @@ class desc_parameterlist(nodes.Part, nodes.Inline, nodes.FixedTextElement):
     In that case each parameter will then be written on its own, indented line.
     """
     child_text_separator = ', '
+    list_left_delim = '('
+    list_right_delim = ')'
 
     def astext(self):
-        return f'({super().astext()})'
+        return f'{self.list_left_delim}{super().astext()}{self.list_right_delim}'
+
+
+class desc_tparameterlist(desc_parameterlist):
+    """Node for a general type parameter list."""
+    list_left_delim = '['
+    list_right_delim = ']'
 
 
 class desc_parameter(nodes.Part, nodes.Inline, nodes.FixedTextElement):
@@ -537,6 +545,7 @@ def setup(app: Sphinx) -> dict[str, Any]:
     app.add_node(desc_type)
     app.add_node(desc_returns)
     app.add_node(desc_parameterlist)
+    app.add_node(desc_tparameterlist)
     app.add_node(desc_parameter)
     app.add_node(desc_optional)
     app.add_node(desc_annotation)

sphinx/domains/python.py

@@ -23,6 +23,7 @@
 from sphinx.domains import Domain, Index, IndexEntry, ObjType
 from sphinx.environment import BuildEnvironment
 from sphinx.locale import _, __
+from sphinx.pycode.parser import Token, TokenProcessor
 from sphinx.roles import XRefRole
 from sphinx.util import logging
 from sphinx.util.docfields import Field, GroupedField, TypedField
@@ -39,10 +40,11 @@
 logger = logging.getLogger(__name__)
 
 
-# REs for Python signatures
+# REs for Python signatures (supports PEP 695)
 py_sig_re = re.compile(
     r'''^ ([\w.]*\.)?            # class name(s)
           (\w+)  \s*             # thing name
+          (?: \[\s*(.*)\s*])?    # optional: type parameters list (PEP 695)
           (?: \(\s*(.*)\s*\)     # optional: arguments
            (?:\s* -> \s* (.*))?  #           return annotation
           )? $                   # and nothing more
@@ -257,6 +259,146 @@ def _unparse_pep_604_annotation(node: ast.Subscript) -> list[Node]:
         return [type_to_xref(annotation, env)]
 
 
+class _TypeParameterListParser(TokenProcessor):
+    def __init__(self, sig: str) -> None:
+        from token import ERRORTOKEN
+
+        # By default, tokenizing "[T = dict[str, Any]]" gives "dict[str,Any]"
+        # instead of dict[str, Any]. In particular, the default parameter value
+        # will not be formatted properly. Therefore, whitespaces are replaced
+        # by some sentinel (bad) value that can be detected as an ERRORTOKEN.
+        self.virtual_ws = '\x00'
+        self.virtual_ws_tok = [ERRORTOKEN, self.virtual_ws]
+
+        signature = ''.join(sig.splitlines()).replace(' ', self.virtual_ws)
+        super().__init__([signature])
+        # Each item is a tuple (name, kind, default, bound) mimicking
+        # inspect.Parameter to allow default values on VAR_POSITIONAL
+        # or VAR_KEYWORD parameters.
+        self.tparams: list[tuple[str, int, Any, Any]] = []
+        # When true, (leading) whitespaces are dropped when fetching a token.
+        # Set it to false when parsing a type bound or a default value so that
+        # they are properly rendered.
+        self.ignore_ws = False
+
+    def fetch_token(self) -> Token | None:
+        if not self.ignore_ws:
+            return super().fetch_token()
+
+        while super().fetch_token() == self.virtual_ws_tok:
+            assert self.current
+        return self.current
+
+    def fetch_tparam_spec(self) -> list[Token]:
+        from token import DEDENT, INDENT, OP
+
+        tokens = []
+        self.ignore_ws = False
+        while self.fetch_token():
+            tokens.append(self.current)
+            for ldelim, rdelim in ['()', '{}', '[]']:
+                if self.current == [OP, ldelim]:
+                    tokens += self.fetch_until([OP, rdelim])
+                    break
+            else:
+                if self.current == INDENT:
+                    tokens += self.fetch_until(DEDENT)
+                elif self.current.match([OP, ':'], [OP, '='], [OP, ',']):
+                    tokens.pop()
+                    break
+        self.ignore_ws = True
+        return tokens
+
+    def parse(self) -> None:
+        from token import NAME, OP
+
+        def build_identifier(tokens: Iterable[Token]) -> str:
+            ws = self.virtual_ws_tok
+            ident = ''.join(' ' if tok == ws else tok.value for tok in tokens)
+            return ident.strip()
+
+        while self.fetch_token():
+            if self.current == NAME:
+                tpname: list[str] = build_identifier([self.current])
+                if self.previous and self.previous.match([OP, '*'], [OP, '**']):
+                    if self.previous == [OP, '*']:
+                        tpkind = Parameter.VAR_POSITIONAL
+                    else:
+                        tpkind = Parameter.VAR_KEYWORD
+                else:
+                    tpkind = Parameter.POSITIONAL_OR_KEYWORD
+
+                tpbound, tpdefault = Parameter.empty, Parameter.empty
+
+                self.fetch_token()  # whitespaces (before) will be ignored
+                if self.current and self.current.match([OP, ':'], [OP, '=']):
+                    if self.current == [OP, ':']:
+                        tpbound = build_identifier(self.fetch_tparam_spec())
+                    if self.current == [OP, '=']:
+                        tpdefault = build_identifier(self.fetch_tparam_spec())
+
+                if tpkind != Parameter.POSITIONAL_OR_KEYWORD and tpbound != Parameter.empty:
+                    raise SyntaxError('type parameter bound or constraint is not allowed '
+                                      f'for {tpkind.description} parameters')
+
+                tparam = (tpname, tpkind, tpdefault, tpbound)
+                self.tparams.append(tparam)
+
+
+def _parse_tplist(
+    tplist: str, env: BuildEnvironment | None = None,
+    multi_line_parameter_list: bool = False,
+) -> addnodes.desc_tparameterlist:
+    """Parse a list of type parameters according to PEP 695."""
+    tparams = addnodes.desc_tparameterlist(tplist)
+    tparams['multi_line_parameter_list'] = multi_line_parameter_list
+    # formal parameter names are interpreted as type parameter names and
+    # type annotations are interpreted as type parameter bounds
+    parser = _TypeParameterListParser(tplist)
+    parser.parse()
+    for (tpname, tpkind, tpdefault, tpbound) in parser.tparams:
+        # no positional-only or keyword-only allowed in a type parameters list
+        assert tpkind not in {Parameter.POSITIONAL_ONLY, Parameter.KEYWORD_ONLY}
+
+        node = addnodes.desc_parameter()
+        if tpkind == Parameter.VAR_POSITIONAL:
+            node += addnodes.desc_sig_operator('', '*')
+        elif tpkind == Parameter.VAR_KEYWORD:
+            node += addnodes.desc_sig_operator('', '**')
+        node += addnodes.desc_sig_name('', tpname)
+
+        if tpbound is not Parameter.empty:
+            type_bound = _parse_annotation(tpbound, env)
+            if not type_bound:
+                continue
+
+            node += addnodes.desc_sig_punctuation('', ':')
+            node += addnodes.desc_sig_space()
+
+            type_bound_expr = addnodes.desc_sig_name('', '', *type_bound)  # type: ignore
+
+            # add delimiters around type bounds written as e.g., "(T1, T2)"
+            if tpbound.startswith('(') and tpbound.endswith(')'):
+                node += addnodes.desc_sig_punctuation('', '(')
+                node += type_bound_expr
+                node += addnodes.desc_sig_punctuation('', ')')
+            else:
+                node += type_bound_expr
+
+        if tpdefault is not Parameter.empty:
+            if tpbound is not Parameter.empty or tpkind != Parameter.POSITIONAL_OR_KEYWORD:
+                node += addnodes.desc_sig_space()
+                node += addnodes.desc_sig_operator('', '=')
+                node += addnodes.desc_sig_space()
+            else:
+                node += addnodes.desc_sig_operator('', '=')
+            node += nodes.inline('', tpdefault, classes=['default_value'],
+                                 support_smartquotes=False)
+
+        tparams += node
+    return tparams
+
+
 def _parse_arglist(
     arglist: str, env: BuildEnvironment | None = None, multi_line_parameter_list: bool = False,
 ) -> addnodes.desc_parameterlist:
@@ -514,7 +656,7 @@ def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]
         m = py_sig_re.match(sig)
         if m is None:
             raise ValueError
-        prefix, name, arglist, retann = m.groups()
+        prefix, name, tplist, arglist, retann = m.groups()
 
         # determine module and class name (if applicable), as well as full name
         modname = self.options.get('module', self.env.ref_context.get('py:module'))
@@ -570,6 +712,14 @@ def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]
             signode += addnodes.desc_addname(nodetext, nodetext)
 
         signode += addnodes.desc_name(name, name)
+
+        if tplist:
+            try:
+                signode += _parse_tplist(tplist, self.env, multi_line_parameter_list)
+            except Exception as exc:
+                logger.warning("could not parse tplist (%r): %s", tplist, exc,
+                               location=signode)
+
         if arglist:
             try:
                 signode += _parse_arglist(arglist, self.env, multi_line_parameter_list)

sphinx/writers/html5.py

@@ -149,7 +149,8 @@ def depart_desc_returns(self, node: Element) -> None:
         self.body.append('</span></span>')
 
     def visit_desc_parameterlist(self, node: Element) -> None:
-        self.body.append('<span class="sig-paren">(</span>')
+        list_left_delim = node.list_left_delim  # type: ignore[attr-defined]
+        self.body.append(f'<span class="sig-paren">{list_left_delim}</span>')
         self.is_first_param = True
         self.optional_param_level = 0
         self.params_left_at_level = 0
@@ -170,7 +171,8 @@ def visit_desc_parameterlist(self, node: Element) -> None:
     def depart_desc_parameterlist(self, node: Element) -> None:
         if node.get('multi_line_parameter_list'):
             self.body.append('</dl>\n\n')
-        self.body.append('<span class="sig-paren">)</span>')
+        list_right_delim = node.list_right_delim  # type: ignore[attr-defined]
+        self.body.append(f'<span class="sig-paren">{list_right_delim}</span>')
 
     # If required parameters are still to come, then put the comma after
     # the parameter.  Otherwise, put the comma before.  This ensures that

sphinx/writers/manpage.py

@@ -184,11 +184,11 @@ def depart_desc_returns(self, node: Element) -> None:
         pass
 
     def visit_desc_parameterlist(self, node: Element) -> None:
-        self.body.append('(')
+        self.body.append(node.list_left_delim)  # type: ignore[attr-defined]
         self.first_param = 1
 
     def depart_desc_parameterlist(self, node: Element) -> None:
-        self.body.append(')')
+        self.body.append(node.list_right_delim)  # type: ignore[attr-defined]
 
     def visit_desc_parameter(self, node: Element) -> None:
         if not self.first_param:

sphinx/writers/texinfo.py

@@ -1462,11 +1462,11 @@ def depart_desc_returns(self, node: Element) -> None:
         pass
 
     def visit_desc_parameterlist(self, node: Element) -> None:
-        self.body.append(' (')
+        self.body.append(f' {node.list_left_delim}')  # type: ignore[attr-defined]
         self.first_param = 1
 
     def depart_desc_parameterlist(self, node: Element) -> None:
-        self.body.append(')')
+        self.body.append(node.list_right_delim)  # type: ignore[attr-defined]
 
     def visit_desc_parameter(self, node: Element) -> None:
         if not self.first_param:

sphinx/writers/text.py

@@ -593,7 +593,7 @@ def depart_desc_returns(self, node: Element) -> None:
         pass
 
     def visit_desc_parameterlist(self, node: Element) -> None:
-        self.add_text('(')
+        self.add_text(node.list_left_delim)  # type: ignore[attr-defined]
         self.is_first_param = True
         self.optional_param_level = 0
         self.params_left_at_level = 0
@@ -609,7 +609,7 @@ def visit_desc_parameterlist(self, node: Element) -> None:
             self.param_separator = self.param_separator.rstrip()
 
     def depart_desc_parameterlist(self, node: Element) -> None:
-        self.add_text(')')
+        self.add_text(node.list_right_delim)  # type: ignore[attr-defined]
 
     def visit_desc_parameter(self, node: Element) -> None:
         on_separate_line = self.multi_line_parameter_list