Fix parsing of PEP 695 functions (#13328)

Author: picnixz

CHANGES.rst

@@ -153,6 +153,8 @@ Bugs fixed
 * #13302, #13319: Use the correct indentation for continuation lines
   in :rst:dir:`productionlist` directives.
   Patch by Adam Turner.
+* #13328: Fix parsing of PEP 695 functions with return annotations.
+  Patch by Bénédikt Tran. Initial work by Arash Badie-Modiri.
 
 Testing
 -------

sphinx/domains/python/_object.py

@@ -41,7 +41,7 @@
 py_sig_re = re.compile(
     r"""^ ([\w.]*\.)?            # class name(s)
           (\w+)  \s*             # thing name
-          (?: \[\s*(.*)\s*])?    # optional: type parameters list
+          (?: \[\s*(.*?)\s*])?   # optional: type parameters list
           (?: \(\s*(.*)\s*\)     # optional: arguments
            (?:\s* -> \s* (.*))?  #           return annotation
           )? $                   # and nothing more

sphinx/ext/autodoc/__init__.py

@@ -66,7 +66,7 @@
     r"""^ ([\w.]+::)?            # explicit module name
           ([\w.]+\.)?            # module and/or class name(s)
           (\w+)  \s*             # thing name
-          (?: \[\s*(.*)\s*])?    # optional: type parameters list
+          (?: \[\s*(.*?)\s*])?   # optional: type parameters list
           (?: \((.*)\)           # optional: arguments
            (?:\s* -> \s* (.*))?  #           return annotation
           )? $                   # and nothing more

tests/test_domains/test_domain_py.py

@@ -50,23 +50,28 @@ def parse(sig):
 
 
 def test_function_signatures():
-    rv = parse('func(a=1) -> int object')
-    assert rv == '(a=1)'
-
-    rv = parse('func(a=1, [b=None])')
-    assert rv == '(a=1, [b=None])'
-
-    rv = parse('func(a=1[, b=None])')
-    assert rv == '(a=1, [b=None])'
-
     rv = parse("compile(source : string, filename, symbol='file')")
     assert rv == "(source : string, filename, symbol='file')"
 
-    rv = parse('func(a=[], [b=None])')
-    assert rv == '(a=[], [b=None])'
-
-    rv = parse('func(a=[][, b=None])')
-    assert rv == '(a=[], [b=None])'
+    for params, expect in [
+        ('(a=1)', '(a=1)'),
+        ('(a: int = 1)', '(a: int = 1)'),
+        ('(a=1, [b=None])', '(a=1, [b=None])'),
+        ('(a=1[, b=None])', '(a=1, [b=None])'),
+        ('(a=[], [b=None])', '(a=[], [b=None])'),
+        ('(a=[][, b=None])', '(a=[], [b=None])'),
+        ('(a: Foo[Bar]=[][, b=None])', '(a: Foo[Bar]=[], [b=None])'),
+    ]:
+        rv = parse(f'func{params}')
+        assert rv == expect
+
+        # Note: 'def f[Foo[Bar]]()' is not valid Python but people might write
+        # it in a reST document to convene the intent of a higher-kinded type
+        # variable.
+        for tparams in ['', '[Foo]', '[Foo[Bar]]']:
+            for retann in ['', '-> Foo', '-> Foo[Bar]', '-> anything else']:
+                rv = parse(f'func{tparams}{params} {retann}'.rstrip())
+                assert rv == expect
 
 
 @pytest.mark.sphinx('dummy', testroot='domain-py')
@@ -1710,6 +1715,10 @@ def test_pep_695_and_pep_696_whitespaces_in_bound(app, tp_list, tptext):
     doctree = restructuredtext.parse(app, text)
     assert doctree.astext() == f'\n\nf{tptext}()\n\n'
 
+    text = f'.. py:function:: f{tp_list}() -> Annotated[T, Qux[int]()]'
+    doctree = restructuredtext.parse(app, text)
+    assert doctree.astext() == f'\n\nf{tptext}() -> Annotated[T, Qux[int]()]\n\n'
+
 
 @pytest.mark.parametrize(
     ('tp_list', 'tptext'),
@@ -1724,6 +1733,10 @@ def test_pep_695_and_pep_696_whitespaces_in_constraints(app, tp_list, tptext):
     doctree = restructuredtext.parse(app, text)
     assert doctree.astext() == f'\n\nf{tptext}()\n\n'
 
+    text = f'.. py:function:: f{tp_list}() -> Annotated[T, Qux[int]()]'
+    doctree = restructuredtext.parse(app, text)
+    assert doctree.astext() == f'\n\nf{tptext}() -> Annotated[T, Qux[int]()]\n\n'
+
 
 @pytest.mark.parametrize(
     ('tp_list', 'tptext'),
@@ -1747,3 +1760,7 @@ def test_pep_695_and_pep_696_whitespaces_in_default(app, tp_list, tptext):
     text = f'.. py:function:: f{tp_list}()'
     doctree = restructuredtext.parse(app, text)
     assert doctree.astext() == f'\n\nf{tptext}()\n\n'
+
+    text = f'.. py:function:: f{tp_list}() -> Annotated[T, Qux[int]()]'
+    doctree = restructuredtext.parse(app, text)
+    assert doctree.astext() == f'\n\nf{tptext}() -> Annotated[T, Qux[int]()]\n\n'

tests/test_extensions/test_ext_autodoc.py

@@ -177,6 +177,28 @@ def g(a='\n'):
     assert formatsig('function', 'f', f, 'a, b, c, d', None) == '(a, b, c, d)'
     assert formatsig('function', 'g', g, None, None) == r"(a='\n')"
 
+    if sys.version_info >= (3, 12):
+        for params, expect in [
+            ('(a=1)', '(a=1)'),
+            ('(a: int=1)', '(a: int = 1)'),  # auto whitespace formatting
+            ('(a:list[T]   =[], b=None)', '(a: list[T] = [], b=None)'),  # idem
+        ]:
+            ns = {}
+            exec(f'def f[T]{params}: pass', ns)  # NoQA: S102
+            f = ns['f']
+            assert formatsig('function', 'f', f, None, None) == expect
+            assert formatsig('function', 'f', f, '...', None) == '(...)'
+            assert formatsig('function', 'f', f, '...', '...') == '(...) -> ...'
+
+            exec(f'def f[T]{params} -> list[T]: return []', ns)  # NoQA: S102
+            f = ns['f']
+            assert formatsig('function', 'f', f, None, None) == f'{expect} -> list[T]'
+            assert formatsig('function', 'f', f, '...', None) == '(...)'
+            assert formatsig('function', 'f', f, '...', '...') == '(...) -> ...'
+
+    # TODO(picnixz): add more test cases for PEP-695 classes as well (though
+    # complex cases are less likely to appear and are painful to test).
+
     # test for classes
     class D:
         pass