Docstrings for classes and properties for extension modules (#2)

* Docstrings for class properties

* Docstrings for classes

* Validates the include-docstrings arg to generate docstrings

* Fixing test template

Author: lmbelo

mypy/stubdoc.py

@@ -72,6 +72,63 @@ def __eq__(self, other: Any) -> bool:
         return False
 
 
+class ClassSig(NamedTuple):
+    name: str
+    base_types: list[str] | None = None
+
+    def format_sig(
+        self,
+        indent: str,
+        types: list[str],
+        methods: list[str],
+        static_properties: list[str],
+        rw_properties: list[str],
+        ro_properties: list[str],
+        docstring: str | None = None,
+    ) -> list[str]:
+
+        output: list[str] = []
+
+        if self.base_types:
+            bases_str = "(%s)" % ", ".join(self.base_types)
+        else:
+            bases_str = ""
+
+        if docstring:
+            sufix = f"\n{indent}    {mypy.util.quote_docstring(docstring)}\n"
+        else:
+            sufix = ""
+
+        if types or static_properties or rw_properties or methods or ro_properties:
+            sig = f"{indent}class {self.name}{bases_str}:"
+            output.append(f"{sig}{sufix}")
+
+            for line in types:
+                if (
+                    output
+                    and output[-1]
+                    and not output[-1].strip().startswith("class")
+                    and line.strip().startswith("class")
+                ):
+                    output.append("")
+
+                output.append(line)
+
+            for line in static_properties:
+                output.append(line)
+            for line in rw_properties:
+                output.append(line)
+            for line in methods:
+                output.append(line)
+            for line in ro_properties:
+                output.append(line)
+        else:
+            sig = f"{indent}class {self.name}{bases_str}: ..."
+            output.append(f"{sig}{sufix}")
+
+        return output
+
+
 class FunctionSig(NamedTuple):
     name: str
     args: list[ArgSig]
@@ -150,6 +207,39 @@ def format_sig(
         return f"{sig}{suffix}"
 
 
+class PropertySig(NamedTuple):
+    name: str
+    prop_type: str
+
+    def format_sig(
+        self,
+        indent: str = "",
+        is_readonly: bool | None = False,
+        is_static: bool | None = False,
+        name_ref: str | None = None,
+        docstring: str | None = None,
+    ) -> str:
+
+        if is_static:
+            if docstring:
+                sufix = f"\n{indent}{mypy.util.quote_docstring(docstring)}"
+            else:
+                sufix = ""
+
+            trailing_comment = "  # read-only" if is_readonly else ""
+            sig = f"{indent}{self.name}: {name_ref}[{self.prop_type}] = ...{trailing_comment}"
+
+            return f"{sig}{sufix}"
+        else:
+            sig = f"{indent}{self.name}: {self.prop_type}"
+            if docstring:
+                sufix = f"\n{indent}{mypy.util.quote_docstring(docstring)}"
+            else:
+                sufix = ""
+
+            return f"{sig}{sufix}"
+
+
 # States of the docstring parser.
 STATE_INIT: Final = 1
 STATE_FUNCTION_NAME: Final = 2

mypy/stubgenc.py

@@ -18,7 +18,9 @@
 from mypy.moduleinspect import is_c_module
 from mypy.stubdoc import (
     ArgSig,
+    ClassSig,
     FunctionSig,
+    PropertySig,
     Sig,
     find_unique_signatures,
     infer_arg_sig_from_anon_docstring,
@@ -649,16 +651,23 @@ def generate_function_stub(
         output.extend(self.format_func_def(inferred, decorators=decorators, docstring=docstring))
         self._fix_iter(ctx, inferred, output)
 
-    def _indent_docstring(self, docstring: str) -> str:
+    def _indent_docstring(
+        self, docstring: str, extra_indent: bool = True, trailing_newline: bool = False
+    ) -> str:
         """Fix indentation of docstring extracted from pybind11 or other binding generators."""
         lines = docstring.splitlines(keepends=True)
-        indent = self._indent + "    "
+        indent = self._indent + ("    " if extra_indent else "")
         if len(lines) > 1:
             if not all(line.startswith(indent) or not line.strip() for line in lines):
                 # if the docstring is not indented, then indent all but the first line
                 for i, line in enumerate(lines[1:]):
                     if line.strip():
-                        lines[i + 1] = indent + line
+                        # ignore any left space to keep the standard ident
+                        lines[i + 1] = indent + line.lstrip()
+
+        if trailing_newline and not lines[-1].endswith("\n"):
+            lines[-1] += "\n"
+
         # if there's a trailing newline, add a final line to visually indent the quoted docstring
         if lines[-1].endswith("\n"):
             if len(lines) > 1:
@@ -728,6 +737,13 @@ def generate_property_stub(
         self.record_name(ctx.name)
         static = self.is_static_property(raw_obj)
         readonly = self.is_property_readonly(raw_obj)
+
+        if docstring:
+            # fields must define its docstring using the same ident
+            # readonly properties generates a function,
+            # which requires an extra ident in the first line
+            docstring = self._indent_docstring(docstring, extra_indent=readonly)
+
         if static:
             ret_type: str | None = self.strip_or_import(self.get_type_annotation(obj))
         else:
@@ -738,25 +754,35 @@ def generate_property_stub(
         if inferred_type is not None:
             inferred_type = self.strip_or_import(inferred_type)
 
+        if not self._include_docstrings:
+            docstring = None
+
         if static:
             classvar = self.add_name("typing.ClassVar")
-            trailing_comment = "  # read-only" if readonly else ""
             if inferred_type is None:
                 inferred_type = self.add_name("_typeshed.Incomplete")
 
+            prop_sig = PropertySig(name, inferred_type)
             static_properties.append(
-                f"{self._indent}{name}: {classvar}[{inferred_type}] = ...{trailing_comment}"
+                prop_sig.format_sig(
+                    indent=self._indent,
+                    is_readonly=readonly,
+                    is_static=True,
+                    name_ref=classvar,
+                    docstring=docstring,
+                )
             )
         else:  # regular property
             if readonly:
                 ro_properties.append(f"{self._indent}@property")
-                sig = FunctionSig(name, [ArgSig("self")], inferred_type)
-                ro_properties.append(sig.format_sig(indent=self._indent))
+                func_sig = FunctionSig(name, [ArgSig("self")], inferred_type)
+                ro_properties.append(func_sig.format_sig(indent=self._indent, docstring=docstring))
             else:
                 if inferred_type is None:
                     inferred_type = self.add_name("_typeshed.Incomplete")
 
-                rw_properties.append(f"{self._indent}{name}: {inferred_type}")
+                prop_sig = PropertySig(name, inferred_type)
+                rw_properties.append(prop_sig.format_sig(indent=self._indent, docstring=docstring))
 
     def get_type_fullname(self, typ: type) -> str:
         """Given a type, return a string representation"""
@@ -859,34 +885,27 @@ def generate_class_stub(
             classvar = self.add_name("typing.ClassVar")
             static_properties.append(f"{self._indent}{attr}: {classvar}[{prop_type_name}] = ...")
 
+        docstring = class_info.docstring if self._include_docstrings else None
+        if docstring:
+            docstring = self._indent_docstring(
+                docstring, extra_indent=False, trailing_newline=True
+            )
+
         self.dedent()
 
         bases = self.get_base_types(cls)
-        if bases:
-            bases_str = "(%s)" % ", ".join(bases)
-        else:
-            bases_str = ""
-        if types or static_properties or rw_properties or methods or ro_properties:
-            output.append(f"{self._indent}class {class_name}{bases_str}:")
-            for line in types:
-                if (
-                    output
-                    and output[-1]
-                    and not output[-1].strip().startswith("class")
-                    and line.strip().startswith("class")
-                ):
-                    output.append("")
-                output.append(line)
-            for line in static_properties:
-                output.append(line)
-            for line in rw_properties:
-                output.append(line)
-            for line in methods:
-                output.append(line)
-            for line in ro_properties:
-                output.append(line)
-        else:
-            output.append(f"{self._indent}class {class_name}{bases_str}: ...")
+        sig = ClassSig(class_name, bases)
+        output.extend(
+            sig.format_sig(
+                indent=self._indent,
+                types=types,
+                methods=methods,
+                static_properties=static_properties,
+                rw_properties=rw_properties,
+                ro_properties=ro_properties,
+                docstring=docstring,
+            )
+        )
 
     def generate_variable_stub(self, name: str, obj: object, output: list[str]) -> None:
         """Generate stub for a single variable using runtime introspection.

test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/__init__.pyi

@@ -34,11 +34,18 @@ class StaticMethods:
 
 class TestStruct:
     field_readwrite: int
+    """(self: pybind11_fixtures.TestStruct) -> int"""
     field_readwrite_docstring: int
+    """some docstring
+    (self: pybind11_fixtures.TestStruct) -> int
+    """
     def __init__(self, *args, **kwargs) -> None:
         """Initialize self.  See help(type(self)) for accurate signature."""
     @property
-    def field_readonly(self) -> int: ...
+    def field_readonly(self) -> int:
+        """some docstring
+        (arg0: pybind11_fixtures.TestStruct) -> int
+        """
 
 def func_incomplete_signature(*args, **kwargs):
     """func_incomplete_signature() -> dummy_sub_namespace::HasNoBinding"""

test-data/pybind11_fixtures/expected_stubs_with_docs/pybind11_fixtures/demo.pyi

@@ -5,7 +5,15 @@ __version__: str
 
 class Point:
     class AngleUnit:
+        """Members:
+
+        radian
+
+        degree
+        """
+
         __members__: ClassVar[dict] = ...  # read-only
+        """__members__(arg0: handle) -> dict"""
         __entries: ClassVar[dict] = ...
         degree: ClassVar[Point.AngleUnit] = ...
         radian: ClassVar[Point.AngleUnit] = ...
@@ -22,12 +30,27 @@ class Point:
         def __ne__(self, other: object) -> bool:
             """__ne__(self: object, other: object) -> bool"""
         @property
-        def name(self) -> str: ...
+        def name(self) -> str:
+            """name(self: handle) -> str
+
+            name(self: handle) -> str
+            """
         @property
-        def value(self) -> int: ...
+        def value(self) -> int:
+            """(arg0: pybind11_fixtures.demo.Point.AngleUnit) -> int"""
 
     class LengthUnit:
+        """Members:
+
+        mm
+
+        pixel
+
+        inch
+        """
+
         __members__: ClassVar[dict] = ...  # read-only
+        """__members__(arg0: handle) -> dict"""
         __entries: ClassVar[dict] = ...
         inch: ClassVar[Point.LengthUnit] = ...
         mm: ClassVar[Point.LengthUnit] = ...
@@ -45,16 +68,29 @@ class Point:
         def __ne__(self, other: object) -> bool:
             """__ne__(self: object, other: object) -> bool"""
         @property
-        def name(self) -> str: ...
+        def name(self) -> str:
+            """name(self: handle) -> str
+
+            name(self: handle) -> str
+            """
         @property
-        def value(self) -> int: ...
+        def value(self) -> int:
+            """(arg0: pybind11_fixtures.demo.Point.LengthUnit) -> int"""
     angle_unit: ClassVar[Point.AngleUnit] = ...
+    """(arg0: object) -> pybind11_fixtures.demo.Point.AngleUnit"""
     length_unit: ClassVar[Point.LengthUnit] = ...
+    """(arg0: object) -> pybind11_fixtures.demo.Point.LengthUnit"""
     x_axis: ClassVar[Point] = ...  # read-only
+    """(arg0: object) -> pybind11_fixtures.demo.Point"""
     y_axis: ClassVar[Point] = ...  # read-only
+    """(arg0: object) -> pybind11_fixtures.demo.Point"""
     origin: ClassVar[Point] = ...
     x: float
+    """some docstring
+    (self: pybind11_fixtures.demo.Point) -> float
+    """
     y: float
+    """(arg0: pybind11_fixtures.demo.Point) -> float"""
     @overload
     def __init__(self) -> None:
         """__init__(*args, **kwargs)
@@ -94,7 +130,8 @@ class Point:
         2. distance_to(self: pybind11_fixtures.demo.Point, other: pybind11_fixtures.demo.Point) -> float
         """
     @property
-    def length(self) -> float: ...
+    def length(self) -> float:
+        """(arg0: pybind11_fixtures.demo.Point) -> float"""
 
 def answer() -> int:
     '''answer() -> int