Missing rename for numpy.ndarray.flags (#128)

- 🐛 Fix missing remap of `numpy.ndarray.flags`
- ✨ Process `scipy.sparse.*` types the same as `numpy.ndarray` with
`--numpy-array-wrap-with-annotated`
- ✨ Support dynamic array size with `--numpy-array-wrap-with-annotated`
- ❗️ Renamed CLI argument `--numpy-array-wrap-with-annotated-fixed-size`
to `--numpy-array-wrap-with-annotated`

Author: sizmailov

CHANGELOG.md

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+Version 1.x (TBA)
+--------------------------
+Changes:
+
+- 🐛 Fix missing remap of `numpy.ndarray.flags` (#128)
+- ✨ Process `scipy.sparse.*` types the same as `numpy.ndarray` with `--numpy-array-wrap-with-annotated` (#128)
+- ✨ Support dynamic array size with `--numpy-array-wrap-with-annotated` (#128)
+- ❗️ Renamed CLI argument `--numpy-array-wrap-with-annotated-fixed-size` to `--numpy-array-wrap-with-annotated` (#128)
+
 
 Version 1.2 (Aug 31, 2023)
 --------------------------

pybind11_stubgen/__init__.py

@@ -9,7 +9,6 @@
 from pybind11_stubgen.parser.interface import IParser
 from pybind11_stubgen.parser.mixins.error_handlers import (
     IgnoreAllErrors,
-    IgnoreFixedErrors,
     IgnoreInvalidExpressionErrors,
     IgnoreInvalidIdentifierErrors,
     IgnoreUnresolvedNameErrors,
@@ -32,6 +31,7 @@
     FixMissingImports,
     FixMissingNoneHashFieldAnnotation,
     FixNumpyArrayDimAnnotation,
+    FixNumpyArrayFlags,
     FixNumpyArrayRemoveParameters,
     FixPEP585CollectionNames,
     FixPybind11EnumStrDoc,
@@ -109,11 +109,12 @@ def regex(pattern_str: str) -> re.Pattern:
 
     numpy_array_fix = parser.add_mutually_exclusive_group()
     numpy_array_fix.add_argument(
-        "--numpy-array-wrap-with-annotated-fixed-size",
+        "--numpy-array-wrap-with-annotated",
         default=False,
         action="store_true",
-        help="Replace 'numpy.ndarray[<TYPE>, [*DIMS]]' with "
-        "'Annotated[numpy.ndarray, TYPE, FixedSize(*DIMS)]'",
+        help="Replace numpy/scipy arrays of "
+        "'ARRAY_T[<TYPE>, [*DIMS], *FLAGS]' format with "
+        "'Annotated[ARRAY_T, TYPE, FixedSize|DynamicSize(*DIMS), *FLAGS]'",
     )
 
     numpy_array_fix.add_argument(
@@ -155,23 +156,20 @@ def regex(pattern_str: str) -> re.Pattern:
 
 
 def stub_parser_from_args(args) -> IParser:
-    error_handlers: list[type] = [
+    error_handlers_top: list[type] = [
         *([IgnoreAllErrors] if args.ignore_all_errors else []),
         *([IgnoreInvalidIdentifierErrors] if args.ignore_invalid_identifiers else []),
         *([IgnoreInvalidExpressionErrors] if args.ignore_invalid_expressions else []),
         *([IgnoreUnresolvedNameErrors] if args.ignore_unresolved_names else []),
-        IgnoreFixedErrors,
+    ]
+    error_handlers_bottom: list[type] = [
         LogErrors,
         SuggestCxxSignatureFix,
         *([TerminateOnFatalErrors] if args.exit_code else []),
     ]
 
     numpy_fixes: list[type] = [
-        *(
-            [FixNumpyArrayDimAnnotation]
-            if args.numpy_array_wrap_with_annotated_fixed_size
-            else []
-        ),
+        *([FixNumpyArrayDimAnnotation] if args.numpy_array_wrap_with_annotated else []),
         *(
             [FixNumpyArrayRemoveParameters]
             if args.numpy_array_remove_parameters
@@ -180,7 +178,7 @@ def stub_parser_from_args(args) -> IParser:
     ]
 
     class Parser(
-        *error_handlers,  # type: ignore[misc]
+        *error_handlers_top,  # type: ignore[misc]
         FixMissing__future__AnnotationsImport,
         FixMissing__all__Attribute,
         FixMissingNoneHashFieldAnnotation,
@@ -191,6 +189,7 @@ class Parser(
         FixMissingFixedSizeImport,
         FixMissingEnumMembersAnnotation,
         *numpy_fixes,  # type: ignore[misc]
+        FixNumpyArrayFlags,
         FixCurrentModulePrefixInTypeNames,
         FixBuiltinTypes,
         FilterClassMembers,
@@ -205,6 +204,7 @@ class Parser(
         ExtractSignaturesFromPybind11Docstrings,
         ParserDispatchMixin,
         BaseParser,
+        *error_handlers_bottom,  # type: ignore[misc]
     ):
         pass
 

pybind11_stubgen/parser/mixins/error_handlers.py

@@ -43,30 +43,6 @@ def report_error(self, error: ParserError) -> None:
         super().report_error(error)
 
 
-class IgnoreFixedErrors(IParser):
-    def report_error(self, error: ParserError):
-        if isinstance(error, NameResolutionError):
-            if error.name[0] in ["pybind11_builtins", "PyCapsule", "module"]:
-                return
-        elif isinstance(error, InvalidExpressionError):
-            if error.expression.startswith("FixedSize"):
-                # https://github.com/pybind/pybind11/pull/4679
-                return
-        elif isinstance(error, InvalidIdentifierError):
-            name = error.name
-            if (
-                name.startswith("ItemsView[")
-                and name.endswith("]")
-                or name.startswith("KeysView[")
-                and name.endswith("]")
-                or name.startswith("ValuesView[")
-                and name.endswith("]")
-            ):
-                return
-
-        super().report_error(error)
-
-
 class IgnoreUnresolvedNameErrors(IParser):
     def __init__(self):
         super().__init__()
@@ -154,8 +130,8 @@ def __init__(self):
         self.__found_fatal_errors = False
 
     def report_error(self, error: ParserError):
-        super().report_error(error)
         self.__found_fatal_errors = True
+        super().report_error(error)
 
     def finalize(self):
         super().finalize()

pybind11_stubgen/parser/mixins/filter.py

@@ -74,16 +74,6 @@ def handle_attribute(self, path: QualifiedName, attr: Any) -> Attribute | None:
             return None
         return super().handle_attribute(path, attr)
 
-    def handle_bases(
-        self, path: QualifiedName, bases: tuple[type, ...]
-    ) -> list[QualifiedName]:
-        result = []
-        for base in super().handle_bases(path, bases):
-            if base[0] == "pybind11_builtins":
-                break
-            result.append(base)
-        return result
-
 
 class FilterPybindInternals(IParser):
     __attribute_blacklist: set[Identifier] = {*map(Identifier, ("__entries",))}

pybind11_stubgen/parser/mixins/fix.py

@@ -7,7 +7,7 @@
 import types
 from typing import Any
 
-from pybind11_stubgen.parser.errors import NameResolutionError
+from pybind11_stubgen.parser.errors import NameResolutionError, ParserError
 from pybind11_stubgen.parser.interface import IParser
 from pybind11_stubgen.structs import (
     Alias,
@@ -27,7 +27,7 @@
     ResolvedType,
     Value,
 )
-from pybind11_stubgen.typing_ext import FixedSize
+from pybind11_stubgen.typing_ext import DynamicSize, FixedSize
 
 
 class RemoveSelfAnnotation(IParser):
@@ -252,6 +252,12 @@ def handle_type(self, type_: type) -> QualifiedName:
 
         return result
 
+    def report_error(self, error: ParserError):
+        if isinstance(error, NameResolutionError):
+            if error.name[0] in ["PyCapsule"]:
+                return
+        super().report_error(error)
+
 
 class FixRedundantBuiltinsAnnotation(IParser):
     def handle_attribute(self, path: QualifiedName, attr: Any) -> Attribute | None:
@@ -451,7 +457,14 @@ def value_to_repr(self, value: Any) -> str:
 
 
 class FixNumpyArrayDimAnnotation(IParser):
-    __ndarray_name = QualifiedName.from_str("numpy.ndarray")
+    __array_names: set[QualifiedName] = {
+        QualifiedName.from_str("numpy.ndarray"),
+        *(
+            QualifiedName.from_str(f"scipy.sparse.{storage}_{arr}")
+            for storage in ["bsr", "coo", "csr", "csc", "dia", "dok", "lil"]
+            for arr in ["array", "matrix"]
+        ),
+    }
     __annotated_name = QualifiedName.from_str("typing.Annotated")
     numpy_primitive_types: set[QualifiedName] = set(
         map(
@@ -475,47 +488,79 @@ def parse_annotation_str(
         self, annotation_str: str
     ) -> ResolvedType | InvalidExpression | Value:
         # Affects types of the following pattern:
-        #       numpy.ndarray[PRIMITIVE_TYPE[*DIMS]]
-        #       Annotated[numpy.ndarray, PRIMITIVE_TYPE, FixedSize[*DIMS]]
+        #       ARRAY_T[PRIMITIVE_TYPE[*DIMS], *FLAGS]
+        # Replace with:
+        #       Annotated[ARRAY_T, PRIMITIVE_TYPE, FixedSize/DynamicSize[*DIMS], *FLAGS]
 
         result = super().parse_annotation_str(annotation_str)
         if (
             not isinstance(result, ResolvedType)
-            or result.name != self.__ndarray_name
+            or result.name not in self.__array_names
             or result.parameters is None
-            or len(result.parameters) != 1
+            or len(result.parameters) == 0
         ):
             return result
-        param = result.parameters[0]
+
+        scalar_with_dims = result.parameters[0]  # e.g. numpy.float64[32, 32]
+        flags = result.parameters[1:]
+
         if (
-            not isinstance(param, ResolvedType)
-            or param.name not in self.numpy_primitive_types
-            or param.parameters is None
-            or any(not isinstance(dim, Value) for dim in param.parameters)
+            not isinstance(scalar_with_dims, ResolvedType)
+            or scalar_with_dims.name not in self.numpy_primitive_types
+            or (
+                scalar_with_dims.parameters is not None
+                and any(
+                    not isinstance(dim, Value) for dim in scalar_with_dims.parameters
+                )
+            )
         ):
             return result
 
-        # isinstance check is redundant, but makes mypy happy
-        dims = [int(dim.repr) for dim in param.parameters if isinstance(dim, Value)]
-
-        # override result with Annotated[...]
         result = ResolvedType(
             name=self.__annotated_name,
             parameters=[
-                ResolvedType(self.__ndarray_name),
-                ResolvedType(param.name),
+                self.parse_annotation_str(str(result.name)),
+                ResolvedType(scalar_with_dims.name),
             ],
         )
 
-        if param.parameters is not None:
-            # TRICK: Use `self.parse_type` to make `FixedSize`
-            #        properly added to the list of imports
-            self.handle_type(FixedSize)
-            assert result.parameters is not None
-            result.parameters += [self.handle_value(FixedSize(*dims))]
+        if (
+            scalar_with_dims.parameters is not None
+            and len(scalar_with_dims.parameters) >= 0
+        ):
+            result.parameters += [
+                self.handle_value(
+                    self._cook_dimension_parameters(scalar_with_dims.parameters)
+                )
+            ]
+
+        result.parameters += flags
 
         return result
 
+    def _cook_dimension_parameters(
+        self, dimensions: list[Value]
+    ) -> FixedSize | DynamicSize:
+        all_ints = True
+        new_params = []
+        for dim_param in dimensions:
+            try:
+                dim = int(dim_param.repr)
+            except ValueError:
+                dim = dim_param.repr
+                all_ints = False
+            new_params.append(dim)
+
+        if all_ints:
+            return_t = FixedSize
+        else:
+            return_t = DynamicSize
+
+        # TRICK: Use `self.handle_type` to make `FixedSize`/`DynamicSize`
+        #        properly added to the list of imports
+        self.handle_type(FixedSize)
+        return return_t(*new_params)
+
 
 class FixNumpyArrayRemoveParameters(IParser):
     __ndarray_name = QualifiedName.from_str("numpy.ndarray")
@@ -529,6 +574,34 @@ def parse_annotation_str(
         return result
 
 
+class FixNumpyArrayFlags(IParser):
+    __ndarray_name = QualifiedName.from_str("numpy.ndarray")
+    __flags: set[QualifiedName] = {
+        QualifiedName.from_str("flags.writeable"),
+        QualifiedName.from_str("flags.c_contiguous"),
+        QualifiedName.from_str("flags.f_contiguous"),
+    }
+
+    def parse_annotation_str(
+        self, annotation_str: str
+    ) -> ResolvedType | InvalidExpression | Value:
+        result = super().parse_annotation_str(annotation_str)
+        if isinstance(result, ResolvedType) and result.name == self.__ndarray_name:
+            if result.parameters is not None:
+                for param in result.parameters:
+                    if param.name in self.__flags:
+                        param.name = QualifiedName.from_str(
+                            f"numpy.ndarray.{param.name}"
+                        )
+
+        return result
+
+    def report_error(self, error: ParserError) -> None:
+        if isinstance(error, NameResolutionError) and error.name in self.__flags:
+            return
+        super().report_error(error)
+
+
 class FixRedundantMethodsFromBuiltinObject(IParser):
     def handle_method(self, path: QualifiedName, method: Any) -> list[Method]:
         result = super().handle_method(path, method)

pybind11_stubgen/parser/mixins/parse.py

@@ -10,6 +10,7 @@
     InvalidExpressionError,
     InvalidIdentifierError,
     NameResolutionError,
+    ParserError,
 )
 from pybind11_stubgen.parser.interface import IParser
 from pybind11_stubgen.structs import (
@@ -379,6 +380,12 @@ def value_to_repr(self, value: Any) -> str:
             return "..."
         return repr(value)
 
+    def report_error(self, error: ParserError):
+        if isinstance(error, NameResolutionError):
+            if error.name[0] == "module":
+                return
+        super().report_error(error)
+
     def _get_method_modifier(self, args: list[Argument]) -> Modifier:
         if len(args) == 0:
             return "static"
@@ -808,3 +815,31 @@ def _strip_empty_lines(self, doc_lines: list[str]) -> Docstring | None:
         if len(result) == 0:
             return None
         return Docstring(result)
+
+    def report_error(self, error: ParserError) -> None:
+        if isinstance(error, NameResolutionError):
+            if error.name[0] == "pybind11_builtins":
+                return
+        if isinstance(error, InvalidIdentifierError):
+            name = error.name
+            if (
+                name.startswith("ItemsView[")
+                and name.endswith("]")
+                or name.startswith("KeysView[")
+                and name.endswith("]")
+                or name.startswith("ValuesView[")
+                and name.endswith("]")
+            ):
+                return
+
+        super().report_error(error)
+
+    def handle_bases(
+        self, path: QualifiedName, bases: tuple[type, ...]
+    ) -> list[QualifiedName]:
+        result = []
+        for base in super().handle_bases(path, bases):
+            if base[0] == "pybind11_builtins":
+                break
+            result.append(base)
+        return result