Improve build tools and fix mypy warnings (#48)

* Improve build tools

* Update stubs

* Fix mypy warnings

Author: gentlegiantJGC

build_requires.py

@@ -1,3 +1,5 @@
+# mypy: disable-error-code=no-redef
+
 from typing import Union, Mapping
 
 from setuptools import build_meta

mypy.ini

@@ -2,6 +2,14 @@
 disallow_untyped_defs = True
 check_untyped_defs = True
 warn_return_any = True
-python_version = 3.11
-mypy_path = $MYPY_CONFIG_FILE_DIR/src
-packages = amulet
+python_version = 3.12
+explicit_package_bases = True
+mypy_path = $MYPY_CONFIG_FILE_DIR/src,$MYPY_CONFIG_FILE_DIR/tests
+files = 
+    src,
+    tests,
+    tools,
+    get_compiler,
+    build_requires.py,
+    requirements.py,
+    setup.py

pyproject.toml

@@ -23,7 +23,9 @@ classifiers = [
 [project.optional-dependencies]
 dev = [
     "setuptools>=42",
+    "types-setuptools",
     "versioneer",
+    "types-versioneer",
     "packaging",
     "wheel",
     "pybind11_stubgen>=2.5.4",

setup.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 import platform
 from tempfile import TemporaryDirectory
+from typing import TypeAlias, TYPE_CHECKING
 
 from setuptools import setup, Extension, Command
 from setuptools.command.build_ext import build_ext
@@ -13,15 +14,20 @@
 import requirements
 
 
-def fix_path(path: str) -> str:
+def fix_path(path: str | os.PathLike[str]) -> str:
     return os.path.realpath(path).replace(os.sep, "/")
 
 
 cmdclass: dict[str, type[Command]] = versioneer.get_cmdclass()
 
+if TYPE_CHECKING:
+    BuildExt: TypeAlias = build_ext
+else:
+    BuildExt = cmdclass.get("build_ext", build_ext)
 
-class CMakeBuild(cmdclass.get("build_ext", build_ext)):
-    def build_extension(self, ext):
+
+class CMakeBuild(BuildExt):
+    def build_extension(self, ext: Extension) -> None:
         import pybind11
         import amulet.pybind11_extensions
 
@@ -74,7 +80,7 @@ def build_extension(self, ext):
                 raise RuntimeError("Error installing amulet-leveldb")
 
 
-cmdclass["build_ext"] = CMakeBuild
+cmdclass["build_ext"] = CMakeBuild  # type: ignore
 
 
 setup(

src/amulet/leveldb/_leveldb.pyi

@@ -2,6 +2,6 @@ from __future__ import annotations
 
 import types
 
-__all__ = ["init"]
+__all__: list[str] = ["init"]
 
 def init(arg0: types.ModuleType) -> None: ...

tests/test_amulet_leveldb/__init__.pyi

@@ -4,7 +4,7 @@ import faulthandler as _faulthandler
 
 from . import _test_amulet_leveldb
 
-__all__ = ["compiler_config"]
+__all__: list[str] = ["compiler_config"]
 
 def _init() -> None: ...
 

tests/test_amulet_leveldb/_test_amulet_leveldb.pyi

@@ -2,6 +2,6 @@ from __future__ import annotations
 
 import types
 
-__all__ = ["init"]
+__all__: list[str] = ["init"]
 
 def init(arg0: types.ModuleType) -> None: ...

tests/test_db.py

@@ -7,6 +7,7 @@
 import weakref
 import time
 from concurrent.futures import ThreadPoolExecutor
+from collections.abc import Iterable, Sequence
 
 from amulet.leveldb import LevelDB, LevelDBException
 
@@ -16,7 +17,7 @@
 full_db = {**incr_db, **num_db}
 
 
-def get_directory_size(path: str):
+def get_directory_size(path: str) -> int:
     return sum(
         (os.path.getsize(item.path) for item in os.scandir(path) if item.is_file())
     )
@@ -443,7 +444,7 @@ def test_thread_write(self) -> None:
             t2 = -time.time()
             db = LevelDB(path, True)
 
-            def add(values) -> None:
+            def add(values: Iterable[bytes]) -> None:
                 for v in values:
                     db.put(v, v)
 
@@ -469,7 +470,7 @@ def test_thread_read(self) -> None:
         with TemporaryDirectory() as path:
             db = LevelDB(path, True)
 
-            def add(values) -> None:
+            def add(values: Iterable[bytes]) -> None:
                 for v in values:
                     db.put(v, v)
 
@@ -502,7 +503,7 @@ def add(values) -> None:
             m2 = {}
             t2 = -time.time()
 
-            def read(values) -> None:
+            def read(values: Sequence[bytes]) -> None:
                 it = db.create_iterator()
                 it.seek(values[0])
                 for _ in range(count):

tools/cmake_generate.py

@@ -14,7 +14,7 @@ def fix_path(path: str) -> str:
 RootDir = fix_path(os.path.dirname(os.path.dirname(__file__)))
 
 
-def main():
+def main() -> None:
     platform_args = []
     if sys.platform == "win32":
         platform_args.extend(["-G", "Visual Studio 17 2022"])

tools/generate_pybind_stubs.py

@@ -7,16 +7,28 @@
 import pybind11_stubgen
 from pybind11_stubgen.structs import Identifier
 from pybind11_stubgen.parser.mixins.filter import FilterClassMembers
-from pybind11_stubgen import main as pybind11_stubgen_main
+
+
+ForwardRefPattern = re.compile(r"ForwardRef\('(?P<variable>[a-zA-Z_][a-zA-Z0-9_]*)'\)")
+
+QuotePattern = re.compile(r"'(?P<variable>[a-zA-Z_][a-zA-Z0-9_]*)'")
+
+
+def fix_value(value: str) -> str:
+    value = value.replace("NoneType", "None")
+    value = ForwardRefPattern.sub(lambda match: match.group("variable"), value)
+    value = QuotePattern.sub(lambda match: match.group("variable"), value)
+    return value
+
 
 UnionPattern = re.compile(
-    r"^(?P<variable>[a-zA-Z_][a-zA-Z0-9_]*): types\.UnionType\s*#\s*value = (?P<value>.*)$",
+    r"^(?P<variable>[a-zA-Z_][a-zA-Z0-9_]*): (types\.UnionType|typing\._UnionGenericAlias)\s*#\s*value = (?P<value>.*)$",
     flags=re.MULTILINE,
 )
 
 
-def union_sub_func(match: re.Match) -> str:
-    return f'{match.group("variable")}: typing.TypeAlias = {match.group("value")}'
+def union_sub_func(match: re.Match[str]) -> str:
+    return f'{match.group("variable")}: typing.TypeAlias = {fix_value(match.group("value"))}'
 
 
 ClassVarUnionPattern = re.compile(
@@ -26,7 +38,7 @@ def union_sub_func(match: re.Match) -> str:
 
 
 def class_var_union_sub_func(match: re.Match) -> str:
-    return f'{match.group("variable")}: typing.ClassVar[typing.TypeAlias] = {match.group("value")}'
+    return f'{match.group("variable")}: typing.TypeAlias = {fix_value(match.group("value"))}'
 
 
 VersionPattern = re.compile(r"(?P<var>[a-zA-Z0-9_].*): str = '.*?'")
@@ -36,13 +48,20 @@ def str_sub_func(match: re.Match) -> str:
     return f"{match.group('var')}: str"
 
 
+CompilerConfigPattern = re.compile(r"compiler_config: dict.*")
+
+
+def compiler_config_sub_func(match: re.Match) -> str:
+    return "compiler_config: dict"
+
+
 EqPattern = re.compile(
     r"(?P<indent>[ \t]+)def __eq__\(self, arg0: (?P<other>[a-zA-Z1-9.]+)\) -> (?P<return>[a-zA-Z1-9.]+):"
     r"(?P<ellipsis_docstring>\s*((\.\.\.)|(\"\"\"(.|\n)*?\"\"\")))"
 )
 
 
-def eq_sub_func(match: re.Match) -> str:
+def eq_sub_func(match: re.Match[str]) -> str:
     """
     if one - add @overload and overloaded signature
 
@@ -80,7 +99,7 @@ def eq_sub_func(match: re.Match) -> str:
 
 
 def generic_alias_sub_func(match: re.Match) -> str:
-    return f"{match.group('variable')}: typing.TypeAlias = {match.group('value')}"
+    return f'{match.group("variable")}: typing.TypeAlias = {fix_value(match.group("value"))}'
 
 
 def get_module_path(name: str) -> str:
@@ -95,72 +114,37 @@ def get_package_dir(name: str) -> str:
     return os.path.realpath(os.path.dirname(get_module_path(name)))
 
 
-def patch_stubgen():
+def patch_stubgen() -> None:
+    class_member_blacklist: set[Identifier] = FilterClassMembers._FilterClassMembers__class_member_blacklist  # type: ignore
+    attribute_blacklist: set[Identifier] = FilterClassMembers._FilterClassMembers__attribute_blacklist  # type: ignore
+
     # Is there a better way to add items to the blacklist?
     # Pybind11
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("_pybind11_conduit_v1_")
-    )
+    class_member_blacklist.add(Identifier("_pybind11_conduit_v1_"))
     # Python
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__new__")
-    )
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__subclasshook__")
-    )
+    class_member_blacklist.add(Identifier("__new__"))
+    class_member_blacklist.add(Identifier("__subclasshook__"))
     # Pickle
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__getnewargs__")
-    )
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__getstate__")
-    )
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__setstate__")
-    )
+    class_member_blacklist.add(Identifier("__getnewargs__"))
+    class_member_blacklist.add(Identifier("__getstate__"))
+    class_member_blacklist.add(Identifier("__setstate__"))
     # ABC
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__abstractmethods__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__orig_bases__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__parameters__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("_abc_impl")
-    )
+    attribute_blacklist.add(Identifier("__abstractmethods__"))
+    attribute_blacklist.add(Identifier("__orig_bases__"))
+    attribute_blacklist.add(Identifier("__parameters__"))
+    attribute_blacklist.add(Identifier("_abc_impl"))
     # Protocol
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__protocol_attrs__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__non_callable_proto_members__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("_is_protocol")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("_is_runtime_protocol")
-    )
+    attribute_blacklist.add(Identifier("__protocol_attrs__"))
+    attribute_blacklist.add(Identifier("__non_callable_proto_members__"))
+    attribute_blacklist.add(Identifier("_is_protocol"))
+    attribute_blacklist.add(Identifier("_is_runtime_protocol"))
     # dataclass
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__dataclass_fields__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__dataclass_params__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__match_args__")
-    )
+    attribute_blacklist.add(Identifier("__dataclass_fields__"))
+    attribute_blacklist.add(Identifier("__dataclass_params__"))
+    attribute_blacklist.add(Identifier("__match_args__"))
     # Buffer protocol
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__buffer__")
-    )
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__release_buffer__")
-    )
+    class_member_blacklist.add(Identifier("__buffer__"))
+    class_member_blacklist.add(Identifier("__release_buffer__"))
 
 
 def main() -> None:
@@ -236,6 +220,7 @@ def main() -> None:
         pyi = UnionPattern.sub(union_sub_func, pyi)
         pyi = ClassVarUnionPattern.sub(class_var_union_sub_func, pyi)
         pyi = VersionPattern.sub(str_sub_func, pyi)
+        pyi = CompilerConfigPattern.sub(compiler_config_sub_func, pyi)
         pyi = GenericAliasPattern.sub(generic_alias_sub_func, pyi)
         pyi = pyi.replace(
             "__hash__: typing.ClassVar[None] = None",
@@ -244,13 +229,12 @@ def main() -> None:
         pyi = EqPattern.sub(eq_sub_func, pyi)
         pyi = pyi.replace("**kwargs)", "**kwargs: typing.Any)")
         pyi_split = [l.rstrip("\r") for l in pyi.split("\n")]
-        for hidden_import in []:
+        for hidden_import in ["typing", "types"]:
             if hidden_import in pyi and f"import {hidden_import}" not in pyi_split:
-                pyi_split.insert(2, f"import {hidden_import}")
-        if "import typing" not in pyi_split:
-            pyi_split.insert(2, "import typing")
-        if "import types" not in pyi_split:
-            pyi_split.insert(2, "import types")
+                pyi_split.insert(
+                    pyi_split.index("from __future__ import annotations") + 1,
+                    f"import {hidden_import}",
+                )
         pyi = "\n".join(pyi_split)
         with open(stub_path, "w", encoding="utf-8") as f:
             f.write(pyi)