Improve build tools and update pymctranslate (#18)

* Improve build tools

* Fix mypy warnings

Author: gentlegiantJGC

build_requires.py

@@ -1,3 +1,5 @@
+# mypy: disable-error-code=no-redef
+
 from typing import Union, Mapping
 
 from setuptools import build_meta

mypy.ini

@@ -3,5 +3,13 @@ disallow_untyped_defs = True
 check_untyped_defs = True
 warn_return_any = True
 python_version = 3.12
+explicit_package_bases = True
 mypy_path = $MYPY_CONFIG_FILE_DIR/src,$MYPY_CONFIG_FILE_DIR/tests
-packages = amulet,test_amulet_game
+files = 
+    src,
+    tests,
+    tools,
+    get_compiler,
+    build_requires.py,
+    requirements.py,
+    setup.py

pyproject.toml

@@ -23,7 +23,9 @@ classifiers = [
 [project.optional-dependencies]
 dev = [
     "setuptools>=42",
+    "types-setuptools",
     "versioneer",
+    "types-versioneer",
     "packaging",
     "wheel",
     "pybind11_stubgen>=2.5.4",

setup.py

@@ -8,6 +8,7 @@
 import gzip
 import pickle
 from tempfile import TemporaryDirectory
+from typing import TypeAlias, TYPE_CHECKING
 
 from setuptools import setup, Extension, Command
 from setuptools.command.build import build
@@ -18,15 +19,20 @@
 import requirements
 
 
-def fix_path(path: str) -> str:
+def fix_path(path: str | os.PathLike[str]) -> str:
     return os.path.realpath(path).replace(os.sep, "/")
 
 
 cmdclass: dict[str, type[Command]] = versioneer.get_cmdclass()
 
+if TYPE_CHECKING:
+    BuildExt: TypeAlias = build_ext
+else:
+    BuildExt = cmdclass.get("build_ext", build_ext)
 
-class CMakeBuild(cmdclass.get("build_ext", build_ext)):
-    def build_extension(self, ext):
+
+class CMakeBuild(BuildExt):
+    def build_extension(self, ext: Extension) -> None:
         import pybind11
         import amulet.pybind11_extensions
         import amulet.io
@@ -88,18 +94,19 @@ def build_extension(self, ext):
 
 
 class MinifyJSON(Command):
-    def initialize_options(self):
+    def initialize_options(self) -> None:
         self.editable_mode = False
-        self.build_lib = None
+        self.build_lib: str | None = None
 
-    def finalize_options(self):
+    def finalize_options(self) -> None:
         self.set_undefined_options("build_py", ("build_lib", "build_lib"))
 
-    def run(self):
+    def run(self) -> None:
         # This is rather janky but it is a stop-gap until the whole library can be ported to C++
         if self.editable_mode:
             src_dir = os.path.abspath("src")
         else:
+            assert self.build_lib is not None
             src_dir = self.build_lib
 
         sys.path.append(src_dir)
@@ -150,7 +157,7 @@ def run(self):
 cmdclass.get("build", build).sub_commands.append(("minify_json", None))
 
 
-cmdclass["build_ext"] = CMakeBuild
+cmdclass["build_ext"] = CMakeBuild  # type: ignore
 
 
 setup(

src/amulet/game/abc/__init__.pyi

@@ -60,8 +60,8 @@ __all__: list[str] = [
     "load_json_block_spec",
     "version",
 ]
-JSONCompatible: (
-    typing._UnionGenericAlias
-)  # value = typing.Union[str, int, float, bool, NoneType, ForwardRef('JSONList'), ForwardRef('JSONDict')]
-JSONDict: typing.TypeAlias = dict[str, "JSONCompatible"]
-JSONList: typing.TypeAlias = list["JSONCompatible"]
+JSONCompatible: typing.TypeAlias = typing.Union[
+    str, int, float, bool, None, JSONList, JSONDict
+]
+JSONDict: typing.TypeAlias = dict[str, JSONCompatible]
+JSONList: typing.TypeAlias = list[JSONCompatible]

src/amulet/game/translate/_functions/walk_input_nbt.py

@@ -274,7 +274,7 @@ def run(self, src: SrcData, state: StateData, dst: DstData) -> None:
                 else:
                     raise TypeError
 
-                for i, value in enumerate(tag):
+                for i, arr_value in enumerate(tag):
                     if self._index is not None and i in self._index:
                         outer_name, outer_type, path = nbt_path
                         new_path = path + ((i, nested_dtype),)

submodules/PyMCTranslate

@@ -19,7 +19,7 @@ def fix_path(path: str) -> str:
 RootDir = fix_path(os.path.dirname(os.path.dirname(__file__)))
 
 
-def main():
+def main() -> None:
     platform_args = []
     if sys.platform == "win32":
         platform_args.extend(["-G", "Visual Studio 17 2022"])

tools/cmake_generate.py

@@ -7,16 +7,28 @@
 import pybind11_stubgen
 from pybind11_stubgen.structs import Identifier
 from pybind11_stubgen.parser.mixins.filter import FilterClassMembers
-from pybind11_stubgen import main as pybind11_stubgen_main
+
+
+ForwardRefPattern = re.compile(r"ForwardRef\('(?P<variable>[a-zA-Z_][a-zA-Z0-9_]*)'\)")
+
+QuotePattern = re.compile(r"'(?P<variable>[a-zA-Z_][a-zA-Z0-9_]*)'")
+
+
+def fix_value(value: str) -> str:
+    value = value.replace("NoneType", "None")
+    value = ForwardRefPattern.sub(lambda match: match.group("variable"), value)
+    value = QuotePattern.sub(lambda match: match.group("variable"), value)
+    return value
+
 
 UnionPattern = re.compile(
-    r"^(?P<variable>[a-zA-Z_][a-zA-Z0-9_]*): types\.UnionType\s*#\s*value = (?P<value>.*)$",
+    r"^(?P<variable>[a-zA-Z_][a-zA-Z0-9_]*): (types\.UnionType|typing\._UnionGenericAlias)\s*#\s*value = (?P<value>.*)$",
     flags=re.MULTILINE,
 )
 
 
-def union_sub_func(match: re.Match) -> str:
-    return f'{match.group("variable")}: typing.TypeAlias = {match.group("value")}'
+def union_sub_func(match: re.Match[str]) -> str:
+    return f'{match.group("variable")}: typing.TypeAlias = {fix_value(match.group("value"))}'
 
 
 ClassVarUnionPattern = re.compile(
@@ -26,7 +38,7 @@ def union_sub_func(match: re.Match) -> str:
 
 
 def class_var_union_sub_func(match: re.Match) -> str:
-    return f'{match.group("variable")}: typing.ClassVar[typing.TypeAlias] = {match.group("value")}'
+    return f'{match.group("variable")}: typing.TypeAlias = {fix_value(match.group("value"))}'
 
 
 VersionPattern = re.compile(r"(?P<var>[a-zA-Z0-9_].*): str = '.*?'")
@@ -36,13 +48,20 @@ def str_sub_func(match: re.Match) -> str:
     return f"{match.group('var')}: str"
 
 
+CompilerConfigPattern = re.compile(r"compiler_config: dict.*")
+
+
+def compiler_config_sub_func(match: re.Match) -> str:
+    return "compiler_config: dict"
+
+
 EqPattern = re.compile(
     r"(?P<indent>[ \t]+)def __eq__\(self, arg0: (?P<other>[a-zA-Z1-9.]+)\) -> (?P<return>[a-zA-Z1-9.]+):"
     r"(?P<ellipsis_docstring>\s*((\.\.\.)|(\"\"\"(.|\n)*?\"\"\")))"
 )
 
 
-def eq_sub_func(match: re.Match) -> str:
+def eq_sub_func(match: re.Match[str]) -> str:
     """
     if one - add @overload and overloaded signature
 
@@ -80,7 +99,7 @@ def eq_sub_func(match: re.Match) -> str:
 
 
 def generic_alias_sub_func(match: re.Match) -> str:
-    return f"{match.group('variable')}: typing.TypeAlias = {match.group('value')}"
+    return f'{match.group("variable")}: typing.TypeAlias = {fix_value(match.group("value"))}'
 
 
 def get_module_path(name: str) -> str:
@@ -95,72 +114,37 @@ def get_package_dir(name: str) -> str:
     return os.path.realpath(os.path.dirname(get_module_path(name)))
 
 
-def patch_stubgen():
+def patch_stubgen() -> None:
+    class_member_blacklist: set[Identifier] = FilterClassMembers._FilterClassMembers__class_member_blacklist  # type: ignore
+    attribute_blacklist: set[Identifier] = FilterClassMembers._FilterClassMembers__attribute_blacklist  # type: ignore
+
     # Is there a better way to add items to the blacklist?
     # Pybind11
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("_pybind11_conduit_v1_")
-    )
+    class_member_blacklist.add(Identifier("_pybind11_conduit_v1_"))
     # Python
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__new__")
-    )
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__subclasshook__")
-    )
+    class_member_blacklist.add(Identifier("__new__"))
+    class_member_blacklist.add(Identifier("__subclasshook__"))
     # Pickle
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__getnewargs__")
-    )
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__getstate__")
-    )
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__setstate__")
-    )
+    class_member_blacklist.add(Identifier("__getnewargs__"))
+    class_member_blacklist.add(Identifier("__getstate__"))
+    class_member_blacklist.add(Identifier("__setstate__"))
     # ABC
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__abstractmethods__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__orig_bases__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__parameters__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("_abc_impl")
-    )
+    attribute_blacklist.add(Identifier("__abstractmethods__"))
+    attribute_blacklist.add(Identifier("__orig_bases__"))
+    attribute_blacklist.add(Identifier("__parameters__"))
+    attribute_blacklist.add(Identifier("_abc_impl"))
     # Protocol
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__protocol_attrs__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__non_callable_proto_members__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("_is_protocol")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("_is_runtime_protocol")
-    )
+    attribute_blacklist.add(Identifier("__protocol_attrs__"))
+    attribute_blacklist.add(Identifier("__non_callable_proto_members__"))
+    attribute_blacklist.add(Identifier("_is_protocol"))
+    attribute_blacklist.add(Identifier("_is_runtime_protocol"))
     # dataclass
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__dataclass_fields__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__dataclass_params__")
-    )
-    FilterClassMembers._FilterClassMembers__attribute_blacklist.add(
-        Identifier("__match_args__")
-    )
+    attribute_blacklist.add(Identifier("__dataclass_fields__"))
+    attribute_blacklist.add(Identifier("__dataclass_params__"))
+    attribute_blacklist.add(Identifier("__match_args__"))
     # Buffer protocol
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__buffer__")
-    )
-    FilterClassMembers._FilterClassMembers__class_member_blacklist.add(
-        Identifier("__release_buffer__")
-    )
+    class_member_blacklist.add(Identifier("__buffer__"))
+    class_member_blacklist.add(Identifier("__release_buffer__"))
 
 
 def main() -> None:
@@ -236,6 +220,7 @@ def main() -> None:
         pyi = UnionPattern.sub(union_sub_func, pyi)
         pyi = ClassVarUnionPattern.sub(class_var_union_sub_func, pyi)
         pyi = VersionPattern.sub(str_sub_func, pyi)
+        pyi = CompilerConfigPattern.sub(compiler_config_sub_func, pyi)
         pyi = GenericAliasPattern.sub(generic_alias_sub_func, pyi)
         pyi = pyi.replace(
             "__hash__: typing.ClassVar[None] = None",