Merge pull request #230 from robotpy/update-gen

Update various generation related tooling

Author: virtuald

docs/tools.rst

@@ -15,7 +15,20 @@ your python package.
 
 This will scan all of your defined includes directories (including those of
 downloaded artifacts) and output something you can paste into the ``generate``
-key of ``pyproject.toml``.
+key of ``pyproject.toml``. By default it will only show files that are not
+present in ``pyproject.toml`` -- to show all files use the ``--all`` argument.
+
+Often there are files that you don't want to wrap. You can add them to the
+``pyproject.toml`` file and they will be ignored. The list accepts glob patterns
+supported by the fnmatch module.
+
+.. code-block:: toml
+
+    [tool.robotpy-build]
+    scan_headers_ignore = [
+        "ignored_header.h",
+        "ignore_dir/*",
+    ]
 
 .. _create_gen:
 
@@ -40,3 +53,15 @@ python package.
 .. code-block:: sh
 
     $ python -m robotpy_build create-imports rpydemo rpydemo._rpydemo
+
+Use the ``--write`` argument to write the file.
+
+To write a list of ``__init__.py`` files, you can specify them in the ``pyproject.toml``
+file like so:
+
+.. code-block:: toml
+
+    [tool.robotpy-build]
+    update_init = ["rpydemo rpydemo._rpydemo"]
+
+To actually update the files, run ``python setup.py update_init``.

robotpy_build/autowrap/generator_data.py

@@ -133,7 +133,7 @@ def get_function_data(
                 data = self._default_fn_data
             report_data.deferred_signatures.append((fn, is_private))
         elif not data.overloads:
-            report_data.deferred_signatures.append((fn, True))
+            report_data.deferred_signatures.append((fn, is_private))
         else:
             # When there is overload data present, we have to actually compute
             # the signature of every function

robotpy_build/command/_built_env.py

@@ -0,0 +1,79 @@
+import importlib.util
+import os
+from os.path import abspath, exists, dirname, join
+
+from setuptools import Command
+
+from .util import get_install_root
+
+
+class _BuiltEnv(Command):
+
+    user_options = [("build-lib=", "d", 'directory to "build" (copy) to')]
+
+    def initialize_options(self):
+        self.build_lib = None
+
+    def finalize_options(self):
+        self.set_undefined_options("build", ("build_lib", "build_lib"))
+
+    def setup_built_env(self):
+
+        # Gather information for n
+        data = {"mapping": {}}
+
+        # OSX-specific: need to set DYLD_LIBRARY_PATH otherwise modules don't
+        # work. Luckily, that information was computed when building the
+        # extensions...
+        env = os.environ.copy()
+        dyld_path = set()
+
+        # Requires information from build_ext to work
+        build_ext = self.get_finalized_command("build_ext")
+        if build_ext.inplace:
+            data["out"] = get_install_root(self)
+        else:
+            data["out"] = self.build_lib
+
+        # Ensure that the associated packages can always be found locally
+        for wrapper in build_ext.wrappers:
+            pkgdir = wrapper.package_name.split(".")
+            init_py = abspath(join(self.build_lib, *pkgdir, "__init__.py"))
+            if exists(init_py):
+                data["mapping"][wrapper.package_name] = init_py
+
+        # Ensure that the built extension can always be found
+        build_ext.resolve_libs()
+        for ext in build_ext.extensions:
+            fname = build_ext.get_ext_filename(ext.name)
+            data["mapping"][ext.name] = abspath(join(self.build_lib, fname))
+
+            rpybuild_libs = getattr(ext, "rpybuild_libs", None)
+            if rpybuild_libs:
+                for pth, _ in rpybuild_libs.values():
+                    dyld_path.add(dirname(pth))
+
+        # OSX-specific
+        if dyld_path:
+            dyld_path = ":".join(dyld_path)
+            if "DYLD_LIBRARY_PATH" in env:
+                dyld_path += ":" + env["DYLD_LIBRARY_PATH"]
+            env["DYLD_LIBRARY_PATH"] = dyld_path
+
+        return data, env
+
+
+class _PackageFinder:
+    """
+    Custom loader to allow loading built modules from their location
+    in the build directory (as opposed to their install location)
+    """
+
+    # Set this to mapping returned from _BuiltEnv.setup_built_env
+    mapping = {}
+
+    @classmethod
+    def find_spec(cls, fullname, path, target=None):
+        m = cls.mapping.get(fullname)
+        if m:
+            return importlib.util.spec_from_file_location(fullname, m)

robotpy_build/command/build_ext.py

@@ -158,6 +158,27 @@ def _spawn(cmd):
                 # Used in build_pyi
                 ext.rpybuild_libs = libs
 
+    def resolve_libs(self):
+        # used in _built_env
+        platform = get_platform()
+        if platform.os == "osx":
+            for wrapper in self.wrappers:
+                wrapper.finalize_extension()
+
+            from ..relink_libs import resolve_libs
+
+            install_root = get_install_root(self)
+
+            for ext in self.extensions:
+                libs = resolve_libs(
+                    install_root,
+                    ext.rpybuild_wrapper,
+                    self.rpybuild_pkgcfg,
+                )
+
+                # Used in build_pyi
+                ext.rpybuild_libs = libs
+
     def run(self):
         # files need to be generated before building can occur
         self.run_command("build_gen")

robotpy_build/command/build_pyi.py

@@ -1,35 +1,29 @@
-import importlib.util
 import json
 import os
-from os.path import abspath, exists, dirname, join
+from os.path import exists, dirname, join
 import subprocess
 import sys
 
 import pybind11_stubgen
-from setuptools import Command
-from distutils.errors import DistutilsError
 
-from .util import get_install_root
+try:
+    from setuptools.errors import BaseError
+except ImportError:
+    from distutils.errors import DistutilsError as BaseError
 
+from ._built_env import _BuiltEnv, _PackageFinder
 
-class GeneratePyiError(DistutilsError):
+
+class GeneratePyiError(BaseError):
     pass
 
 
-class BuildPyi(Command):
+class BuildPyi(_BuiltEnv):
     base_package: str
 
     command_name = "build_pyi"
     description = "Generates pyi files from built extensions"
 
-    user_options = [("build-lib=", "d", 'directory to "build" (copy) to')]
-
-    def initialize_options(self):
-        self.build_lib = None
-
-    def finalize_options(self):
-        self.set_undefined_options("build", ("build_lib", "build_lib"))
-
     def run(self):
         # cannot build pyi files when cross-compiling
         if (
@@ -40,50 +34,18 @@ def run(self):
             return
 
         # Gather information for needed stubs
-        data = {"mapping": {}, "stubs": []}
-
-        # OSX-specific: need to set DYLD_LIBRARY_PATH otherwise modules don't
-        # work. Luckily, that information was computed when building the
-        # extensions...
-        env = os.environ.copy()
-        dyld_path = set()
-
-        # Requires information from build_ext to work
-        build_ext = self.distribution.get_command_obj("build_ext")
-        if build_ext.inplace:
-            data["out"] = get_install_root(self)
-        else:
-            data["out"] = self.build_lib
-
-        # Ensure that the associated packages can always be found locally
-        for wrapper in build_ext.wrappers:
-            pkgdir = wrapper.package_name.split(".")
-            init_py = abspath(join(self.build_lib, *pkgdir, "__init__.py"))
-            if exists(init_py):
-                data["mapping"][wrapper.package_name] = init_py
+        data, env = self.setup_built_env()
+        data["stubs"] = []
 
         # Ensure that the built extension can always be found
+        build_ext = self.get_finalized_command("build_ext")
         for ext in build_ext.extensions:
-            fname = build_ext.get_ext_filename(ext.name)
-            data["mapping"][ext.name] = abspath(join(self.build_lib, fname))
             data["stubs"].append(ext.name)
 
-            rpybuild_libs = getattr(ext, "rpybuild_libs", None)
-            if rpybuild_libs:
-                for pth, _ in rpybuild_libs.values():
-                    dyld_path.add(dirname(pth))
-
         # Don't do anything if nothing is needed
         if not data["stubs"]:
             return
 
-        # OSX-specific
-        if dyld_path:
-            dyld_path = ":".join(dyld_path)
-            if "DYLD_LIBRARY_PATH" in env:
-                dyld_path += ":" + env["DYLD_LIBRARY_PATH"]
-            env["DYLD_LIBRARY_PATH"] = dyld_path
-
         data_json = json.dumps(data)
 
         # Execute in a subprocess in case it crashes
@@ -101,45 +63,6 @@ def run(self):
             pass
 
 
-class _PackageFinder:
-    """
-    Custom loader to allow loading built modules from their location
-    in the build directory (as opposed to their install location)
-    """
-
-    mapping = {}
-
-    @classmethod
-    def find_spec(cls, fullname, path, target=None):
-        m = cls.mapping.get(fullname)
-        if m:
-            return importlib.util.spec_from_file_location(fullname, m)
-
-
-def generate_pyi(module_name: str, pyi_filename: str):
-    print("generating", pyi_filename)
-
-    pybind11_stubgen.FunctionSignature.n_invalid_signatures = 0
-    module = pybind11_stubgen.ModuleStubsGenerator(module_name)
-    module.parse()
-    if pybind11_stubgen.FunctionSignature.n_invalid_signatures > 0:
-        print("FAILED to generate pyi for", module_name, file=sys.stderr)
-        return False
-
-    module.write_setup_py = False
-    with open(pyi_filename, "w") as fp:
-        fp.write("#\n# AUTOMATICALLY GENERATED FILE, DO NOT EDIT!\n#\n\n")
-        fp.write("\n".join(module.to_lines()))
-
-    typed = join(dirname(pyi_filename), "py.typed")
-    print("generating", typed)
-    if not exists(typed):
-        with open(typed, "w") as fp:
-            pass
-
-    return True
-
-
 def main():
     cfg = json.load(sys.stdin)
 

robotpy_build/command/update_init.py

@@ -0,0 +1,73 @@
+import json
+import os
+import subprocess
+import sys
+import typing
+
+try:
+    from setuptools.errors import BaseError
+except ImportError:
+    from distutils.errors import DistutilsError as BaseError
+
+from ._built_env import _BuiltEnv, _PackageFinder
+
+
+class UpdateInitError(BaseError):
+    pass
+
+
+class UpdateInit(_BuiltEnv):
+    update_list: typing.List[str]
+
+    command_name = "update_init"
+    description = (
+        "Updates __init__.py files using settings from tool.robotpy-build.update_init"
+    )
+
+    def run(self):
+        # cannot use when cross-compiling
+        if (
+            "_PYTHON_HOST_PLATFORM" in os.environ
+            or "PYTHON_CROSSENV" in os.environ
+            or not self.update_list
+        ):
+            return
+
+        data, env = self.setup_built_env()
+        data["update_list"] = self.update_list
+
+        data_json = json.dumps(data)
+
+        # Execute in a subprocess in case it crashes
+        args = [sys.executable, "-m", __name__]
+        try:
+            subprocess.run(args, input=data_json.encode("utf-8"), env=env, check=True)
+        except subprocess.CalledProcessError:
+            raise UpdateInitError(
+                "Failed to generate .pyi file (see above, or set RPYBUILD_SKIP_PYI=1 to ignore) via %s"
+                % (args,)
+            ) from None
+
+
+def main():
+    cfg = json.load(sys.stdin)
+
+    # Configure custom loader
+    _PackageFinder.mapping = cfg["mapping"]
+    sys.meta_path.insert(0, _PackageFinder)
+
+    from .. import tool
+
+    # Update init
+
+    for to_update in cfg["update_list"]:
+
+        sys.argv = ["<dummy>", "create-imports", "-w"] + to_update.split(" ", 1)
+
+        retval = tool.main()
+        if retval != 0:
+            break
+
+
+if __name__ == "__main__":
+    main()

robotpy_build/config/pyproject_toml.py

@@ -416,6 +416,14 @@ class RobotpyBuildConfig(Model):
     #: Python package to store version information and robotpy-build metadata in
     base_package: str
 
+    #: List of headers for the scan-headers tool to ignore
+    scan_headers_ignore: List[str] = []
+
+    #: List of python packages with __init__.py to update when ``python setup.py update_init``
+    #: is called -- this is an argument to the ``robotpy-build create-imports`` command, and
+    #: may contain a space and the second argument to create-imports.
+    update_init: List[str] = []
+
     #:
     #: .. seealso:: :class:`.SupportedPlatform`
     #: