Improve and fix support for compiling protobuf files (#19)

The main changes are:

- Avoid running sessions on sources for API repos

API repos are supposed to just ship python files generated from protocol
buffer files. Only a stub `__init__.py` should be included, which
doesn't really need to be checked.
  
Allowing checks on the sources for an API repos triggers a lot of issues
when the protocol buffers files are generated, as they don't pass any
checks. An alternative would be to exclude auto-generated files, but
it's not easy because each tool has its own exclusion mechanism, and
some don't even have any (hello `darglint`), so it is simpler to just
skip testing the stub `__init__.py` file.

- Avoid the need to write a setup.py file for API repos

To do this we declare a distutils entry point in the `pyproject.toml`
file to automatically add the `compile_proto` command using the
`CompileProto` proto class.
  
Then, in the `grpc_tools` module we just add the newly declared command
as a `build` sub-command, so it is executed first. This make all other
subsequent sub-commands will see the generated sources as if they were
already there.
  
The only downside of this is now we don't have an easy way to configure
the command. The options are there, and can be passed via command-line,
like:
  
  ```sh
  python -c 'import setuptools; setuptools.setup()' compile_proto \
      --proto-path=another/path
  ```
  
But there is no easy way to do it in a config file, or at least at the
moment I couldn't find a way to pass arbitrary options to setuptools
commands in `pyproject.toml`. If we need this, we could manually parse
the `pyproject.toml` file and look for options, just like
`setuptools_scm` does.

Author: llucax

pyproject.toml

@@ -36,6 +36,9 @@ dynamic = ["version"]
 name = "Frequenz Energy-as-a-Service GmbH"
 email = "[email protected]"
 
+[project.entry-points."distutils.commands"]
+compile_proto = "frequenz.repo.config.setuptools.grpc_tools:CompileProto"
+
 [project.optional-dependencies]
 actor = []
 api = [

src/frequenz/py.typed

@@ -154,6 +154,16 @@
 
 When configuring APIs it is assumed that they have a gRPC interface.
 
+The project structure is assumed to be as follows:
+
+- `proto/`: Directory containing the `.proto` files.
+- `py/`: Directory containing the Python code. It should only provide
+  a `py.typed` file and a `__init__.py` file. API repositories should not
+  contain any other Python code.
+- `pytests/`: Directory containing the tests for the Python code.
+- `submodules/api-common-protos`: Directory containing the submodule with the
+  `google/api-common-protos` repository.
+
 Normally Frequenz APIs use basic types from
 [`google/api-common-protos`](https://github.com/googleapis/api-common-protos),
 so you need to make sure the proper submodule is added to your project:
@@ -164,17 +174,6 @@
 git commit -m "Add Google api-common-protos submodule" submodules/api-common-protos
 ```
 
-Then you need to create a `setup.py` file with the following content:
-
-```py
-import setuptools
-
-from frequenz.repo.config.setuptools import grpc_tools
-
-if __name__ == "__main__":
-    setuptools.setup(cmdclass=grpc_tools.build_proto_cmdclass())
-```
-
 Then you need to add this package as a build dependency and a few extra
 dependencies to your project, for example:
 
@@ -198,22 +197,33 @@
 package. Of course you need to replace the version numbers with the correct
 ones too.
 
+You should also add the following configuration to your `pyproject.toml` file
+to make sure the generated files are included in the wheel:
+
+```toml
+[tool.setuptools.package-dir]
+"" = "py"
+
+[tool.setuptools.package-data]
+"*" = ["*.pyi"]
+
+[tools.pytest.ini_options]
+testpaths = ["pytests"]
+```
+
 Finally you need to make sure to include the generated `*.pyi` files in the
 source distribution, as well as the Google api-common-protos files, as it
 is not handled automatically yet
-([#13](https://github.com/frequenz-floss/frequenz-repo-config-python/issues/13)):
+([#13](https://github.com/frequenz-floss/frequenz-repo-config-python/issues/13)).
+Make sure to include these lines in the `MANIFEST.in` file:
 
 ```
-recursive-include py *.pyi
 recursive-include submodules/api-common-protos/google *.proto
 ```
 
-If you need to customize how the protobuf files are compiled, you can pass
-a path to look for the protobuf files, a glob pattern to find the protobuf
-files, and a list of paths to include when compiling the protobuf files. By
-default `submodules/api-common-protos` is used as an include path, so if
-yout configured the submodules in a different path, you need to pass the
-correct path to the `include_paths` argument.
+For now there is no way to customize where the protocol files are located,
+where the generated files should be placed, or which extra directories must be
+included when compiling the protocol files.
 """
 
 from . import nox, setuptools

src/frequenz/repo/config/__init__.py

@@ -91,7 +91,10 @@
 
 api_config: _config.Config = dataclasses.replace(
     common_config,
-    source_paths=list(util.replace(common_config.source_paths, {"src": "py"})),
+    opts=api_command_options,
+    # We don't check the sources at all because they are automatically generated.
+    source_paths=[],
+    # We adapt the path to the tests.
     extra_paths=list(util.replace(common_config.extra_paths, {"tests": "pytests"})),
 )
 """Default configuration for APIs.

src/frequenz/repo/config/nox/default.py

@@ -10,7 +10,7 @@
 
 import pathlib
 import tomllib
-from collections.abc import Iterable, Mapping, Set
+from collections.abc import Iterable, Mapping
 from typing import TypeVar
 
 _T = TypeVar("_T")
@@ -36,14 +36,13 @@ def replace(iterable: Iterable[_T], replacements: Mapping[_T, _T], /) -> Iterabl
 
     Args:
         iterable: The iterable to replace elements in.
-        old: The elements to replace.
-        new: The elements to replace with.
+        replacements: A mapping of elements to replace with other elements.
 
-    Returns:
-        An iterable with the elements in `iterable` replaced.
+    Yields:
+        The next element in the iterable, with the replacements applied.
 
     Example:
-        >>> assert list(replace([1, 2, 3], old={1, 2}, new={4, 5})) == [4, 5, 3]
+        >>> assert list(replace([1, 2, 3], {1: 4, 2: 5})) == [4, 5, 3]
     """
     for item in iterable:
         if item in replacements:
@@ -210,7 +209,7 @@ def discover_paths() -> list[str]:
     with open("pyproject.toml", "rb") as toml_file:
         data = tomllib.load(toml_file)
 
-    testpaths = (
+    testpaths: list[str] = (
         data.get("tool", {})
         .get("pytest", {})
         .get("ini_options", {})

src/frequenz/repo/config/nox/util.py

@@ -3,101 +3,99 @@
 
 """Setuptool hooks to build protobuf files.
 
-This module provides a function that returns a dictionary with the required
-machinery to build protobuf files via setuptools.
+This module contains a setuptools command that can be used to compile protocol
+buffer files in a project.
+
+It also runs the command as the first sub-command for the build command, so
+protocol buffer files are compiled automatically before the project is built.
 """
 
 import pathlib
 import subprocess
 import sys
-from collections.abc import Iterable
-from typing import Any
 
 import setuptools
-import setuptools.command.build_py
-
-
-def build_proto_cmdclass(
-    *,
-    proto_path: str = "proto",
-    proto_glob: str = "*.proto",
-    include_paths: Iterable[str] = ("submodules/api-common-protos",),
-) -> dict[str, Any]:
-    """Return a dictionary with the required machinery to build protobuf files.
-
-    This dictionary is meant to be passed as the `cmdclass` argument of
-    `setuptools.setup()`.
-
-    It will add the following commands to setuptools:
-
-        - `compile_proto`: Adds a command to compile the protobuf files to
-          Python files.
-        - `build_py`: Use the `compile_proto` command to build the python files
-          and run the regular `build_py` command, so the protobuf files are
-          create automatically when the python package is created.
-
-    Unless an explicit `include_paths` is passed, the
-    `submodules/api-common-protos` wiil be added to the include paths, so your
-    project should have a submodule with the common google api protos in that
-    path.
-
-    Args:
-        proto_path: Path of the root directory containing the protobuf files.
-        proto_glob: The glob pattern to use to find the protobuf files.
-        include_paths: Paths to include when compiling the protobuf files.
-
-    Returns:
-        Options to pass to `setuptools.setup()` `cmdclass` argument to build
-        protobuf files.
-    """
-
-    class CompileProto(setuptools.Command):
-        """Build the Python protobuf files."""
-
-        description: str = f"compile protobuf files in {proto_path}/**/{proto_glob}/"
-        """Description of the command."""
-
-        user_options: list[str] = []
-        """Options of the command."""
-
-        def initialize_options(self) -> None:
-            """Initialize options."""
-
-        def finalize_options(self) -> None:
-            """Finalize options."""
-
-        def run(self) -> None:
-            """Compile the Python protobuf files."""
-            proto_files = [str(p) for p in pathlib.Path(proto_path).rglob(proto_glob)]
-            protoc_cmd = (
-                [sys.executable, "-m", "grpc_tools.protoc"]
-                + [f"-I{p}" for p in [*include_paths, proto_path]]
-                + """--python_out=py
-                    --grpc_python_out=py
-                    --mypy_out=py
-                    --mypy_grpc_out=py
-                    """.split()
-                + proto_files
-            )
-            print(f"Compiling proto files via: {' '.join(protoc_cmd)}")
-            subprocess.run(protoc_cmd, check=True)
-
-    class BuildPy(setuptools.command.build_py.build_py, CompileProto):
-        """Build the Python protobuf files and run the regular `build_py` command."""
-
-        def run(self) -> None:
-            """Compile the Python protobuf files and run regular `build_py`."""
-            CompileProto.run(self)
-            setuptools.command.build_py.build_py.run(self)
-
-    return {
-        "compile_proto": CompileProto,
-        # Compile the proto files to python files. This is done when building
-        # the wheel, the source distribution (sdist) contains the *.proto files
-        # only. Check the MANIFEST.in file to see which files are included in
-        # the sdist, and the tool.setuptools.package-dir,
-        # tool.setuptools.package-data, and tools.setuptools.packages
-        # configuration keys in pyproject.toml to see which files are included
-        # in the wheel package.
-        "build_py": BuildPy,
-    }
+
+# The typing stub for this module is missing
+import setuptools.command.build  # type: ignore[import]
+
+
+class CompileProto(setuptools.Command):
+    """Build the Python protobuf files."""
+
+    proto_path: str
+    """The path of the root directory containing the protobuf files."""
+
+    proto_glob: str
+    """The glob pattern to use to find the protobuf files."""
+
+    include_paths: str
+    """Comma-separated list of paths to include when compiling the protobuf files."""
+
+    py_path: str
+    """The path of the root directory where the Python files will be generated."""
+
+    description: str = "compile protobuf files"
+    """Description of the command."""
+
+    user_options: list[tuple[str, str | None, str]] = [
+        (
+            "proto-path=",
+            None,
+            "path of the root directory containing the protobuf files",
+        ),
+        ("proto-glob=", None, "glob pattern to use to find the protobuf files"),
+        (
+            "include-paths=",
+            None,
+            "comma-separated list of paths to include when compiling the protobuf files",
+        ),
+        (
+            "py-path=",
+            None,
+            "path of the root directory where the Python files will be generated",
+        ),
+    ]
+    """Options of the command."""
+
+    def initialize_options(self) -> None:
+        """Initialize options."""
+        self.proto_path = "proto"
+        self.proto_glob = "*.proto"
+        self.include_paths = "submodules/api-common-protos"
+        self.py_path = "py"
+
+    def finalize_options(self) -> None:
+        """Finalize options."""
+
+    def run(self) -> None:
+        """Compile the Python protobuf files."""
+        include_paths = self.include_paths.split(",")
+        proto_files = [
+            str(p) for p in pathlib.Path(self.proto_path).rglob(self.proto_glob)
+        ]
+
+        if not proto_files:
+            print(f"No proto files found in {self.proto_path}/**/{self.proto_glob}/")
+            return
+
+        protoc_cmd = (
+            [sys.executable, "-m", "grpc_tools.protoc"]
+            + [f"-I{p}" for p in [*include_paths, self.proto_path]]
+            + [
+                f"--{opt}={self.py_path}"
+                for opt in "python_out grpc_python_out mypy_out mypy_grpc_out".split()
+            ]
+            + proto_files
+        )
+
+        print(f"Compiling proto files via: {' '.join(protoc_cmd)}")
+        subprocess.run(protoc_cmd, check=True)
+
+
+# This adds the compile_proto command to the build sub-command.
+# The name of the command is mapped to the class name in the pyproject.toml file,
+# in the [project.entry-points.distutils.commands] section.
+# The None value is an optional function that can be used to determine if the
+# sub-command should be executed or not.
+setuptools.command.build.build.sub_commands.insert(0, ("compile_proto", None))