Unify protobuf generation configuration

A new module `protobuf` is added to handle the common `protobuf`
generation documentation which is used by both the `setuptools` plugin
to generate the Python files and the the `mkdocs` function to generate
the API documentation pages.

The `pyproject.toml` configuration key is changed from
`tool.frequenz-repo-config.setuptools.grpc_tools` to
`tool.frequenz-repo-config.protobuf` as it is more generic now.

Signed-off-by: Leandro Lucarella <[email protected]>

Author: llucax

src/frequenz/repo/config/__init__.py

@@ -205,32 +205,62 @@
         - path/to/my/custom/script.py
 ```
 
-If your project also provides *protobuf* files, you can also generate the API
-documentation for them adding one more line to the previous script:
+# APIs
+
+## Protobuf configuation
+
+Support is provided to generate files from *protobuf* files.  To do this, it is possible
+to configure the options to use while generating the files for different purposes
+(language bindings, documentation, etc.).
+
+The configuration can be done in the `pyproject.toml` file as follows:
+
+```toml
+[tool.frequenz_repo_config.protobuf]
+# Location of the proto files relative to the root of the repository (default: "proto")
+proto_path = "proto_files"
+# Glob pattern to use to find the proto files in the proto_path (default: "*.proto")
+proto_glob = "*.prt"  # Default: "*.proto"
+# List of paths to pass to the protoc compiler as include paths (default:
+# ["submodules/api-common-protos", "submodules/frequenz-api-common/proto"])
+include_paths = ["submodules/api-common-protos"]
+# Path where to generate the Python files (default: "py")
+py_path = "generated"
+# Path where to generate the documentation files (default: "protobuf-reference")
+docs_path = "API"
+```
+
+If the defaults are not suitable for you (for example you need to use more or less
+submodules or your proto files are located somewhere else), please adjust the
+configuration to match your project structure.
+
+### `mkdocs` API reference generation
+
+If your project provides *protobuf* files, you can also generate the API
+documentation for them adding one more line to the script provided in the common
+section:
 
 ```python
 from frequenz.repo.config import mkdocs
 
 mkdocs.generate_python_api_pages("my_sources", "API-py")
-mkdocs.generate_protobuf_api_pages("my_protos", "API-proto")
+mkdocs.generate_protobuf_api_pages()
 ```
 
-# APIs
+This will use the configuration in the `pyproject.toml` file and requires `docker` to
+run (it uses the `pseudomuto/protoc-gen-doc` docker image.
 
-## `setuptools` gRPC support
+### `setuptools` gRPC support
 
 When configuring APIs it is assumed that they have a gRPC interface.
 
-The project structure is assumed to be as follows:
+The project structure is assumed to be as described in the *Protobuf configuration*
+section plus the following:
 
-- `proto/`: Directory containing the `.proto` files.
-- `py/`: Directory containing the Python code. It should only provide
-  a `py.typed` file and a `__init__.py` file. API repositories should not
-  contain any other Python code.
 - `pytests/`: Directory containing the tests for the Python code.
-- `submodules/api-common-protos`: Directory containing the submodule with the
+- `submodules/api-common-protos`: Directory containing the Git submodule with the
   `google/api-common-protos` repository.
-- `submodules/frequenz-api-common`: Directory containing the submodule with the
+- `submodules/frequenz-api-common`: Directory containing the Git submodule with the
   `frequenz-floss/frequenz-api-common` repository.
 
 Normally Frequenz APIs use basic types from
@@ -296,24 +326,6 @@
 recursive-include submodules/frequenz-api-common/proto *.proto
 ```
 
-If the defaults are not suitable for you (for example you need to use more or less
-submodules or your proto files are located somewhere else, you can customize how
-the protocol files are generated by adding the following section to your
-`pyproject.toml` file:
-
-```toml
-[tool.frequenz_repo_config.setuptools.grpc_tools]
-# Location of the proto files relative to the root of the repository (default: "proto")
-proto_path = "proto_files"
-# Glob pattern to use to find the proto files in the proto_path (default: "*.proto")
-proto_glob = "*.prt"  # Default: "*.proto"
-# List of paths to pass to the protoc compiler as include paths (default:
-# ["submodules/api-common-protos", "submodules/frequenz-api-common/proto"])
-include_paths = ["submodules/api-common-protos"]
-# Path where to generate the Python files (default: "py")
-py_path = "generated"
-```
-
 Please adapt the instructions above to your project structure if you need to change the
 defaults.
 """

src/frequenz/repo/config/mkdocs.py

@@ -19,6 +19,8 @@
 
 import mkdocs_gen_files
 
+from . import protobuf as _protobuf
+
 
 def _is_internal(path_parts: Tuple[str, ...]) -> bool:
     """Tell if the path is internal judging by the parts.
@@ -99,19 +101,21 @@ def generate_protobuf_api_pages(
     # type ignore because mkdocs_gen_files uses a very weird module-level
     # __getattr__() which messes up the type system
     nav = mkdocs_gen_files.Nav()  # type: ignore
+    config = _protobuf.ProtobufConfig.from_pyproject_toml(
+        proto_path=src_path, docs_path=dst_path
+    )
 
     cwd = Path.cwd()
 
     with tempfile.TemporaryDirectory(prefix="mkdocs-protobuf-reference-") as tmp_path:
-        for path in sorted(Path(src_path).rglob("*.proto")):
-            doc_path = path.relative_to(src_path).with_suffix(".md")
-            full_doc_path = Path(dst_path, doc_path)
-            parts = tuple(path.relative_to(src_path).parts)
+        for path in sorted(Path(config.proto_path).rglob("*.proto")):
+            doc_path = path.relative_to(config.proto_path).with_suffix(".md")
+            full_doc_path = Path(config.docs_path, doc_path)
+            parts = tuple(path.relative_to(config.proto_path).parts)
             nav[parts] = doc_path.as_posix()
             doc_tmp_path = tmp_path / doc_path
             doc_tmp_path.parent.mkdir(parents=True, exist_ok=True)
             try:
-                # TODO: Get arguments from setuptools.grpc
                 subprocess.run(
                     [
                         "docker",
@@ -120,9 +124,8 @@ def generate_protobuf_api_pages(
                         f"-v{cwd}:{cwd}",
                         f"-v{tmp_path}:{tmp_path}",
                         "pseudomuto/protoc-gen-doc",
-                        f"-I{cwd / src_path}",
-                        f"-I{cwd}/submodules/api-common-protos",
-                        f"-I{cwd}/submodules/frequenz-api-common/proto",
+                        f"-I{cwd / config.proto_path}",
+                        *(f"-I{cwd / p}" for p in config.include_paths),
                         f"--doc_opt=markdown,{doc_path.name}",
                         f"--doc_out={tmp_path / doc_path.parent}",
                         str(cwd / path),
@@ -139,5 +142,5 @@ def generate_protobuf_api_pages(
 
             mkdocs_gen_files.set_edit_path(full_doc_path, Path("..") / path)
 
-    with mkdocs_gen_files.open(Path(dst_path) / "SUMMARY.md", "w") as nav_file:
+    with mkdocs_gen_files.open(Path(config.docs_path) / "SUMMARY.md", "w") as nav_file:
         nav_file.writelines(nav.build_literate_nav())

src/frequenz/repo/config/protobuf.py

@@ -0,0 +1,83 @@
+# License: MIT
+# Copyright © 2023 Frequenz Energy-as-a-Service GmbH
+
+"""Manages the configuration to generate files from the protobuf files."""
+
+import dataclasses
+import logging
+import pathlib
+import tomllib
+from collections.abc import Sequence
+from typing import Any, Self
+
+_logger = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass(frozen=True, kw_only=True)
+class ProtobufConfig:
+    """A configuration for the protobuf files.
+
+    The configuration can be loaded from the `pyproject.toml` file using the class
+    method `from_pyproject_toml()`.
+    """
+
+    proto_path: str = "proto"
+    """The path of the root directory containing the protobuf files."""
+
+    proto_glob: str = "*.proto"
+    """The glob pattern to use to find the protobuf files."""
+
+    include_paths: Sequence[str] = (
+        "submodules/api-common-protos",
+        "submodules/frequenz-api-common/proto",
+    )
+    """The paths to add to the include path when compiling the protobuf files."""
+
+    py_path: str = "py"
+    """The path of the root directory where the Python files will be generated."""
+
+    docs_path: str = "protobuf-reference"
+    """The path of the root directory where the documentation files will be generated."""
+
+    @classmethod
+    def from_pyproject_toml(
+        cls, /, path: str = "pyproject.toml", **defaults: Any
+    ) -> Self:
+        """Create a new configuration by loading the options from a `pyproject.toml` file.
+
+        The options are read from the `[tool.frequenz-repo-config.protobuf]`
+        section of the `pyproject.toml` file.
+
+        Args:
+            path: The path to the `pyproject.toml` file.
+            **defaults: The default values for the options missing in the file.  If
+                a default is missing too, then the default in this class will be used.
+
+        Returns:
+            The configuration.
+        """
+        try:
+            with pathlib.Path(path).open("rb") as toml_file:
+                pyproject_toml = tomllib.load(toml_file)
+        except FileNotFoundError:
+            return cls(**defaults)
+        except (IOError, OSError) as err:
+            _logger.warning("WARNING: Failed to load pyproject.toml: %s", err)
+            return cls(**defaults)
+
+        try:
+            config = pyproject_toml["tool"]["frequenz-repo-config"]["protobuf"]
+        except KeyError:
+            return cls(**defaults)
+
+        known_keys = frozenset(defaults.keys())
+        config_keys = frozenset(config.keys())
+        if unknown_keys := config_keys - known_keys:
+            _logger.warning(
+                "WARNING: There are some configuration keys in pyproject.toml we don't "
+                "know about and will be ignored: %s",
+                ", ".join(f"'{k}'" for k in unknown_keys),
+            )
+
+        attrs = dict(defaults, **{k: config[k] for k in (known_keys & config_keys)})
+        return cls(**attrs)

src/frequenz/repo/config/setuptools/grpc_tools.py

@@ -18,6 +18,8 @@
 import setuptools as _setuptools
 import setuptools.command.build as _build_command
 
+from .. import protobuf as _protobuf
+
 
 class CompileProto(_setuptools.Command):
     """Build the Python protobuf files."""
@@ -57,69 +59,18 @@ class CompileProto(_setuptools.Command):
     ]
     """Options of the command."""
 
-    DEFAULT_OPTIONS: dict[str, str] = {
-        "proto_path": "proto",
-        "proto_glob": "*.proto",
-        "include_paths": "submodules/api-common-protos,submodules/frequenz-api-common/proto",
-        "py_path": "py",
-    }
-
     def initialize_options(self) -> None:
         """Initialize options."""
-        options = self._get_options_from_pyproject_toml(self.DEFAULT_OPTIONS)
+        config = _protobuf.ProtobufConfig.from_pyproject_toml()
 
-        self.proto_path = options["proto_path"]
-        self.proto_glob = options["proto_glob"]
-        self.include_paths = options["include_paths"]
-        self.py_path = options["py_path"]
+        self.proto_path = config.proto_path
+        self.proto_glob = config.proto_glob
+        self.include_paths = ",".join(config.include_paths)
+        self.py_path = config.py_path
 
     def finalize_options(self) -> None:
         """Finalize options."""
 
-    def _get_options_from_pyproject_toml(
-        self, defaults: dict[str, str]
-    ) -> dict[str, str]:
-        """Get the options from the pyproject.toml file.
-
-        The options are read from the `[tool.frequenz-repo-config.setuptools.grpc_tools]`
-        section of the pyproject.toml file.
-
-        Args:
-            defaults: The default values for the options.
-
-        Returns:
-            The options read from the pyproject.toml file.
-        """
-        try:
-            with _pathlib.Path("pyproject.toml").open("rb") as toml_file:
-                pyproject_toml = _tomllib.load(toml_file)
-        except FileNotFoundError:
-            return defaults
-        except (IOError, OSError) as err:
-            print(f"WARNING: Failed to load pyproject.toml: {err}")
-            return defaults
-
-        try:
-            config = pyproject_toml["tool"]["frequenz-repo-config"]["setuptools"][
-                "grpc_tools"
-            ]
-        except KeyError:
-            return defaults
-
-        known_keys = frozenset(defaults.keys())
-        config_keys = frozenset(config.keys())
-        if unknown_keys := config_keys - known_keys:
-            print(
-                "WARNING: There are some configuration keys in pyproject.toml we don't "
-                "know about and will be ignored: "
-                + ", ".join(f"'{k}'" for k in unknown_keys)
-            )
-
-        if "include_paths" in config:
-            config["include_paths"] = ",".join(config["include_paths"])
-
-        return dict(defaults, **{k: config[k] for k in (known_keys & config_keys)})
-
     def run(self) -> None:
         """Compile the Python protobuf files."""
         include_paths = self.include_paths.split(",")