Add support for frequenz-api-common (frequenz-floss#42)

- Improve message when no proto files are found
- Allow configuring via pyproject.toml
- Add `frequenz-api-common` to the default submodules
- Remove `sumodules/` from the submodule name
- Widen the dependencies in the docs
- Remove unnecessary force option to git submodules add
- Remove `--recursive` from submodules update
- Make integration tests more verbose
- Remove unnecessary `type: ignore`

Fixes frequenz-floss#40.

Author: llucax

cookiecutter/hooks/post_gen_project.py

@@ -182,7 +182,6 @@ def mkdir(self) -> None:
                 "git",
                 "submodule",
                 "add",
-                "-f",
                 "--name",
                 submodule.name,
                 submodule.url,
@@ -194,7 +193,7 @@ def mkdir(self) -> None:
             note_on_failure=f"Please add submodule `{submodule.name}` manually.",
         )
     try_run(
-        ["git", "submodule", "update", "--init", "--recursive"],
+        ["git", "submodule", "update", "--init"],
         verbose=True,
         warn_on_error=True,
         warn_on_bad_status="Failed to initialize git submodules!",

cookiecutter/{{cookiecutter.github_repo_name}}/.gitmodules

@@ -1,5 +1,8 @@
 {% if cookiecutter.type == "api" -%}
-[submodule "submodules/api-common-protos"]
+[submodule "api-common-protos"]
 	path = submodules/api-common-protos
 	url = https://github.com/googleapis/api-common-protos.git
+[submodule "frequenz-api-common"]
+	path = submodules/frequenz-api-common
+	url = https://github.com/frequenz-floss/frequenz-api-common.git
 {% endif -%}

cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [ # TODO(cookiecutter): Remove and add more if appropriate
 ]
 {%- elif cookiecutter.type == "api" %}
 dependencies = [ # TODO(cookiecutter): Remove and add more if appropriate
+  "frequenz-api-common >= 0.2.0, < 0.3.0",
   "googleapis-common-protos >= 1.56.2, < 2",
   "grpcio >= 1.51.1, < 2",
 ]

src/frequenz/repo/config/__init__.py

@@ -1,7 +1,7 @@
 # License: MIT
 # Copyright © 2023 Frequenz Energy-as-a-Service GmbH
 
-"""Frequenz project setup tools and common configuration.
+r"""Frequenz project setup tools and common configuration.
 
 The tools are provided to configure the main types of repositories most commonly used at
 Frequenz, defined in
@@ -166,15 +166,21 @@
 - `pytests/`: Directory containing the tests for the Python code.
 - `submodules/api-common-protos`: Directory containing the submodule with the
   `google/api-common-protos` repository.
+- `submodules/frequenz-api-common`: Directory containing the submodule with the
+  `frequenz-floss/frequenz-api-common` repository.
 
 Normally Frequenz APIs use basic types from
-[`google/api-common-protos`](https://github.com/googleapis/api-common-protos),
-so you need to make sure the proper submodule is added to your project:
+[`google/api-common-protos`](https://github.com/googleapis/api-common-protos) and
+[`frequenz-floss/frequenz-api-common`](https://github.com/frequenz-floss/frequenz-api-common),
+so you need to make sure the proper submodules are added to your project:
 
 ```sh
 mkdir submodules
-git submodule add https://github.com:googleapis/api-common-protos.git submodules/api-common-protos
-git commit -m "Add Google api-common-protos submodule" submodules/api-common-protos
+git submodule add https://github.com/googleapis/api-common-protos.git \
+        submodules/api-common-protos
+git submodule add https://github.com/frequenz-floss/frequenz-api-common.git \
+        submodules/frequenz-api-common
+git commit -m "Add api-common-protos and frequenz-api-common submodules" submodules
 ```
 
 Then you need to add this package as a build dependency and a few extra
@@ -190,8 +196,9 @@
 
 [project]
 dependencies = [
-  "googleapis-common-protos == 1.56.2",
-  "grpcio == 1.51.0",
+  "frequenz-api-common >= 0.2.0, < 0.3.0",
+  "googleapis-common-protos >= 1.56.2, < 2",
+  "grpcio >= 1.51.1, < 2",
 ]
 ```
 
@@ -222,11 +229,29 @@
 
 ```
 recursive-include submodules/api-common-protos/google *.proto
+recursive-include submodules/frequenz-api-common/proto *.proto
 ```
 
-For now there is no way to customize where the protocol files are located,
-where the generated files should be placed, or which extra directories must be
-included when compiling the protocol files.
+If the defaults are not suitable for you (for example you need to use more or less
+submodules or your proto files are located somewhere else, you can customize how
+the protocol files are generated by adding the following section to your
+`pyproject.toml` file:
+
+```toml
+[tool.frequenz_repo_config.setuptools.grpc_tools]
+# Location of the proto files relative to the root of the repository (default: "proto")
+proto_path = "proto_files"
+# Glob pattern to use to find the proto files in the proto_path (default: "*.proto")
+proto_glob = "*.prt"  # Default: "*.proto"
+# List of paths to pass to the protoc compiler as include paths (default:
+# ["submodules/api-common-protos", "submodules/frequenz-api-common/proto"])
+include_paths = ["submodules/api-common-protos"]
+# Path where to generate the Python files (default: "py")
+py_path = "generated"
+```
+
+Please adapt the instructions above to your project structure if you need to change the
+defaults.
 """
 
 from . import nox, setuptools

src/frequenz/repo/config/setuptools/grpc_tools.py

@@ -13,11 +13,10 @@
 import pathlib as _pathlib
 import subprocess as _subprocess
 import sys as _sys
+import tomllib as _tomllib
 
 import setuptools as _setuptools
-
-# The typing stub for this module is missing
-import setuptools.command.build as _build_command  # type: ignore[import]
+import setuptools.command.build as _build_command
 
 
 class CompileProto(_setuptools.Command):
@@ -58,16 +57,69 @@ class CompileProto(_setuptools.Command):
     ]
     """Options of the command."""
 
+    DEFAULT_OPTIONS: dict[str, str] = {
+        "proto_path": "proto",
+        "proto_glob": "*.proto",
+        "include_paths": "submodules/api-common-protos,submodules/frequenz-api-common/proto",
+        "py_path": "py",
+    }
+
     def initialize_options(self) -> None:
         """Initialize options."""
-        self.proto_path = "proto"
-        self.proto_glob = "*.proto"
-        self.include_paths = "submodules/api-common-protos"
-        self.py_path = "py"
+        options = self._get_options_from_pyproject_toml(self.DEFAULT_OPTIONS)
+
+        self.proto_path = options["proto_path"]
+        self.proto_glob = options["proto_glob"]
+        self.include_paths = options["include_paths"]
+        self.py_path = options["py_path"]
 
     def finalize_options(self) -> None:
         """Finalize options."""
 
+    def _get_options_from_pyproject_toml(
+        self, defaults: dict[str, str]
+    ) -> dict[str, str]:
+        """Get the options from the pyproject.toml file.
+
+        The options are read from the `[tool.frequenz-repo-config.setuptools.grpc_tools]`
+        section of the pyproject.toml file.
+
+        Args:
+            defaults: The default values for the options.
+
+        Returns:
+            The options read from the pyproject.toml file.
+        """
+        try:
+            with _pathlib.Path("pyproject.toml").open("rb") as toml_file:
+                pyproject_toml = _tomllib.load(toml_file)
+        except FileNotFoundError:
+            return defaults
+        except (IOError, OSError) as err:
+            print(f"WARNING: Failed to load pyproject.toml: {err}")
+            return defaults
+
+        try:
+            config = pyproject_toml["tool"]["frequenz-repo-config"]["setuptools"][
+                "grpc_tools"
+            ]
+        except KeyError:
+            return defaults
+
+        known_keys = frozenset(defaults.keys())
+        config_keys = frozenset(config.keys())
+        if unknown_keys := config_keys - known_keys:
+            print(
+                "WARNING: There are some configuration keys in pyproject.toml we don't "
+                "know about and will be ignored: "
+                + ", ".join(f"'{k}'" for k in unknown_keys)
+            )
+
+        if "include_paths" in config:
+            config["include_paths"] = ",".join(config["include_paths"])
+
+        return dict(defaults, **{k: config[k] for k in (known_keys & config_keys)})
+
     def run(self) -> None:
         """Compile the Python protobuf files."""
         include_paths = self.include_paths.split(",")
@@ -76,7 +128,10 @@ def run(self) -> None:
         ]
 
         if not proto_files:
-            print(f"No proto files found in {self.proto_path}/**/{self.proto_glob}/")
+            print(
+                f"No proto files found in {self.proto_path}/**/{self.proto_glob}/, "
+                "skipping compilation of proto files."
+            )
             return
 
         protoc_cmd = (

tests/integration/test_cookiecutter_generation.py

@@ -16,25 +16,30 @@
 def test_generation(tmp_path: pathlib.Path, repo_type: str) -> None:
     """Test generation of a new repo."""
     cwd = pathlib.Path().cwd()
-    subprocess.run(
-        [
-            "cookiecutter",
-            "--no-input",
-            cwd / "cookiecutter",
-            f"type={repo_type}",
-            "name=test",
-            "description=Test description",
-        ],
-        cwd=tmp_path,
-        check=True,
+    _run(
+        tmp_path,
+        "cookiecutter",
+        "--no-input",
+        str(cwd / "cookiecutter"),
+        f"type={repo_type}",
+        "name=test",
+        "description=Test description",
     )
+
     subdirs = list(tmp_path.iterdir())
     assert len(subdirs) == 1
     repo_path = subdirs[0]
-    subprocess.run("python3 -m venv .venv".split(), cwd=repo_path, check=True)
-    subprocess.run(
-        ". .venv/bin/activate; pip install .[dev-noxfile]; nox",
-        shell=True,
-        cwd=repo_path,
-        check=True,
-    )
+    _run(repo_path, "python3", "-m", "venv", ".venv")
+
+    cmd = ". .venv/bin/activate; pip install .[dev-noxfile]; nox"
+    print()
+    print(f"Running in shell [{cwd}]: {cmd}")
+    subprocess.run(cmd, shell=True, cwd=repo_path, check=True)
+
+
+def _run(cwd: pathlib.Path, *cmd: str) -> subprocess.CompletedProcess[bytes]:
+    print()
+    print("-" * 80)
+    print(f"Running [{cwd}]: {' '.join(cmd)}")
+    print()
+    return subprocess.run(cmd, cwd=cwd, check=True)