Generate documentation for proto files

A new function is added to `frequenz.repo.config.mkdocs` to generate
documentation pages for protobuf files with `pseudomuto/protoc-gen-doc`.

The cookiecutter template uses the new function to generate the pages
for API-type projects automatically, adding a new top-level tab to the
generated site called "Protobuf API Reference".

Signed-off-by: Leandro Lucarella <[email protected]>

Author: llucax

cookiecutter/hooks/post_gen_project.py

@@ -68,6 +68,8 @@ def main() -> None:
     print("nox")
     print("# To generate and serve the documentation:")
     print("pip install .[dev-mkdocs]")
+    if cookiecutter.type == "api":
+        print("# Requires docker")
     print("mkdocs serve")
     print()
     if warnings := do_sanity_checks():

cookiecutter/{{cookiecutter.github_repo_name}}/docs/SUMMARY.md

@@ -1,6 +1,7 @@
 * [Home](index.md)
 {%- if cookiecutter.type == "api" %}
-* [Python API Reference](reference/)
+* [Protobuf API Reference](protobuf-reference/)
+* [Python API Reference](python-reference/)
 {%- else %}
 * [API Reference](reference/)
 {%- endif %}

cookiecutter/{{cookiecutter.github_repo_name}}/docs/mkdocstrings_autoapi.py

@@ -5,4 +5,7 @@
 
 from frequenz.repo.config import mkdocs
 
-mkdocs.generate_api_pages("{{cookiecutter | src_path}}")
+mkdocs.generate_python_api_pages("{{cookiecutter | src_path}}", "{{'python-' if cookiecutter.type == 'api'}}reference")
+{%- if cookiecutter.type == 'api' %}
+mkdocs.generate_protobuf_api_pages()
+{%- endif %}

src/frequenz/repo/config/__init__.py

@@ -180,7 +180,7 @@
 ```
 
 By default this script will look for files in the `src/` directory and generate the
-documentation files in the `reference/` directory inside `mkdocs` output directory
+documentation files in the `python-reference/` directory inside `mkdocs` output directory
 (`site` by defaul).
 
 If you need to customize the above paths, you can create a new script to use with the
@@ -189,7 +189,7 @@
 ```python
 from frequenz.repo.config import mkdocs
 
-mkdocs.generate_api_pages("my_sources", "API")
+mkdocs.generate_python_api_pages("my_sources", "API")
 ```
 
 Where `my_sources` is the directory containing the source files and `API` is the
@@ -205,6 +205,16 @@
         - path/to/my/custom/script.py
 ```
 
+If your project also provides *protobuf* files, you can also generate the API
+documentation for them adding one more line to the previous script:
+
+```python
+from frequenz.repo.config import mkdocs
+
+mkdocs.generate_python_api_pages("my_sources", "API-py")
+mkdocs.generate_protobuf_api_pages("my_protos", "API-proto")
+```
+
 # APIs
 
 ## `setuptools` gRPC support

src/frequenz/repo/config/mkdocs.py

@@ -12,6 +12,8 @@
 https://mkdocstrings.github.io/recipes/#automatic-code-reference-pages
 """
 
+import subprocess
+import tempfile
 from pathlib import Path
 from typing import Tuple
 
@@ -34,7 +36,9 @@ def with_underscore_not_init(part: str) -> bool:
     return any(p for p in path_parts if with_underscore_not_init(p))
 
 
-def generate_api_pages(src_path: str = "src", dst_path: str = "reference") -> None:
+def generate_python_api_pages(
+    src_path: str = "src", dst_path: str = "python-reference"
+) -> None:
     """Generate API documentation pages for the code.
 
     Internal modules (those starting with an underscore except from `__init__`) are
@@ -74,3 +78,66 @@ def generate_api_pages(src_path: str = "src", dst_path: str = "reference") -> No
 
     with mkdocs_gen_files.open(Path(dst_path) / "SUMMARY.md", "w") as nav_file:
         nav_file.writelines(nav.build_literate_nav())
+
+
+def generate_protobuf_api_pages(
+    src_path: str = "proto", dst_path: str = "protobuf-reference"
+) -> None:
+    """Generate API documentation pages for the code.
+
+    Internal modules (those starting with an underscore except from `__init__`) are
+    not included.
+
+    A summary page is generated as `SUMMARY.md` which is compatible with the
+    `mkdocs-literary-nav` plugin.
+
+    Args:
+        src_path: Path where the code is located.
+        dst_path: Path where the documentation should be generated.  This is relative
+            to the output directory of mkdocs.
+    """
+    # type ignore because mkdocs_gen_files uses a very weird module-level
+    # __getattr__() which messes up the type system
+    nav = mkdocs_gen_files.Nav()  # type: ignore
+
+    cwd = Path.cwd()
+
+    with tempfile.TemporaryDirectory(prefix="mkdocs-protobuf-reference-") as tmp_path:
+        for path in sorted(Path(src_path).rglob("*.proto")):
+            doc_path = path.relative_to(src_path).with_suffix(".md")
+            full_doc_path = Path(dst_path, doc_path)
+            parts = tuple(path.relative_to(src_path).parts)
+            nav[parts] = doc_path.as_posix()
+            doc_tmp_path = tmp_path / doc_path
+            doc_tmp_path.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                # TODO: Get arguments from setuptools.grpc
+                subprocess.run(
+                    [
+                        "docker",
+                        "run",
+                        "--rm",
+                        f"-v{cwd}:{cwd}",
+                        f"-v{tmp_path}:{tmp_path}",
+                        "pseudomuto/protoc-gen-doc",
+                        f"-I{cwd / src_path}",
+                        f"-I{cwd}/submodules/api-common-protos",
+                        f"-I{cwd}/submodules/frequenz-api-common/proto",
+                        f"--doc_opt=markdown,{doc_path.name}",
+                        f"--doc_out={tmp_path / doc_path.parent}",
+                        str(cwd / path),
+                    ],
+                    check=True,
+                )
+            except subprocess.CalledProcessError as error:
+                print(f"Error generating protobuf reference page: {error}")
+
+            with doc_tmp_path.open() as input_file, mkdocs_gen_files.open(
+                full_doc_path, "w"
+            ) as output_file:
+                output_file.write(input_file.read())
+
+            mkdocs_gen_files.set_edit_path(full_doc_path, Path("..") / path)
+
+    with mkdocs_gen_files.open(Path(dst_path) / "SUMMARY.md", "w") as nav_file:
+        nav_file.writelines(nav.build_literate_nav())