Move mkdocstrings-macros integration to a library (#349)

This PR moves the documentation macros functionality from individual
template repositories into a central location, making it available as a
`macros` *pluglet*. Key improvements include:

- Removes duplicated boilerplate code
- Adds a modular `macros` *pluglet* with common functionality
- Simplifies template setup by using git-aware version info
- Includes migration script for existing repositories
- Updates release notes and documentation

The PR also bumps template dependencies and repository configuration
versions in preparation for the 0.12 release.

Fixes #179.

Author: llucax

RELEASE_NOTES.md

@@ -2,7 +2,7 @@
 
 ## Summary
 
-<!-- Here goes a general summary of what this release is about -->
+This release introduces a new MkDocs macros *pluglet* system that simplifies documentation setup and provides enhanced functionality for version information and code annotations. It also includes changes to how pytest warnings are handled in templates.
 
 ## Upgrading
 
@@ -15,22 +15,28 @@
 
 ### Cookiecutter template
 
-All upgrading should be done via the migration script or regenerating the templates. But you might still need to adapt your code:
+All upgrading should be done via the migration script or regenerating the templates.
+
+```bash
+curl -sSL https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/v0.12/cookiecutter/migrate.py | python3
+```
+
+But you might still need to adapt your code:
 
 - `pytest` now uses `-Werror` by default (but still treat deprecations as normal warnings), so if your tests run with warnings, they will now be turned to errors, and you'll need to fix them.
 
+- Projects using `docs/_scripts/macros.py` with customized scripts can use the new provided utility functions. See the [`mkdocstrings_macros` documentation](https://frequenz-floss.github.io/frequenz-repo-config-python/v0.12/reference/frequenz/repo/config/mkdocs/mkdocstrings_macros/) for the new features and setup.
+
 ## New Features
 
-<!-- Here goes the main new features and examples or instructions on how to use them -->
+- Two new modules were introduced to facilitate the configuration of `macros` for use within docstrings via `mkdocstrings`: [`mkdocstrings_macros`](https://frequenz-floss.github.io/frequenz-repo-config-python/v0.12/reference/frequenz/repo/config/mkdocs/mkdocstrings_macros/) and [`annotations`](https://frequenz-floss.github.io/frequenz-repo-config-python/v0.12/reference/frequenz/repo/config/mkdocs/annotations/).
 
 ### Cookiecutter template
 
 - `pytest` now uses `-Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning` by default. Deprecations are still treated as warnings, as when testing with the `pytest_min` session is normal to get deprecation warnings as we are using old versions of dependencies.
 
 ## Bug Fixes
 
-<!-- Here goes notable bug fixes that are worth a special mention or explanation -->
-
 ### Cookiecutter template
 
 - Fixed a compatibility issue in the macros doc script with `mkdocsstrings` 0.28.

cookiecutter/migrate.py

@@ -20,6 +20,7 @@
 And remember to follow any manual instructions for each run.
 """  # noqa: E501
 
+import hashlib
 import os
 import subprocess
 import tempfile
@@ -30,8 +31,8 @@
 def main() -> None:
     """Run the migration steps."""
     add_default_pytest_options()
-
-    # Add a separation line like this one after each migration step.
+    print("=" * 72)
+    migrate_mkdocs_macros()
     print("=" * 72)
 
 
@@ -64,6 +65,92 @@ def add_default_pytest_options() -> None:
     )
 
 
+def migrate_mkdocs_macros() -> None:
+    """Migrate from custom macros.py to standard module."""
+    macros_file = Path("docs/_scripts/macros.py")
+    mkdocs_yaml = Path("mkdocs.yaml")
+    if not mkdocs_yaml.exists():
+        mkdocs_yaml = Path("mkdocs.yml")
+
+    known_hashes = {
+        "47a991286132471b6cb666577beb89e78c0f5d4975c53f0dcb319c4338a2c3cb",
+        "6bb960c72b370ac77918f49d7a35f39c0ddb58fe52cf2d12caa2577098fd8469",
+        "7351276ac314955a343bab09d1602e50300887291f841643e9fb79c94acc923c",
+        "8fa5f9f3fd928e17f590e3ab056434474633259d615971404db0d2f3034adb62",
+        "ba3ff5f1612b3dd22372a8ca95394b8ea468f18dcefc494c73811c8433fcb880",
+        "dd32e8759abc43232bb3db5b33c0a7cf8d8442db6135c594968c499d8bae0ce5",
+    }
+
+    print("Checking if docs/_scripts/macros.py can be migrated...")
+
+    file_hash = calculate_file_sha256_skip_lines(macros_file, 2)
+    if not file_hash:
+        return
+
+    if file_hash not in known_hashes:
+        manual_step("The macros.py file seems to be customized. You have two options:")
+        manual_step("")
+        manual_step(
+            "1. Switch to the standard module (if you don't have custom macros):"
+        )
+        manual_step("   a. Update mkdocs.yaml to use the standard module:")
+        manual_step(
+            '      module_name: docs/_scripts/macros -> modules: ["frequenz.repo.config.mkdocs.mkdocstrings_macros"]'  # noqa: E501
+        )
+        manual_step("   b. Remove docs/_scripts/macros.py")
+        manual_step("")
+        manual_step("2. Keep your custom macros but use the standard functionality:")
+        manual_step("   a. Update mkdocs.yaml:")
+        manual_step("      - Keep using module_name: docs/_scripts/macros")
+        manual_step("   b. Update your macros.py to be minimal:")
+        manual_step("      ```python")
+        manual_step(
+            "      from frequenz.repo.config.mkdocs.mkdocstrings_macros import hook_env_with_everything"  # noqa: E501
+        )
+        manual_step("")
+        manual_step("      def define_env(env):")
+        manual_step("          # Add your custom variables, filters, and macros here")
+        manual_step("          env.variables.my_var = 'Example'")
+        manual_step("          env.filters.my_filter = lambda x: x.upper()")
+        manual_step("")
+        manual_step(
+            "          # This must be at the end to enable all standard features"
+        )
+        manual_step("          hook_env_with_everything(env)")
+        manual_step("      ```")
+        manual_step("")
+        manual_step("See the docs for more details:")
+        manual_step(
+            "https://frequenz-floss.github.io/frequenz-repo-config-python/v0.12/reference/frequenz/repo/config/mkdocs/mkdocstrings_macros/"  # noqa: E501
+        )
+        return
+
+    if not mkdocs_yaml.exists():
+        print("mkdocs.yaml/yml not found, skipping macros migration")
+        return
+
+    content = mkdocs_yaml.read_text(encoding="utf-8")
+    if "module_name: docs/_scripts/macros" not in content:
+        print("Custom macros configuration not found in mkdocs.yaml")
+        return
+
+    print("Updating mkdocs.yaml to use standard module...")
+    new_content = content.replace(
+        "module_name: docs/_scripts/macros",
+        'modules: ["frequenz.repo.config.mkdocs.mkdocstrings_macros"]',
+    )
+    new_content = new_content.replace(
+        "# inside docstrings. See the comment in `docs/_scripts/macros.py` for more\n"
+        "  # details\n",
+        "# inside docstrings.\n",
+    )
+
+    replace_file_contents_atomically(mkdocs_yaml, content, new_content)
+
+    print("Removing docs/_scripts/macros.py...")
+    macros_file.unlink()
+
+
 def apply_patch(patch_content: str) -> None:
     """Apply a patch using the patch utility."""
     subprocess.run(["patch", "-p1"], input=patch_content.encode(), check=True)
@@ -134,5 +221,25 @@ def manual_step(message: str) -> None:
     print(f"\033[0;33m>>> {message}\033[0m")
 
 
+def calculate_file_sha256_skip_lines(filepath: Path, skip_lines: int) -> str | None:
+    """Calculate SHA256 of file contents excluding the first N lines.
+
+    Args:
+        filepath: Path to the file to hash
+        skip_lines: Number of lines to skip at the beginning
+
+    Returns:
+        The SHA256 hex digest, or None if the file doesn't exist
+    """
+    if not filepath.exists():
+        return None
+
+    # Read file and normalize line endings to LF
+    content = filepath.read_text(encoding="utf-8").replace("\r\n", "\n")
+    # Skip first N lines and ensure there's a trailing newline
+    remaining_content = "\n".join(content.splitlines()[skip_lines:]) + "\n"
+    return hashlib.sha256(remaining_content.encode()).hexdigest()
+
+
 if __name__ == "__main__":
     main()

cookiecutter/{{cookiecutter.github_repo_name}}/docs/_scripts/macros.py

@@ -129,10 +129,9 @@ plugins:
 {%- endif %}
             - https://typing-extensions.readthedocs.io/en/stable/objects.inv
   # Note this plugin must be loaded after mkdocstrings to be able to use macros
-  # inside docstrings. See the comment in `docs/_scripts/macros.py` for more
-  # details
+  # inside docstrings.
   - macros:
-      module_name: docs/_scripts/macros
+      modules: ["frequenz.repo.config.mkdocs.mkdocstrings_macros"]
       on_undefined: strict
       on_error_fail: true
   - search

cookiecutter/{{cookiecutter.github_repo_name}}/mkdocs.yml

@@ -3,17 +3,17 @@
 
 [build-system]
 requires = [
-  "setuptools == 75.5.0",
+  "setuptools == 75.8.0",
   "setuptools_scm[toml] == 8.1.0",
-  "frequenz-repo-config[{{cookiecutter.type}}] == 0.11.0",
+  "frequenz-repo-config[{{cookiecutter.type}}] == 0.12.0",
 {%- if cookiecutter.type == "api" %}
   # We need to pin the protobuf, grpcio and  grpcio-tools dependencies to make
   # sure the code is generated using the minimum supported versions, as older
   # versions can't work with code that was generated with newer versions.
   # https://protobuf.dev/support/cross-version-runtime-guarantee/#backwards
-  "protobuf == 5.28.0",
-  "grpcio-tools == 1.66.1",
-  "grpcio == 1.66.1",
+  "protobuf == 5.29.3",
+  "grpcio-tools == 1.70.0",
+  "grpcio == 1.70.0",
 {%- endif %}
 ]
 build-backend = "setuptools.build_meta"
@@ -48,27 +48,27 @@ dependencies = [
   # Make sure to update the version for cross-referencing also in the
   # mkdocs.yml file when changing the version here (look for the config key
   # plugins.mkdocstrings.handlers.python.import)
-  "frequenz-sdk >= 1.0.0rc1300, < 1.0.0rc1400",
+  "frequenz-sdk >= 1.0.0rc1500, < 1.0.0rc1600",
 ]
 {%- elif cookiecutter.type == "app" %}
 dependencies = [
   "typing-extensions == 4.12.2",
   # Make sure to update the version for cross-referencing also in the
   # mkdocs.yml file when changing the version here (look for the config key
   # plugins.mkdocstrings.handlers.python.import)
-  "frequenz-sdk == 1.0.0rc1300",
+  "frequenz-sdk == 1.0.0rc1500",
 ]
 {%- elif cookiecutter.type == "api" %}
 dependencies = [
-  "frequenz-api-common >= 0.6.2, < 0.7.0",
+  "frequenz-api-common >= 0.6.3, < 0.7.0",
   # We can't widen beyond the current value unless we bump the minimum
   # requirements too because of protobuf cross-version runtime guarantees:
   # https://protobuf.dev/support/cross-version-runtime-guarantee/#major
-  "protobuf >= 5.28.0, < 7", # Do not widen beyond 7!
+  "protobuf >= 5.29.3, < 7", # Do not widen beyond 7!
   # We couldn't find any document with a spec about the cross-version runtime
   # guarantee for grpcio, so unless we find one in the future, we'll assume
   # major version jumps are not compatible
-  "grpcio >= 1.66.1, < 2", # Do not widen beyond 2!
+  "grpcio >= 1.70.0, < 2", # Do not widen beyond 2!
 ]
 {%- else %}
 dependencies = [
@@ -87,48 +87,48 @@ dev-flake8 = [
   "flake8 == 7.1.1",
   "flake8-docstrings == 1.7.0",
   "flake8-pyproject == 1.2.3",  # For reading the flake8 config from pyproject.toml
-  "pydoclint == 0.5.9",
+  "pydoclint == 0.6.0",
   "pydocstyle == 6.3.0",
 ]
-dev-formatting = ["black == 24.10.0", "isort == 5.13.2"]
+dev-formatting = ["black == 25.1.0", "isort == 6.0.0"]
 dev-mkdocs = [
   "Markdown == 3.7",
-  "black == 24.10.0",
+  "black == 25.1.0",
   "mike == 2.1.3",
   "mkdocs-gen-files == 0.5.0",
   "mkdocs-literate-nav == 0.6.1",
   "mkdocs-macros-plugin == 1.3.7",
-  "mkdocs-material == 9.5.45",
+  "mkdocs-material == 9.6.2",
   "mkdocstrings[python] == 0.28.0",
   "mkdocstrings-python == 1.14.0",
-  "frequenz-repo-config[{{cookiecutter.type}}] == 0.11.0",
+  "frequenz-repo-config[{{cookiecutter.type}}] == 0.12.0",
 ]
 dev-mypy = [
   "mypy == 1.9.0",
 {%- if cookiecutter.type == "api" %}
-  "grpc-stubs == 1.53.0.2",
+  "grpc-stubs == 1.53.0.5",
 {%- endif %}
-  "types-Markdown == 3.7.0.20240822",
+  "types-Markdown == 3.7.0.20241204",
   # For checking the noxfile, docs/ script, and tests
   "{{cookiecutter.pypi_package_name}}[dev-mkdocs,dev-noxfile,dev-pytest]",
 ]
 dev-noxfile = [
   "nox == 2024.10.9",
-  "frequenz-repo-config[{{cookiecutter.type}}] == 0.11.0",
+  "frequenz-repo-config[{{cookiecutter.type}}] == 0.12.0",
 ]
 dev-pylint = [
   # dev-pytest already defines a dependency to pylint because of the examples
   # For checking the noxfile, docs/ script, and tests
   "{{cookiecutter.pypi_package_name}}[dev-mkdocs,dev-noxfile,dev-pytest]",
 ]
 dev-pytest = [
-  "pytest == 8.3.3",
-  "pylint == 3.3.1",       # We need this to check for the examples
-  "frequenz-repo-config[extra-lint-examples] == 0.11.0",
+  "pytest == 8.3.4",
+  "pylint == 3.3.4",       # We need this to check for the examples
+  "frequenz-repo-config[extra-lint-examples] == 0.12.0",
 {%- if cookiecutter.type != "api" %}
   "pytest-mock == 3.14.0",
-  "pytest-asyncio == 0.24.0",
-  "async-solipsism == 0.6",
+  "pytest-asyncio == 0.25.3",
+  "async-solipsism == 0.7",
 {%- endif %}
 ]
 dev = [