Add golden testing for generated files

Testing the generated templates are correct is hard. For now we are
testing that the basics work by running `nox` on some generated projects
but this will not test bugs in documentation, comments, etc. Nor will
test GitHub Actions workflows or other advanced configuration.

By adding golden tests we at least ensure that once the results are
manually tested, we won't introduce unexpected regressions.

We considered using the pytest-golden plugin, but unfortunately it does
not support using whole files and directory trees as goldens, so we
built our own ad-hoc solution for now.

To update the golden files the test file needs to be temporarily updated
to set `UPDATE_GOLDEN` to `True` to replace the golden files with the
newly generated files by running `pytest` again, which is not great, but
it is good enough for now.

Signed-off-by: Leandro Lucarella <[email protected]>

Author: llucax

CONTRIBUTING.md

@@ -60,6 +60,41 @@ nox -R -s pylint -- test/test_*.py
 nox -R -s mypy -- test/test_*.py
 ```
 
+### Golden Tests
+
+To test the generated files using the Cookiecutter templates, the [golden
+testing](https://en.wikipedia.org/wiki/Characterization_test) technique is used
+to ensure that changes in the templates don't occur unexpectedly.
+
+If a golden test fails, a `diff` of the contents will be provided in the test
+results.
+
+Failures in the golden tests could indicate two things:
+
+1. The generated files don't match the golden files because an unintended
+   change was introduced. For example, there may be a bug that needs to be fixed
+   so that the generated files match the golden files again.
+
+2. The generated files don't match the golden files because an intended change
+   was introduced. In this case, the golden files need to be updated.
+
+In the latter case, manually updating files is complicated and error-prone, so
+a simpler (though hacky) way is provided.
+
+To update the golden files, simply edit the
+`tests/integration/test_cookiecutter_generation.py` file and temporarily set
+`UPDATE_GOLDEN` to `True`. Then, run the test again using `pytest` as usual.
+This will replace the existing golden files (stored in `tests_golden/`) with
+the newly generated files.
+
+Note that if you rename, or remove golden files, you should also manually
+remove the files that were affected. An easy way to make sure there are no old
+unused golden files left is to just wipe the whole `tests_golden/` directory
+before running `pytest` to generate the new ones.
+
+**Please ensure that all introduced changes are intended before updating the
+golden files.**
+
 ### Building the documentation
 
 To build the documentation, first install the dependencies (if you didn't

tests/integration/test_cookiecutter_generation.py

@@ -5,12 +5,49 @@
 
 import pathlib
 import re
+import shutil
 import subprocess
 
 import pytest
 
 from frequenz.repo import config
 
+UPDATE_GOLDEN: bool = False
+"""Set to True to update the golden files.
+
+After setting to True you need to run the tests once to update the golden files.
+
+Make sure to review the changes before committing them and setting this back to False.
+"""
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("repo_type", [*config.RepositoryType])
+def test_golden(
+    tmp_path: pathlib.Path,
+    repo_type: config.RepositoryType,
+    request: pytest.FixtureRequest,
+) -> None:
+    """Test generation of a new repo comparing it to a golden tree."""
+    cwd = pathlib.Path().cwd()
+    golden_path = (
+        cwd
+        / "tests_golden"
+        / request.path.relative_to(cwd / "tests").with_suffix("")
+        / repo_type.value
+    )
+
+    generated_repo_path, run_result = _generate_repo(
+        repo_type, tmp_path, capture_output=True
+    )
+    stdout, stderr = _filter_generation_output(run_result)
+    _assert_golden_file(golden_path, "cookiecutter-stdout", stdout)
+    _assert_golden_file(golden_path, "cookiecutter-stderr", stderr)
+    _assert_golden_tree(
+        generated_repo_path=generated_repo_path,
+        golden_tree=golden_path / generated_repo_path.name,
+    )
+
 
 @pytest.mark.integration
 @pytest.mark.parametrize("repo_type", [*config.RepositoryType])
@@ -66,6 +103,144 @@ def _run(
     return subprocess.run(cmd, cwd=cwd, check=check, capture_output=capture_output)
 
 
+def _read_golden_file(golden_path: pathlib.Path, name: str) -> str:
+    """Read a golden file.
+
+    File names will be appended with ".txt".
+
+    Args:
+        golden_path: The path to the directory containing the golden files.
+        name: The name of the golden file.
+
+    Returns:
+        The contents of the file.
+    """
+    with open(
+        golden_path / f"{name}.txt", "r", encoding="utf8", errors="replace"
+    ) as golden_file:
+        return golden_file.read()
+
+
+def _write_golden_file(golden_path: pathlib.Path, name: str, contents: str) -> int:
+    """Write a golden file.
+
+    File names will be appended with ".txt".
+
+    Args:
+        golden_path: The path to the directory containing the golden files.
+        name: The name of the golden file.
+
+    Returns:
+        The number of bytes written.
+    """
+    golden_path.mkdir(parents=True, exist_ok=True)
+    with open(
+        golden_path / f"{name}.txt", "w", encoding="utf8", errors="replace"
+    ) as golden_file:
+        return golden_file.write(contents)
+
+
+def _assert_golden_file(
+    golden_path: pathlib.Path,
+    name: str,
+    new_result: str | bytes | subprocess.CompletedProcess[bytes],
+) -> None:
+    if isinstance(new_result, subprocess.CompletedProcess):
+        _assert_golden_file(golden_path, f"{name}-stdout", new_result.stdout)
+        _assert_golden_file(golden_path, f"{name}-stderr", new_result.stderr)
+        return
+
+    if isinstance(new_result, bytes):
+        new_result = new_result.decode("utf-8", "replace")
+    if UPDATE_GOLDEN:
+        _write_golden_file(golden_path, name, new_result)
+    else:
+        assert new_result == _read_golden_file(golden_path, name)
+
+
+def _read_golden_tree(
+    *, generated_repo_path: pathlib.Path, golden_tree: pathlib.Path
+) -> subprocess.CompletedProcess[bytes]:
+    """Read a golden tree.
+
+    The `diff` command is used to compare the generated tree with the golden tree.
+
+    Args:
+        generated_repo_path: The path to the generated repository tree.
+        golden_tree: The path to the golden tree.
+
+    Returns:
+        The result of the `diff` command.
+    """
+    return _run(
+        generated_repo_path,
+        "diff",
+        "-ru",
+        str(generated_repo_path),
+        str(golden_tree),
+        check=False,
+        capture_output=True,
+    )
+
+
+def _write_golden_tree(
+    *, generated_repo_path: pathlib.Path, golden_tree: pathlib.Path
+) -> None:
+    """Write a golden tree.
+
+    Replace all files in the golden tree with the files from the generated tree.
+
+    Args:
+        generated_repo_path: The path to the generated repository tree.
+        golden_tree: The path to the golden tree.
+    """
+    if golden_tree.exists():
+        shutil.rmtree(golden_tree)
+    shutil.copytree(generated_repo_path, golden_tree, dirs_exist_ok=True)
+
+
+def _assert_golden_tree(
+    *, generated_repo_path: pathlib.Path, golden_tree: pathlib.Path
+) -> None:
+    if UPDATE_GOLDEN:
+        _write_golden_tree(
+            generated_repo_path=generated_repo_path, golden_tree=golden_tree
+        )
+    else:
+        result = _read_golden_tree(
+            generated_repo_path=generated_repo_path, golden_tree=golden_tree
+        )
+        if result.returncode != 0:
+            print("Generated repo differs from golden repo:")
+            print()
+            print("STDOUT:")
+            print("-" * 80)
+            print(result.stdout.decode("utf-8"))
+            print("-" * 80)
+            print("STDERR:")
+            print("-" * 80)
+            print(result.stderr.decode("utf-8"))
+            print("-" * 80)
+            assert result.returncode == 0
+
+
+def _filter_generation_output(
+    result: subprocess.CompletedProcess[bytes], /
+) -> tuple[bytes, bytes]:
+    """Filter out some lines from the output.
+
+    This is necessary because the output of cookiecutter is not deterministic, so we
+    just remove all non-deterministic lines. These are lines that contain the path to
+    the generated repo (a temporary directory).
+    """
+    stdout = b"\n".join(
+        l
+        for l in result.stdout.splitlines()
+        if not l.startswith((b"WARNING: The replay file's `_template` (",))
+    )
+    return stdout, result.stderr
+
+
 def _update_pyproject_repo_config_dep(
     *,
     repo_config_path: pathlib.Path,