Add golden tests for generated files (#87)

Testing the generated templates are correct is hard. For now we are
testing that the basics work by running `nox` on some generated projects
but this will not test bugs in documentation, comments, etc. Nor will
test GitHub Actions workflows or other advanced configuration.

By adding golden tests we at least ensure that once the results are
manually tested, we won't introduce unexpected regressions.

We considered using the pytest-golden plugin, but unfortunately it does
not support using whole files and directory trees as goldens, so we
built our own ad-hoc solution for now.

To update the golden files the test file needs to be temporarily updated
to set `UPDATE_GOLDEN` to `True` to replace the golden files with the
newly generated files by running `pytest` again, which is not great, but
it is good enough for now.

Fixes #50.

Author: llucax

.github/labeler.yml

@@ -7,42 +7,71 @@
 # https://github.com/marketplace/actions/labeler
 
 "part:docs":
-  - "**/*.md"
-  - "docs/**"
-  - "examples/**"
-  - LICENSE
+  - any:
+      - "**/*.md"
+      - "docs/**"
+      - "examples/**"
+      - LICENSE
+    all:
+      - "!tests/**"
+      - "!tests_golden/**"
 
 "part:tests":
   - "tests/**"
+  - "tests_golden/**"
 
 "part:tooling":
-  - "**/*.ini"
-  - "**/*.toml"
-  - "**/*.yaml"
-  - "**/*.yml"
-  - ".git*"
-  - ".git*/**"
-  - CODEOWNERS
-  - MANIFEST.in
-  - noxfile.py
+  - any:
+      - "**/*.ini"
+      - "**/*.toml"
+      - "**/*.yaml"
+      - "**/*.yml"
+      - ".git*"
+      - ".git*/**"
+      - CODEOWNERS
+      - MANIFEST.in
+      - noxfile.py
+    all:
+      - "!tests/**"
+      - "!tests_golden/**"
 
 "part:ci":
-  - "**/.github/*labeler.*"
-  - "**/.github/dependabot.*"
-  - "**/.github/workflows/*"
+  - any:
+      - "**/.github/*labeler.*"
+      - "**/.github/dependabot.*"
+      - "**/.github/workflows/*"
+    all:
+      - "!tests/**"
+      - "!tests_golden/**"
 
 "part:cookiecutter":
-  - "cookiecutter/**"
+  - any:
+      - "cookiecutter/**"
+    all:
+      - "!tests/**"
+      - "!tests_golden/**"
 
 "part:mkdocs":
-  - "**/docs/*.py"
-  - "**/mkdocs.*"
-  - "src/frequenz/repo/config/mkdocs*"
+  - any:
+      - "**/docs/*.py"
+      - "**/mkdocs.*"
+      - "src/frequenz/repo/config/mkdocs*"
+    all:
+      - "!tests/**"
+      - "!tests_golden/**"
 
 "part:nox":
-  - "**/noxfile.py"
-  - "src/frequenz/repo/config/nox/**"
+  - any:
+      - "**/noxfile.py"
+      - "src/frequenz/repo/config/nox/**"
+    all:
+      - "!tests/**"
+      - "!tests_golden/**"
 
 "part:protobuf":
-  - "src/frequenz/repo/config/setuptools/grpc*"
-  - "src/frequenz/repo/config/protobuf*"
+  - any:
+      - "src/frequenz/repo/config/setuptools/grpc*"
+      - "src/frequenz/repo/config/protobuf*"
+    all:
+      - "!tests/**"
+      - "!tests_golden/**"

.gitignore

@@ -14,8 +14,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
-lib64/
 parts/
 sdist/
 var/

CONTRIBUTING.md

@@ -60,6 +60,41 @@ nox -R -s pylint -- test/test_*.py
 nox -R -s mypy -- test/test_*.py
 ```
 
+### Golden Tests
+
+To test the generated files using the Cookiecutter templates, the [golden
+testing](https://en.wikipedia.org/wiki/Characterization_test) technique is used
+to ensure that changes in the templates don't occur unexpectedly.
+
+If a golden test fails, a `diff` of the contents will be provided in the test
+results.
+
+Failures in the golden tests could indicate two things:
+
+1. The generated files don't match the golden files because an unintended
+   change was introduced. For example, there may be a bug that needs to be fixed
+   so that the generated files match the golden files again.
+
+2. The generated files don't match the golden files because an intended change
+   was introduced. In this case, the golden files need to be updated.
+
+In the latter case, manually updating files is complicated and error-prone, so
+a simpler (though hacky) way is provided.
+
+To update the golden files, simply edit the
+`tests/integration/test_cookiecutter_generation.py` file and temporarily set
+`UPDATE_GOLDEN` to `True`. Then, run the test again using `pytest` as usual.
+This will replace the existing golden files (stored in `tests_golden/`) with
+the newly generated files.
+
+Note that if you rename, or remove golden files, you should also manually
+remove the files that were affected. An easy way to make sure there are no old
+unused golden files left is to just wipe the whole `tests_golden/` directory
+before running `pytest` to generate the new ones.
+
+**Please ensure that all introduced changes are intended before updating the
+golden files.**
+
 ### Building the documentation
 
 To build the documentation, first install the dependencies (if you didn't

cookiecutter/hooks/post_gen_project.py

@@ -10,6 +10,7 @@
 import configparser as _configparser
 import dataclasses as _dataclasses
 import json as _json
+import os
 import pathlib as _pathlib
 import shutil as _shutil
 import subprocess as _subprocess
@@ -189,6 +190,11 @@ def initialize_git_submodules() -> bool:
     Returns:
         Whether any git submodules were initialized.
     """
+    if is_golden_testing():
+        # We don't use external tools when running tests because they are too flaky, as
+        # different versions emit different outputs
+        return False
+
     gitmodules_path = _pathlib.Path(".gitmodules")
 
     if not gitmodules_path.exists():
@@ -287,8 +293,14 @@ def initialize_git_repo() -> bool:
     Returns:
         Whether the project was initialized as a git repository.
     """
+    if is_golden_testing():
+        # We don't use external tools when running tests because they are too flaky, as
+        # different versions emit different outputs
+        return False
+
     if _pathlib.Path(".git").exists():
         return False
+
     print()
     note("Initializing git repository...")
     try_run(
@@ -309,6 +321,11 @@ def commit_git_changes(*, first_commit: bool) -> None:
     Args:
         first_commit: Whether this is the first commit in the git repository.
     """
+    if is_golden_testing():
+        # We don't use external tools when running tests because they are too flaky, as
+        # different versions emit different outputs
+        return
+
     if not try_run(["git", "status", "--porcelain"]):
         return
 
@@ -367,6 +384,11 @@ def is_file_empty(path: _pathlib.Path) -> bool:
 
 def print_generated_tree() -> None:
     """Print the generated files tree."""
+    if is_golden_testing():
+        # We don't use external tools when running tests because they are too flaky, as
+        # different versions emit different outputs
+        return
+
     result = try_run(["tree"])
     if result is not None and result.returncode == 0:
         print()
@@ -376,12 +398,13 @@ def print_todos() -> None:
     """Print all TODOs in the generated project."""
     todo_str = "TODO(cookiecutter):"
     repo = cookiecutter.github_repo_name
-    cmd = ["grep", "-r", "--color", rf"\<{todo_str}.*", "."]
+    cmd = rf"grep -r --color '\<{todo_str}.*' . | sort"
     try_run(
         cmd,
         warn_on_error=True,
-        warn_on_bad_status=f"No `{todo_str}` found using `{' '.join(cmd)}`",
+        warn_on_bad_status=f"No `{todo_str}` found using `{cmd}`",
         note_on_failure=f"Please search for `{todo_str}` in `{repo}/` manually.",
+        shell=True,
     )
     print()
     note(
@@ -467,13 +490,14 @@ def finish_model_setup() -> None:
 
 
 def try_run(
-    cmd: list[str],
+    cmd: list[str] | str,
     /,
     *,
     warn_on_error: bool = False,
     warn_on_bad_status: str | None = None,
     note_on_failure: str | None = None,
     verbose: bool = False,
+    shell: bool = False,
 ) -> _subprocess.CompletedProcess[Any] | None:
     """Try to run a command.
 
@@ -487,17 +511,20 @@ def try_run(
         note_on_failure: If not `None`, print this note if the command fails (either
             because of an error or a non-zero status code).
         verbose: Whether to print the command before running it.
+        shell: Whether to run the command in a shell. If `True`, `cmd` must be a
+            string, otherwise it must be a list of strings.
 
     Returns:
         The result of the command or `None` if the command could not be run because
         of an error.
     """
+    assert isinstance(cmd, str) and shell or isinstance(cmd, list) and not shell
     result = None
     failed = False
     if verbose:
         print(f"Executing: {' '.join(cmd)}")
     try:
-        result = _subprocess.run(cmd, check=False)
+        result = _subprocess.run(cmd, check=False, shell=shell)
     except (OSError, _subprocess.CalledProcessError) as exc:
         if warn_on_error:
             warn(f"Failed to run the search command `{' '.join(cmd)}`: {exc}")
@@ -513,6 +540,15 @@ def try_run(
     return result
 
 
+def is_golden_testing() -> bool:
+    """Return `True` if we are running as part of a golden testing.
+
+    Returns:
+        Whether we are running as part of a golden testing.
+    """
+    return os.environ.get("GOLDEN_TEST", None) is not None
+
+
 def recursive_overwrite_move(src: _pathlib.Path, dst: _pathlib.Path) -> None:
     """Recursively move a directory overwriting the target files if they exist.
 

pyproject.toml

@@ -132,5 +132,6 @@ testpaths = ["tests"]
 markers = [
   "integration: integration tests (deselect with '-m \"not integration\"')",
 ]
+
 [tool.setuptools_scm]
 version_scheme = "post-release"