From 6a97e5cb9948c83e79cb6ee1b37a58a54f38c7fb Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Tue, 3 Jun 2025 10:58:48 +0200 Subject: [PATCH 1/2] Improve filtering of warnings in pytest This updates the cookiecutter templates to ignore warnings by using the separate `filterwarnings` options instead of using command-line options. This allows using regex, which is needed to add a new ignore for protobuf gencode version warnings, that includes the protobuf version, which will change dynamically. Signed-off-by: Leandro Lucarella --- RELEASE_NOTES.md | 2 +- cookiecutter/migrate.py | 118 +++++++++++++++++- .../pyproject.toml | 13 +- pyproject.toml | 13 +- .../actor/frequenz-actor-test/pyproject.toml | 13 +- .../api/frequenz-api-test/pyproject.toml | 12 ++ .../app/frequenz-app-test/pyproject.toml | 13 +- .../lib/frequenz-test-python/pyproject.toml | 13 +- .../model/frequenz-model-test/pyproject.toml | 13 +- 9 files changed, 202 insertions(+), 8 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b99e3a30..db808f03 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -26,7 +26,7 @@ But you might still need to adapt your code: ### Cookiecutter template - +- New warning ignores for protobuf gencode versions in pytest. ## Bug Fixes diff --git a/cookiecutter/migrate.py b/cookiecutter/migrate.py index c0e1dba0..e4b3eae1 100644 --- a/cookiecutter/migrate.py +++ b/cookiecutter/migrate.py @@ -22,20 +22,136 @@ import hashlib import os +import re import subprocess import tempfile from pathlib import Path -from typing import SupportsIndex +from typing import Final, SupportsIndex def main() -> None: """Run the migration steps.""" # Add a separation line like this one after each migration step. print("=" * 72) + migrate_filterwarnings(Path("pyproject.toml")) + print("=" * 72) print("Migration script finished. Remember to follow any manual instructions.") print("=" * 72) +# pylint: disable-next=too-many-locals,too-many-statements,too-many-branches +def migrate_filterwarnings(path: Path) -> None: + """Migrate the filterwarnings configuration in pyproject.toml files.""" + print(f"Migrating from pytest addopts to filterwarnings in {path}...") + # Patterns to identify and clean existing addopts flags + addopts_re: Final = re.compile(r'^(\s*)addopts\s*=\s*"(.*)"') + filterwarnings_re: Final = re.compile(r"^(\s*)filterwarnings\s*=\s*(.*)") + w_flag_re: Final = re.compile(r"^-W=?(.*)$") + unwanted_flags: Final = { + "-W=all", + "-Werror", + "-Wdefault::DeprecationWarning", + "-Wdefault::PendingDeprecationWarning", + } + + text = path.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + new_lines: list[str] = [] + modified = False + addopts_found = False + has_filterwarnings = False + has_w_flags = False + w_flags: list[str] = [] + + for line in lines: + filterwarnings_match = filterwarnings_re.match(line) + if filterwarnings_match: + has_filterwarnings = True + addopts_match = addopts_re.match(line) + if addopts_match and not modified: + addopts_found = True + indent, inner = addopts_match.group(1), addopts_match.group(2) + tokens = inner.split() + remaining_tokens: list[str] = [] + extra_specs: list[str] = [] + + for tok in tokens: + if tok in unwanted_flags: + # Discard it; it will be replaced by base_specs + continue + + w_match = w_flag_re.match(tok) + if w_match: + w_flags.append(tok) + has_w_flags = True + spec = w_match.group(1) + if spec: + # Convert this -W... into a filterwarnings spec + extra_specs.append(spec) + else: + # Keep any non -W token + remaining_tokens.append(tok) + + # Base filterwarnings specs to replace unwanted flags + base_specs = map( + str.strip, + r""" + "error", + "once::DeprecationWarning", + "once::PendingDeprecationWarning", + # We ignore warnings about protobuf gencode version being one version older + # than the current version, as this is supported by protobuf, and we expect to + # have such cases. If we go too far, we will get a proper error anyways. + # We use a raw string (single quotes) to avoid the need to escape special + # characters as this is a regex. + 'ignore:Protobuf gencode version .*exactly one major version older.*:UserWarning', + """.strip().splitlines(), + ) + + # Rebuild addopts line without unwanted flags + new_addopts_value = " ".join(remaining_tokens) + new_lines.append(f'{indent}addopts = "{new_addopts_value}"\n') + + # Build the filterwarnings block + new_lines.append(f"{indent}filterwarnings = [\n") + # This is fine, indent is defined only once, so even if it is a closure + # bound late, the value will always be the same. + # pylint: disable-next=cell-var-from-loop + new_lines.extend(map(lambda s: f"{indent} {s}\n", base_specs)) + for spec in extra_specs: + new_lines.append(f'{indent} "{spec}",\n') + new_lines.append(f"{indent}]\n") + + modified = True + else: + new_lines.append(line) + + if modified and not has_filterwarnings: + print(f"Updated {path} to use filterwarnings.") + path.write_text("".join(new_lines), encoding="utf-8") + return + + if has_filterwarnings and not has_w_flags: + print( + f"The file {path} already has a `filterwarnings` section and has no " + "-W flags in `addopts`, it is probably already migrated." + ) + elif has_filterwarnings and has_w_flags: + print( + f"The file {path} already has a `filterwarnings` section, but also " + f"has -W flags in `addopts` ({' '.join(w_flags)!r}), it looks like " + "it is half-migrated, you should probably migrate it manually. Avoid using -W " + "flags in `addopts` if there is a `filterwarnings` section." + ) + if not addopts_found: + print(f"No 'addopts' found in {path}.") + + manual_step( + f"No changes done to {path}. " + "Please double check no manual steps are required." + ) + + def apply_patch(patch_content: str) -> None: """Apply a patch using the patch utility.""" subprocess.run(["patch", "-p1"], input=patch_content.encode(), check=True) diff --git a/cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml b/cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml index 3ee77e64..b1bb97bc 100644 --- a/cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml +++ b/cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml @@ -202,8 +202,19 @@ disable = [ ] [tool.pytest.ini_options] +addopts = "-vv" +filterwarnings = [ + "error", + "once::DeprecationWarning", + "once::PendingDeprecationWarning", + # We ignore warnings about protobuf gencode version being one version older + # than the current version, as this is supported by protobuf, and we expect to + # have such cases. If we go too far, we will get a proper error anyways. + # We use a raw string (single quotes) to avoid the need to escape special + # characters as this is a regex. + 'ignore:Protobuf gencode version .*exactly one major version older.*:UserWarning', +] {%- if cookiecutter.type != "api" %} -addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv" testpaths = ["tests", "src"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" diff --git a/pyproject.toml b/pyproject.toml index 183bc4e2..09403de5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -206,7 +206,18 @@ module = [ ignore_missing_imports = true [tool.pytest.ini_options] -addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv" +addopts = "-vv" +filterwarnings = [ + "error", + "once::DeprecationWarning", + "once::PendingDeprecationWarning", + # We ignore warnings about protobuf gencode version being one version older + # than the current version, as this is supported by protobuf, and we expect to + # have such cases. If we go too far, we will get a proper error anyways. + # We use a raw string (single quotes) to avoid the need to escape special + # characters as this is a regex. + 'ignore:Protobuf gencode version .*exactly one major version older.*:UserWarning', +] testpaths = ["src", "tests"] markers = [ "integration: integration tests (deselect with '-m \"not integration\"')", diff --git a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/pyproject.toml b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/pyproject.toml index 4528a8af..bb0651de 100644 --- a/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/pyproject.toml +++ b/tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/pyproject.toml @@ -153,7 +153,18 @@ disable = [ ] [tool.pytest.ini_options] -addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv" +addopts = "-vv" +filterwarnings = [ + "error", + "once::DeprecationWarning", + "once::PendingDeprecationWarning", + # We ignore warnings about protobuf gencode version being one version older + # than the current version, as this is supported by protobuf, and we expect to + # have such cases. If we go too far, we will get a proper error anyways. + # We use a raw string (single quotes) to avoid the need to escape special + # characters as this is a regex. + 'ignore:Protobuf gencode version .*exactly one major version older.*:UserWarning', +] testpaths = ["tests", "src"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" diff --git a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/pyproject.toml b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/pyproject.toml index e063a5dc..0a85f0ba 100644 --- a/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/pyproject.toml +++ b/tests_golden/integration/test_cookiecutter_generation/api/frequenz-api-test/pyproject.toml @@ -161,6 +161,18 @@ disable = [ ] [tool.pytest.ini_options] +addopts = "-vv" +filterwarnings = [ + "error", + "once::DeprecationWarning", + "once::PendingDeprecationWarning", + # We ignore warnings about protobuf gencode version being one version older + # than the current version, as this is supported by protobuf, and we expect to + # have such cases. If we go too far, we will get a proper error anyways. + # We use a raw string (single quotes) to avoid the need to escape special + # characters as this is a regex. + 'ignore:Protobuf gencode version .*exactly one major version older.*:UserWarning', +] testpaths = ["pytests"] [tool.mypy] diff --git a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/pyproject.toml b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/pyproject.toml index dd744057..1c8a80f3 100644 --- a/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/pyproject.toml +++ b/tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/pyproject.toml @@ -152,7 +152,18 @@ disable = [ ] [tool.pytest.ini_options] -addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv" +addopts = "-vv" +filterwarnings = [ + "error", + "once::DeprecationWarning", + "once::PendingDeprecationWarning", + # We ignore warnings about protobuf gencode version being one version older + # than the current version, as this is supported by protobuf, and we expect to + # have such cases. If we go too far, we will get a proper error anyways. + # We use a raw string (single quotes) to avoid the need to escape special + # characters as this is a regex. + 'ignore:Protobuf gencode version .*exactly one major version older.*:UserWarning', +] testpaths = ["tests", "src"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" diff --git a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/pyproject.toml b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/pyproject.toml index 160f5c5a..03916e86 100644 --- a/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/pyproject.toml +++ b/tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/pyproject.toml @@ -149,7 +149,18 @@ disable = [ ] [tool.pytest.ini_options] -addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv" +addopts = "-vv" +filterwarnings = [ + "error", + "once::DeprecationWarning", + "once::PendingDeprecationWarning", + # We ignore warnings about protobuf gencode version being one version older + # than the current version, as this is supported by protobuf, and we expect to + # have such cases. If we go too far, we will get a proper error anyways. + # We use a raw string (single quotes) to avoid the need to escape special + # characters as this is a regex. + 'ignore:Protobuf gencode version .*exactly one major version older.*:UserWarning', +] testpaths = ["tests", "src"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" diff --git a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/pyproject.toml b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/pyproject.toml index f841ae70..bcd2216c 100644 --- a/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/pyproject.toml +++ b/tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/pyproject.toml @@ -153,7 +153,18 @@ disable = [ ] [tool.pytest.ini_options] -addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv" +addopts = "-vv" +filterwarnings = [ + "error", + "once::DeprecationWarning", + "once::PendingDeprecationWarning", + # We ignore warnings about protobuf gencode version being one version older + # than the current version, as this is supported by protobuf, and we expect to + # have such cases. If we go too far, we will get a proper error anyways. + # We use a raw string (single quotes) to avoid the need to escape special + # characters as this is a regex. + 'ignore:Protobuf gencode version .*exactly one major version older.*:UserWarning', +] testpaths = ["tests", "src"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" From b95e15611d9f5e680f0d5a4983bcefd3008601d0 Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Tue, 3 Jun 2025 10:59:27 +0200 Subject: [PATCH 2/2] Update version for the migration script in the release notes Signed-off-by: Leandro Lucarella --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index db808f03..6fbb0c5c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -13,7 +13,7 @@ All upgrading should be done via the migration script or regenerating the templates. ```bash -curl -sSL https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/v0.12/cookiecutter/migrate.py | python3 +curl -sSL https://raw.githubusercontent.com/frequenz-floss/frequenz-repo-config-python/v0.14/cookiecutter/migrate.py | python3 ``` But you might still need to adapt your code: