Merge pull request #3234 from CoolCat467/pre-commit-tests-sync

Add tool to keep test-requirements in sync with pre-commit

Author: CoolCat467

.pre-commit-config.yaml

@@ -58,6 +58,13 @@ repos:
         pass_filenames: false
         additional_dependencies: ["astor", "attrs", "black", "ruff"]
         files: ^src\/trio\/_core\/(_run|(_i(o_(common|epoll|kqueue|windows)|nstrumentation)))\.py$
+      - id: sync-test-requirements
+        name: synchronize test requirements
+        language: python
+        entry: python src/trio/_tools/sync_requirements.py
+        pass_filenames: false
+        additional_dependencies: ["pyyaml"]
+        files: ^(test-requirements\.txt)|(\.pre-commit-config\.yaml)$
   - repo: https://github.com/astral-sh/uv-pre-commit
     rev: 0.7.2
     hooks:

src/trio/_tests/tools/test_sync_requirements.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from trio._tests.pytest_plugin import skip_if_optional_else_raise
+
+# imports in gen_exports that are not in `install_requires` in requirements
+try:
+    import yaml  # noqa: F401
+except ImportError as error:
+    skip_if_optional_else_raise(error)
+
+from trio._tools.sync_requirements import (
+    update_requirements,
+    yield_pre_commit_version_data,
+)
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+def test_yield_pre_commit_version_data() -> None:
+    text = """
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.0
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 25.1.0
+"""
+    results = tuple(yield_pre_commit_version_data(text))
+    assert results == (
+        ("ruff-pre-commit", "0.11.0"),
+        ("black-pre-commit-mirror", "25.1.0"),
+    )
+
+
+def test_update_requirements(
+    tmp_path: Path,
+) -> None:
+    requirements_file = tmp_path / "requirements.txt"
+    assert not requirements_file.exists()
+    requirements_file.write_text(
+        """# comment
+  # also comment but spaces line start
+waffles are delicious no equals
+black==3.1.4 ; specific version thingy
+mypy==1.15.0
+ruff==1.2.5
+# required by soupy cat""",
+        encoding="utf-8",
+    )
+    assert update_requirements(requirements_file, {"black": "3.1.5", "ruff": "1.2.7"})
+    assert (
+        requirements_file.read_text(encoding="utf-8")
+        == """# comment
+  # also comment but spaces line start
+waffles are delicious no equals
+black==3.1.5 ; specific version thingy
+mypy==1.15.0
+ruff==1.2.7
+# required by soupy cat"""
+    )
+
+
+def test_update_requirements_no_changes(
+    tmp_path: Path,
+) -> None:
+    requirements_file = tmp_path / "requirements.txt"
+    assert not requirements_file.exists()
+    original = """# comment
+  # also comment but spaces line start
+waffles are delicious no equals
+black==3.1.4 ; specific version thingy
+mypy==1.15.0
+ruff==1.2.5
+# required by soupy cat"""
+    requirements_file.write_text(original, encoding="utf-8")
+    assert not update_requirements(
+        requirements_file, {"black": "3.1.4", "ruff": "1.2.5"}
+    )
+    assert requirements_file.read_text(encoding="utf-8") == original

src/trio/_tools/sync_requirements.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+"""Sync Requirements - Automatically upgrade test requirements pinned
+versions from pre-commit config file."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from yaml import load as load_yaml
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from yaml import CLoader as _CLoader, Loader as _Loader
+
+    Loader: type[_CLoader | _Loader]
+
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
+
+
+def yield_pre_commit_version_data(
+    pre_commit_text: str,
+) -> Generator[tuple[str, str], None, None]:
+    """Yield (name, rev) tuples from pre-commit config file."""
+    pre_commit_config = load_yaml(pre_commit_text, Loader)
+    for repo in pre_commit_config["repos"]:
+        if "repo" not in repo or "rev" not in repo:
+            continue
+        url = repo["repo"]
+        name = url.rsplit("/", 1)[-1]
+        rev = repo["rev"].removeprefix("v")
+        yield name, rev
+
+
+def update_requirements(
+    requirements: Path,
+    version_data: dict[str, str],
+) -> bool:
+    """Return if updated requirements file.
+
+    Update requirements file to match versions in version_data."""
+    changed = False
+    old_lines = requirements.read_text(encoding="utf-8").splitlines(True)
+
+    with requirements.open("w", encoding="utf-8") as file:
+        for line in old_lines:
+            # If comment or not version mark line, ignore.
+            if line.startswith("#") or "==" not in line:
+                file.write(line)
+                continue
+            name, rest = line.split("==", 1)
+            # Maintain extra markers if they exist
+            old_version = rest.strip()
+            extra = "\n"
+            if ";" in rest:
+                old_version, extra = rest.split(";", 1)
+                old_version = old_version.strip()
+                extra = " ;" + extra
+            version = version_data.get(name)
+            # If does not exist, skip
+            if version is None:
+                file.write(line)
+                continue
+            # Otherwise might have changed
+            new_line = f"{name}=={version}{extra}"
+            if new_line != line:
+                if not changed:
+                    changed = True
+                    print("Changed test requirements version to match pre-commit")
+                print(f"{name}=={old_version} -> {name}=={version}")
+            file.write(new_line)
+    return changed
+
+
+def main() -> int:
+    """Run program."""
+
+    source_root = Path.cwd().absolute()
+
+    # Double-check we found the right directory
+    assert (source_root / "LICENSE").exists()
+    pre_commit = source_root / ".pre-commit-config.yaml"
+    test_requirements = source_root / "test-requirements.txt"
+
+    pre_commit_text = pre_commit.read_text(encoding="utf-8")
+
+    # Get tool versions from pre-commit
+    # Get correct names
+    pre_commit_versions = {
+        name.removesuffix("-mirror").removesuffix("-pre-commit"): version
+        for name, version in yield_pre_commit_version_data(pre_commit_text)
+    }
+    changed = update_requirements(test_requirements, pre_commit_versions)
+    return int(changed)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

test-requirements.in

@@ -27,6 +27,8 @@ types-pyOpenSSL
 # annotations in doc files
 types-docutils
 sphinx
+# sync-requirements
+types-PyYAML
 
 # Trio's own dependencies
 cffi; os_name == "nt"

test-requirements.txt

@@ -179,6 +179,8 @@ types-docutils==0.21.0.20241128
     # via -r test-requirements.in
 types-pyopenssl==24.1.0.20240722
     # via -r test-requirements.in
+types-pyyaml==6.0.12.20250326
+    # via -r test-requirements.in
 types-setuptools==80.0.0.20250429
     # via types-cffi
 typing-extensions==4.13.2