feat: add a tool to update internal dependencies (#1321)

Before this change the updates to the dependencies would happen very
seldomly, with this script, I propose we do it before each minor version
release. Adding a shell script and adding a reminder to the release
process may help with that.

Author: aignas

DEVELOPING.md

@@ -1,5 +1,16 @@
 # For Developers
 
+## Updating internal dependencies
+
+1. Modify the `./python/pip_install/tools/requirements.txt` file and run:
+   ```
+   bazel run //tools/private/update_deps:update_pip_deps
+   ```
+1. Bump the coverage dependencies using the script using:
+   ```
+   bazel run //tools/private/update_deps:update_coverage_deps <VERSION>
+   ```
+
 ## Releasing
 
 Start from a clean checkout at `main`.

MODULE.bazel

@@ -15,6 +15,7 @@ internal_deps = use_extension("@rules_python//python/extensions/private:internal
 internal_deps.install()
 use_repo(
     internal_deps,
+    # START: maintained by 'bazel run //tools/private:update_pip_deps'
     "pypi__build",
     "pypi__click",
     "pypi__colorama",
@@ -29,6 +30,7 @@ use_repo(
     "pypi__tomli",
     "pypi__wheel",
     "pypi__zipp",
+    # END: maintained by 'bazel run //tools/private:update_pip_deps'
 )
 
 # We need to do another use_extension call to expose the "pythons_hub"

python/pip_install/BUILD.bazel

@@ -9,6 +9,18 @@ filegroup(
     visibility = ["//:__pkg__"],
 )
 
+filegroup(
+    name = "repositories",
+    srcs = ["repositories.bzl"],
+    visibility = ["//tools/private/update_deps:__pkg__"],
+)
+
+filegroup(
+    name = "requirements_txt",
+    srcs = ["tools/requirements.txt"],
+    visibility = ["//tools/private/update_deps:__pkg__"],
+)
+
 filegroup(
     name = "bzl",
     srcs = glob(["*.bzl"]) + [

python/pip_install/repositories.bzl

@@ -20,6 +20,7 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 load("//:version.bzl", "MINIMUM_BAZEL_VERSION")
 
 _RULE_DEPS = [
+    # START: maintained by 'bazel run //tools/private:update_pip_deps'
     (
         "pypi__build",
         "https://files.pythonhosted.org/packages/03/97/f58c723ff036a8d8b4d3115377c0a37ed05c1f68dd9a0d66dab5e82c5c1c/build-0.9.0-py3-none-any.whl",
@@ -35,11 +36,21 @@ _RULE_DEPS = [
         "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl",
         "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6",
     ),
+    (
+        "pypi__importlib_metadata",
+        "https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl",
+        "bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
+    ),
     (
         "pypi__installer",
         "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl",
         "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53",
     ),
+    (
+        "pypi__more_itertools",
+        "https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl",
+        "c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb",
+    ),
     (
         "pypi__packaging",
         "https://files.pythonhosted.org/packages/8f/7b/42582927d281d7cb035609cd3a543ffac89b74f3f4ee8e1c50914bcb57eb/packaging-22.0-py3-none-any.whl",
@@ -75,21 +86,12 @@ _RULE_DEPS = [
         "https://files.pythonhosted.org/packages/bd/7c/d38a0b30ce22fc26ed7dbc087c6d00851fb3395e9d0dac40bec1f905030c/wheel-0.38.4-py3-none-any.whl",
         "b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8",
     ),
-    (
-        "pypi__importlib_metadata",
-        "https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl",
-        "bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
-    ),
     (
         "pypi__zipp",
         "https://files.pythonhosted.org/packages/f4/50/cc72c5bcd48f6e98219fc4a88a5227e9e28b81637a99c49feba1d51f4d50/zipp-1.0.0-py2.py3-none-any.whl",
         "8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656",
     ),
-    (
-        "pypi__more_itertools",
-        "https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl",
-        "c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb",
-    ),
+    # END: maintained by 'bazel run //tools/private:update_pip_deps'
 ]
 
 _GENERIC_WHEEL = """\

python/pip_install/tools/requirements.txt

@@ -0,0 +1,14 @@
+build==0.9
+click==8.0.1
+colorama
+importlib_metadata==1.4.0
+installer
+more_itertools==8.13.0
+packaging==22.0
+pep517
+pip==22.3.1
+pip_tools==6.12.1
+setuptools==60.10
+tomli
+wheel==0.38.4
+zipp==1.0.0

python/private/BUILD.bazel

@@ -24,6 +24,12 @@ filegroup(
     visibility = ["//python:__pkg__"],
 )
 
+filegroup(
+    name = "coverage_deps",
+    srcs = ["coverage_deps.bzl"],
+    visibility = ["//tools/private/update_deps:__pkg__"],
+)
+
 # Filegroup of bzl files that can be used by downstream rules for documentation generation
 filegroup(
     name = "bzl",

python/private/coverage_deps.bzl

@@ -19,8 +19,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 load("//python/private:version_label.bzl", "version_label")
 
-# Update with './tools/update_coverage_deps.py <version>'
-#START: managed by update_coverage_deps.py script
+# START: maintained by 'bazel run //tools/private:update_coverage_deps'
 _coverage_deps = {
     "cp310": {
         "aarch64-apple-darwin": (
@@ -95,7 +94,7 @@ _coverage_deps = {
         ),
     },
 }
-#END: managed by update_coverage_deps.py script
+# END: maintained by 'bazel run //tools/private:update_coverage_deps'
 
 _coverage_patch = Label("//python/private:coverage.patch")
 

tools/private/update_deps/BUILD.bazel

@@ -0,0 +1,76 @@
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+load("//python:py_binary.bzl", "py_binary")
+load("//python:py_library.bzl", "py_library")
+load("//python:py_test.bzl", "py_test")
+
+licenses(["notice"])
+
+py_library(
+    name = "args",
+    srcs = ["args.py"],
+    imports = ["../../.."],
+    deps = ["//python/runfiles"],
+)
+
+py_library(
+    name = "update_file",
+    srcs = ["update_file.py"],
+    imports = ["../../.."],
+)
+
+py_binary(
+    name = "update_coverage_deps",
+    srcs = ["update_coverage_deps.py"],
+    data = [
+        "//python/private:coverage_deps",
+    ],
+    env = {
+        "UPDATE_FILE": "$(rlocationpath //python/private:coverage_deps)",
+    },
+    imports = ["../../.."],
+    deps = [
+        ":args",
+        ":update_file",
+    ],
+)
+
+py_binary(
+    name = "update_pip_deps",
+    srcs = ["update_pip_deps.py"],
+    data = [
+        "//:MODULE.bazel",
+        "//python/pip_install:repositories",
+        "//python/pip_install:requirements_txt",
+    ],
+    env = {
+        "MODULE_BAZEL": "$(rlocationpath //:MODULE.bazel)",
+        "REPOSITORIES_BZL": "$(rlocationpath //python/pip_install:repositories)",
+        "REQUIREMENTS_TXT": "$(rlocationpath //python/pip_install:requirements_txt)",
+    },
+    imports = ["../../.."],
+    deps = [
+        ":args",
+        ":update_file",
+    ],
+)
+
+py_test(
+    name = "update_file_test",
+    srcs = ["update_file_test.py"],
+    imports = ["../../.."],
+    deps = [
+        ":update_file",
+    ],
+)

tools/private/update_deps/args.py

@@ -0,0 +1,35 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A small library for common arguments when updating files."""
+
+import pathlib
+
+from python.runfiles import runfiles
+
+
+def path_from_runfiles(input: str) -> pathlib.Path:
+    """A helper to create a path from runfiles.
+
+    Args:
+        input: the string input to construct a path.
+
+    Returns:
+        the pathlib.Path path to a file which is verified to exist.
+    """
+    path = pathlib.Path(runfiles.Create().Rlocation(input))
+    if not path.exists():
+        raise ValueError(f"Path '{path}' does not exist")
+
+    return path

tools/update_coverage_deps.py …vate/update_deps/update_coverage_deps.pytools/update_coverage_deps.py renamed to tools/private/update_deps/update_coverage_deps.py tools/update_coverage_deps.py renamed to tools/private/update_deps/update_coverage_deps.py

@@ -22,6 +22,7 @@
 import argparse
 import difflib
 import json
+import os
 import pathlib
 import sys
 import textwrap
@@ -30,6 +31,9 @@
 from typing import Any
 from urllib import request
 
+from tools.private.update_deps.args import path_from_runfiles
+from tools.private.update_deps.update_file import update_file
+
 # This should be kept in sync with //python:versions.bzl
 _supported_platforms = {
     # Windows is unsupported right now
@@ -110,64 +114,6 @@ def _map(
     )
 
 
-def _writelines(path: pathlib.Path, lines: list[str]):
-    with open(path, "w") as f:
-        f.writelines(lines)
-
-
-def _difflines(path: pathlib.Path, lines: list[str]):
-    with open(path) as f:
-        input = f.readlines()
-
-    rules_python = pathlib.Path(__file__).parent.parent
-    p = path.relative_to(rules_python)
-
-    print(f"Diff of the changes that would be made to '{p}':")
-    for line in difflib.unified_diff(
-        input,
-        lines,
-        fromfile=f"a/{p}",
-        tofile=f"b/{p}",
-    ):
-        print(line, end="")
-
-    # Add an empty line at the end of the diff
-    print()
-
-
-def _update_file(
-    path: pathlib.Path,
-    snippet: str,
-    start_marker: str,
-    end_marker: str,
-    dry_run: bool = True,
-):
-    with open(path) as f:
-        input = f.readlines()
-
-    out = []
-    skip = False
-    for line in input:
-        if skip:
-            if not line.startswith(end_marker):
-                continue
-
-            skip = False
-
-        out.append(line)
-
-        if not line.startswith(start_marker):
-            continue
-
-        skip = True
-        out.extend([f"{line}\n" for line in snippet.splitlines()])
-
-    if dry_run:
-        _difflines(path, out)
-    else:
-        _writelines(path, out)
-
-
 def _parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(__doc__)
     parser.add_argument(
@@ -193,6 +139,12 @@ def _parse_args() -> argparse.Namespace:
         action="store_true",
         help="Wether to write to files",
     )
+    parser.add_argument(
+        "--update-file",
+        type=path_from_runfiles,
+        default=os.environ.get("UPDATE_FILE"),
+        help="The path for the file to be updated, defaults to the value taken from UPDATE_FILE",
+    )
     return parser.parse_args()
 
 
@@ -230,14 +182,12 @@ def main():
 
     urls.sort(key=lambda x: f"{x.python}_{x.platform}")
 
-    rules_python = pathlib.Path(__file__).parent.parent
-
     # Update the coverage_deps, which are used to register deps
-    _update_file(
-        path=rules_python / "python" / "private" / "coverage_deps.bzl",
+    update_file(
+        path=args.update_file,
         snippet=f"_coverage_deps = {repr(Deps(urls))}\n",
-        start_marker="#START: managed by update_coverage_deps.py script",
-        end_marker="#END: managed by update_coverage_deps.py script",
+        start_marker="# START: maintained by 'bazel run //tools/private:update_coverage_deps'",
+        end_marker="# END: maintained by 'bazel run //tools/private:update_coverage_deps'",
         dry_run=args.dry_run,
     )