Pypi updates to the bzlmod migration script (#6584)

This PR involves:
* Ignore repos starting with `pypi_` from direct deps since they are
usually not referenced directly. If referenced directly, these repos
will be fixed in the second part of the script ("parsing the error"
part).
* If referenced repo starts with `pypi_<name>`, raise an error since it
needs to be changed to `pypi//<name>`.
* Update of a python test to cover these new cases.
* Update of golden files in the test environment.
* Smaller updates to the script, usually due to readability.

Author: kotlaja

.bazelci/migration_tool_presubmit.yml

@@ -16,7 +16,7 @@ tasks:
     name: python
     platform: ${{ platform }}
     shell_commands:
-      - cd tools/bzlmod_migration_test_examples/py_extension && python3 ../migration_test.py
+      - cd tools/bzlmod_migration_test_examples/py_extension && python3 migration_test.py
   simple_deps_extension:
     name: simple_deps
     platform: ${{ platform }}

tools/bzlmod_migration_test_examples/migration_test.py

@@ -7,7 +7,7 @@
 
 class BazelBuildTest(unittest.TestCase):
     """
-    A test suite for verifying Bzlmod migration tool for maven and python extensions.
+    A test suite for verifying Bzlmod migration tool for maven and module extensions.
     """
 
     _CREATED_FILES = [
@@ -17,16 +17,21 @@ class BazelBuildTest(unittest.TestCase):
         "migration_info.md",
         "query_direct_deps",
         "resolved_deps.py",
-        "extension_for_",
     ]
 
     def _cleanup_created_files(self):
         """
         Remove files which were created by migration tool.
         """
+        for file_name in self._CREATED_FILES:
+            file_path = os.path.join(os.getcwd(), file_name)
+            if os.path.exists(file_path):
+                os.remove(file_path)
+
+        # Remove all created extensions.
         my_dir = os.getcwd()
         for fname in os.listdir(my_dir):
-            if any(fname.startswith(p) for p in self._CREATED_FILES):
+            if fname.startswith("extension_for_"):
                 os.remove(os.path.join(my_dir, fname))
 
     def _run_command(self, command):

tools/bzlmod_migration_test_examples/py_extension/BUILD

@@ -16,3 +16,8 @@ py_binary(
         "@my_deps_2//attrs"
     ],
 )
+
+alias(
+    name = "yamllint",
+    actual = "@pypi_yamllint//:rules_python_wheel_entry_point_yamllint",
+)

tools/bzlmod_migration_test_examples/py_extension/MODULE.bazel.golden

@@ -1,23 +1,30 @@
 module(name = "py_extension", version="")
 
-bazel_dep(name = "rules_python", version = "1.6.3")
+bazel_dep(name = "rules_python", version = "1.7.0")
 # -- bazel_dep definitions -- #
 
 # -- use_repo_rule statements -- #
 
 
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 
+pip.parse(
+    hub_name = "pypi",
+    requirements_lock = "//:requirements_lock_pypi.txt",
+    python_version = "3.11",
+)
+use_repo(pip, "pypi")
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.defaults(python_version = "3.11")
+python.toolchain(python_version = "3.11")
+
 pip.parse(
     hub_name = "my_deps_2",
     requirements_lock = "//:requirements_lock_2.txt",
     python_version = "3.11",
 )
 use_repo(pip, "my_deps_2")
 
-python = use_extension("@rules_python//python/extensions:python.bzl", "python")
-python.defaults(python_version = "3.11")
-python.toolchain(python_version = "3.11")
 
 pip.parse(
     hub_name = "my_deps_1",
@@ -26,7 +33,6 @@ pip.parse(
 )
 use_repo(pip, "my_deps_1")
 
-
 # -- End of pip extensions -- #
 
 # -- repo definitions -- #

tools/bzlmod_migration_test_examples/py_extension/WORKSPACE

@@ -33,12 +33,21 @@ pip_parse(
     requirements_lock = "@py_extension//:requirements_lock_2.txt",
 )
 
+pip_parse(
+    name = "pypi",
+    python_interpreter_target = "@python_3_11_host//:python",
+    requirements_lock = "//:requirements_lock_pypi.txt",
+)
+
 load("@my_deps_1//:requirements.bzl", install_deps_1 = "install_deps")
 install_deps_1()
 
 load("@my_deps_2//:requirements.bzl", install_deps_2 = "install_deps")
 install_deps_2()
 
+load("@pypi//:requirements.bzl", install_deps_pypi = "install_deps")
+install_deps_pypi()
+
 # Python toolchains
 # -------------------
 load("@rules_python//python:repositories.bzl", "python_register_toolchains", "python_register_multi_toolchains")

tools/bzlmod_migration_test_examples/py_extension/migration_info.md.golden

@@ -4,9 +4,11 @@ Command for local testing:
 bazel build --enable_bzlmod --noenable_workspace //...
 ```
 ## Direct dependencies:
+* pypi
+* rules_python
+* pypi_yamllint
 * my_deps_2
 * my_deps_1
-* rules_python
 ## Migration of `rules_python`:
 
 <details>
@@ -15,9 +17,9 @@ bazel build --enable_bzlmod --noenable_workspace //...
 #### Location
 ```python
 Repository rules_python instantiated at:
-  /usr/local/google/home/kotlaja/github/bazel-central-registry/tools/bzlmod_migration_test_examples/py_extension/WORKSPACE:9:13: in <toplevel>
+  /usr/local/google/home/kotlaja/github/bazel_central_registry_at_head/bazel-central-registry/tools/bzlmod_migration_test_examples/py_extension/WORKSPACE:9:13: in <toplevel>
 Repository rule http_archive defined at:
-  /usr/local/google/home/kotlaja/.cache/bazel/_bazel_kotlaja/5f3795b531c0dd1f2cc34c69d9c35fed/external/bazel_tools/tools/build_defs/repo/http.bzl:387:31: in <toplevel>
+  /usr/local/google/home/kotlaja/.cache/bazel/_bazel_kotlaja/1236e0e34acc2577fcfb7bd9f570e739/external/bazel_tools/tools/build_defs/repo/http.bzl:387:31: in <toplevel>
 
 ```
 
@@ -41,35 +43,22 @@ Found partially name matches in BCR: `rules_python_gazelle_plugin`
 
 It has been introduced as a Bazel module:
 
-	bazel_dep(name = "rules_python", version = "1.6.3")
-## Migration of `my_deps_2`
+	bazel_dep(name = "rules_python", version = "1.7.0")
+## Migration of `pypi`
 It has been introduced as a python extension, with python_version=3.11:
 
 ```
 pip.parse(
-    hub_name = "my_deps_2",
-    requirements_lock = "//:requirements_lock_2.txt",
+    hub_name = "pypi",
+    requirements_lock = "//:requirements_lock_pypi.txt",
     python_version = "3.11",
 )
-use_repo(pip, "my_deps_2")
-
+use_repo(pip, "pypi")
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.defaults(python_version = "3.11")
 python.toolchain(python_version = "3.11")
 ```
-## Migration of `my_deps_1`
-It has been introduced as a python extension, with python_version=3.11:
 
-```
-pip.parse(
-    hub_name = "my_deps_1",
-    requirements_lock = "//:requirements_lock_1.txt",
-    python_version = "3.11",
-)
-use_repo(pip, "my_deps_1")
-
-
-```
 ## Migration of `rules_python`:
 
 <details>
@@ -78,9 +67,9 @@ use_repo(pip, "my_deps_1")
 #### Location
 ```python
 Repository rules_python instantiated at:
-  /usr/local/google/home/kotlaja/github/bazel-central-registry/tools/bzlmod_migration_test_examples/py_extension/WORKSPACE:9:13: in <toplevel>
+  /usr/local/google/home/kotlaja/github/bazel_central_registry_at_head/bazel-central-registry/tools/bzlmod_migration_test_examples/py_extension/WORKSPACE:9:13: in <toplevel>
 Repository rule http_archive defined at:
-  /usr/local/google/home/kotlaja/.cache/bazel/_bazel_kotlaja/5f3795b531c0dd1f2cc34c69d9c35fed/external/bazel_tools/tools/build_defs/repo/http.bzl:387:31: in <toplevel>
+  /usr/local/google/home/kotlaja/.cache/bazel/_bazel_kotlaja/1236e0e34acc2577fcfb7bd9f570e739/external/bazel_tools/tools/build_defs/repo/http.bzl:387:31: in <toplevel>
 
 ```
 
@@ -103,3 +92,29 @@ Found perfect name match in BCR: `rules_python`
 Found partially name matches in BCR: `rules_python_gazelle_plugin`
 
 This module has already been added inside the MODULE.bazel file
+## Migration of `my_deps_2`
+It has been introduced as a python extension, with python_version=3.11:
+
+```
+pip.parse(
+    hub_name = "my_deps_2",
+    requirements_lock = "//:requirements_lock_2.txt",
+    python_version = "3.11",
+)
+use_repo(pip, "my_deps_2")
+
+```
+
+## Migration of `my_deps_1`
+It has been introduced as a python extension, with python_version=3.11:
+
+```
+pip.parse(
+    hub_name = "my_deps_1",
+    requirements_lock = "//:requirements_lock_1.txt",
+    python_version = "3.11",
+)
+use_repo(pip, "my_deps_1")
+
+```
+

tools/bzlmod_migration_test_examples/py_extension/migration_test.py

@@ -0,0 +1,115 @@
+import unittest
+import subprocess
+import os
+import sys
+from unittest import main
+
+
+class BazelBuildTest(unittest.TestCase):
+    """
+    A test suite for verifying Bzlmod migration tool for python extensions.
+    """
+
+    _CREATED_FILES = [
+        "MODULE.bazel",
+        "MODULE.bazel.lock",
+        "WORKSPACE.bzlmod",
+        "migration_info.md",
+        "query_direct_deps",
+        "resolved_deps.py",
+    ]
+
+    def _cleanup_created_files(self):
+        """
+        Remove files which were created by migration tool.
+        """
+        for file_name in self._CREATED_FILES:
+            file_path = os.path.join(os.getcwd(), file_name)
+            if os.path.exists(file_path):
+                os.remove(file_path)
+
+    def _run_command(self, command, expected_failure=False):
+        """
+        Helper function to run a command and return its result.
+        It captures `stdout`, `stderr` and `returncode` for debugging.
+        """
+        try:
+            result = subprocess.run(command, capture_output=True, text=True, check=True)
+            return result
+        except FileNotFoundError:
+            self.fail("Command not found.")
+        except subprocess.CalledProcessError as e:
+            if expected_failure:
+                return e
+            self.fail(f"Command failed with exit code {e.returncode}:\nSTDOUT:\n{e.stdout}\nSTDERR:\n{e.stderr}")
+
+    def _print_message(self, message):
+        GREEN = "\033[92m"
+        RESET = "\033[0m"
+        print(f"{GREEN}{message}{RESET}")
+
+    def modify_build_file(self, old, new):
+        with open("BUILD", "r") as f:
+            original_content = f.read()
+        with open("BUILD", "w") as f:
+            modified_content = str(original_content).replace(old, new)
+            f.write(modified_content)
+
+    def test_migration_of_module_deps(self):
+        self._cleanup_created_files()
+        try:
+            targets = "//..."
+
+            # Verify bazel build is successful with enabled workspace
+            print("\n--- Running bazel build with enabled workspace ---")
+            result = self._run_command(
+                ["bazel", "build", "--nobuild", "--enable_workspace", "--noenable_bzlmod", targets]
+            )
+            assert result.returncode == 0
+            self._print_message("Success.")
+
+            # Run migration script
+            print("\n--- Running migration script ---")
+            result = self._run_command(
+                [sys.executable, "../../migrate_to_bzlmod.py", "-t=//..."], expected_failure=True
+            )
+            assert result.returncode == 1
+            assert "Update pip dependency reference from" in result.stderr
+            assert os.path.exists(
+                "migration_info.md"
+            ), "File 'migration_info.md' should be created during migration, but it doesn't exist."
+
+            self._print_message("Expected error: User need to modify pypi reference.")
+
+            # Verify Bzlmod have error
+            print("\n--- Running bazel build with enabled bzlmod ---")
+            result = self._run_command(
+                ["bazel", "build", "--noenable_workspace", "--enable_bzlmod", "//..."], expected_failure=True
+            )
+            assert result.returncode == 1
+            self._print_message("Expected error: Manual change for pypi reference is needed.")
+
+            # Modify BUILD file
+            self.modify_build_file("@pypi_yamllint//:rules_python_wheel_entry_point_yamllint", "@pypi//yamllint")
+            print("\n--- Modifying BUILD file ---")
+            self._print_message("Success.")
+
+            # Rerun migration script
+            print("\n--- Running migration script ---")
+            result = self._run_command([sys.executable, "../../migrate_to_bzlmod.py", "-t=//..."])
+            assert result.returncode == 0
+            self._print_message("Success.")
+
+            # Verify MODULE.bazel was created successfully
+            print("\n--- Running bazel build with enabled bzlmod ---")
+            result = self._run_command(["bazel", "build", "--noenable_workspace", "--enable_bzlmod", "//..."])
+            assert result.returncode == 0
+            self._print_message("Success.")
+        finally:
+            # Restore BUILD file to the initial content
+            self.modify_build_file("@pypi//yamllint", "@pypi_yamllint//:rules_python_wheel_entry_point_yamllint")
+            self._cleanup_created_files()
+
+
+if __name__ == "__main__":
+    main()

tools/bzlmod_migration_test_examples/py_extension/requirements_lock_pypi.txt

@@ -0,0 +1,3 @@
+yamllint==1.37.1
+PyYAML==6.0.2
+pathspec==0.12.1