feat(bzlmod): Moving register.toolchains internal (#1238)

This commit moves the register.toolchains bzlmod call to inside
of rules_python.  Instead of a user having to call register.toolchains
in their MODULE.bazel, rules_python/MODULE.bazel calls it
on the internal hub.

This is a breaking change if you are using register.toolchains inside
of submodules.  Using register.toolchains inside of submodules is
not recommended anyways.  This is now broken because we are not
creating a repo for every Python version toolchain.  All of the
toochain calls exist now in the hub's repo BUILD.bazel file.

Author: chrislovecnm

.bazelrc

@@ -3,8 +3,8 @@
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points
 
 test --test_output=errors
 

BZLMOD_SUPPORT.md

@@ -31,7 +31,7 @@ A second example, in [examples/bzlmod_build_file_generation](examples/bzlmod_bui
 
 This rule set does not have full feature partity with the older `WORKSPACE` type configuration:
 
-1. Multiple python versions are not yet supported, as demonstrated in [this](examples/multi_python_versions) example.
+1. Multiple pip extensions are not yet supported, as demonstrated in [this](examples/multi_python_versions) example.
 2. Gazelle does not support finding deps in sub-modules.  For instance we can have a dep like ` "@our_other_module//other_module/pkg:lib",` in a `py_test` definition.
 
 Check ["issues"](/bazelbuild/rules_python/issues) for an up to date list.

MODULE.bazel

@@ -47,5 +47,10 @@ use_repo(
     "pypi__coverage_cp39_x86_64-unknown-linux-gnu",
 )
 
+# We need to do another use_extension call to expose the "pythons_hub"
+# repo.
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 use_repo(python, "pythons_hub")
+
+# This call registers the Python toolchains.
+register_toolchains("@pythons_hub//:all")

README.md

@@ -53,32 +53,39 @@ To import rules_python in your project, you first need to add it to your
 
 To register a hermetic Python toolchain rather than rely on a system-installed interpreter for runtime execution, you can add to the `MODULE.bazel` file:
 
-```python
+```starlark
 # Find the latest version number here: https://github.com/bazelbuild/rules_python/releases
 # and change the version number if needed in the line below.
-bazel_dep(name = "rules_python", version = "0.20.0")
+bazel_dep(name = "rules_python", version = "0.21.0")
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(
+    name = "python",
+    configure_coverage_tool = True,
+    is_default = True,
+    python_version = "3.9",
+)
 
-# You do not have to use pip for the toolchain, but most people
-# will use it for the dependency management.
-pip = use_extension("@rules_python//python:extensions.bzl", "pip")
+interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter")
+interpreter.install(
+    name = "interpreter",
+    python_name = "python",
+)
+use_repo(interpreter, "interpreter")
 
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
     name = "pip",
+    incompatible_generate_aliases = True,
+    python_interpreter_target = "@interpreter//:python",
     requirements_lock = "//:requirements_lock.txt",
+    requirements_windows = "//:requirements_windows.txt",
 )
-
 use_repo(pip, "pip")
-
-# Register a specific python toolchain instead of using the host version
-python = use_extension("@rules_python//python:extensions.bzl", "python")
-
-use_repo(python, "python3_10_toolchains")
-
-register_toolchains(
-    "@python3_10_toolchains//:all",
-)
 ```
 
+For more documentation see the bzlmod examples under the [examples](examples) folder.
+
 ### Using a WORKSPACE file
 
 To import rules_python in your project, you first need to add it to your

examples/bzlmod/BUILD.bazel

@@ -8,9 +8,6 @@
 load("@bazel_skylib//rules:build_test.bzl", "build_test")
 load("@pip//:requirements.bzl", "all_requirements", "all_whl_requirements", "requirement")
 load("@python_39//:defs.bzl", py_test_with_transition = "py_test")
-
-# This is not working yet till the toolchain hub registration is working
-# load("@python_310//:defs.bzl", py_binary_310 = "py_binary")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 
@@ -50,22 +47,6 @@ py_binary(
     ],
 )
 
-# This is still WIP.  Not working till we have the toolchain
-# registration functioning.
-
-# This is used for testing mulitple versions of Python. This is
-# used only when you need to support multiple versions of Python
-# in the same project.
-# py_binary_310(
-#     name = "main_310",
-#     srcs = ["__main__.py"],
-#     main = "__main__.py",
-#     visibility = ["//:__subpackages__"],
-#     deps = [
-#         ":lib",
-#     ],
-# )
-
 # see https://bazel.build/reference/be/python#py_test
 py_test(
     name = "test",

examples/bzlmod/MODULE.bazel

@@ -15,14 +15,10 @@ local_path_override(
 # We also use the same value in the python.host_python_interpreter call.
 PYTHON_NAME_39 = "python_39"
 
-PYTHON_39_TOOLCHAINS = PYTHON_NAME_39 + "_toolchains"
-
 INTERPRETER_NAME_39 = "interpreter_39"
 
 PYTHON_NAME_310 = "python_310"
 
-PYTHON_310_TOOLCHAINS = PYTHON_NAME_310 + "_toolchains"
-
 INTERPRETER_NAME_310 = "interpreter_310"
 
 # We next initialize the python toolchain using the extension.
@@ -50,25 +46,9 @@ python.toolchain(
     python_version = "3.10",
 )
 
-# use_repo imports one or more repos generated by the given module extension
-# into the scope of the current module. We are importing the various repos
-# created by the above python.toolchain calls.
-use_repo(
-    python,
-    PYTHON_NAME_39,
-    PYTHON_39_TOOLCHAINS,
-    PYTHON_NAME_310,
-    PYTHON_310_TOOLCHAINS,
-)
-
-# This call registers the Python toolchains.
-# Note: there is work under way to move this code to within
-# rules_python, and the user won't have to make this call,
-# unless they are registering custom toolchains.
-register_toolchains(
-    "@{}//:all".format(PYTHON_39_TOOLCHAINS),
-    "@{}//:all".format(PYTHON_310_TOOLCHAINS),
-)
+# You only need to load this repositories if you are using muiltple Python versions.
+# See the tests folder for various examples.
+use_repo(python, PYTHON_NAME_39, "python_aliases")
 
 # The interpreter extension discovers the platform specific Python binary.
 # It creates a symlink to the binary, and we pass the label to the following

examples/bzlmod/other_module/MODULE.bazel

@@ -12,12 +12,8 @@ bazel_dep(name = "rules_python", version = "")
 # testing purposes.
 PYTHON_NAME_39 = "python_39"
 
-PYTHON_39_TOOLCHAINS = PYTHON_NAME_39 + "_toolchains"
-
 PYTHON_NAME_311 = "python_311"
 
-PYTHON_311_TOOLCHAINS = PYTHON_NAME_311 + "_toolchains"
-
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 python.toolchain(
     # This name is used in the various use_repo statements
@@ -42,16 +38,5 @@ python.toolchain(
 use_repo(
     python,
     PYTHON_NAME_39,
-    PYTHON_39_TOOLCHAINS,
     PYTHON_NAME_311,
-    PYTHON_311_TOOLCHAINS,
-)
-
-# This call registers the Python toolchains.
-# Note: there is work under way to move this code to within
-# rules_python, and the user won't have to make this call,
-# unless they are registering custom toolchains.
-register_toolchains(
-    "@{}//:all".format(PYTHON_39_TOOLCHAINS),
-    "@{}//:all".format(PYTHON_311_TOOLCHAINS),
 )

examples/bzlmod/tests/BUILD.bazel

@@ -0,0 +1,142 @@
+load("@python_aliases//3.10:defs.bzl", py_binary_3_10 = "py_binary", py_test_3_10 = "py_test")
+load("@python_aliases//3.11:defs.bzl", py_binary_3_11 = "py_binary", py_test_3_11 = "py_test")
+load("@python_aliases//3.9:defs.bzl", py_binary_3_9 = "py_binary", py_test_3_9 = "py_test")
+load("@rules_python//python:defs.bzl", "py_binary", "py_test")
+
+py_binary(
+    name = "version_default",
+    srcs = ["version.py"],
+    main = "version.py",
+)
+
+py_binary_3_9(
+    name = "version_3_9",
+    srcs = ["version.py"],
+    main = "version.py",
+)
+
+py_binary_3_10(
+    name = "version_3_10",
+    srcs = ["version.py"],
+    main = "version.py",
+)
+
+py_binary_3_11(
+    name = "version_3_11",
+    srcs = ["version.py"],
+    main = "version.py",
+)
+
+# This is a work in progress and the commented
+# tests will not work  until we can support
+# multiple pips with bzlmod.
+
+#py_test(
+#    name = "my_lib_default_test",
+#    srcs = ["my_lib_test.py"],
+#    main = "my_lib_test.py",
+#    deps = ["//libs/my_lib"],
+#)
+
+#py_test_3_9(
+#    name = "my_lib_3_9_test",
+#    srcs = ["my_lib_test.py"],
+#    main = "my_lib_test.py",
+#    deps = ["//libs/my_lib"],
+#)
+
+#py_test_3_10(
+#    name = "my_lib_3_10_test",
+#    srcs = ["my_lib_test.py"],
+#    main = "my_lib_test.py",
+#    deps = ["//libs/my_lib"],
+#)
+
+#py_test_3_11(
+#    name = "my_lib_3_11_test",
+#    srcs = ["my_lib_test.py"],
+#    main = "my_lib_test.py",
+#    deps = ["//libs/my_lib"],
+#)
+
+py_test(
+    name = "version_default_test",
+    srcs = ["version_test.py"],
+    env = {"VERSION_CHECK": "3.9"},  # The default defined in the WORKSPACE.
+    main = "version_test.py",
+)
+
+py_test_3_9(
+    name = "version_3_9_test",
+    srcs = ["version_test.py"],
+    env = {"VERSION_CHECK": "3.9"},
+    main = "version_test.py",
+)
+
+py_test_3_10(
+    name = "version_3_10_test",
+    srcs = ["version_test.py"],
+    env = {"VERSION_CHECK": "3.10"},
+    main = "version_test.py",
+)
+
+py_test_3_11(
+    name = "version_3_11_test",
+    srcs = ["version_test.py"],
+    env = {"VERSION_CHECK": "3.11"},
+    main = "version_test.py",
+)
+
+py_test(
+    name = "version_default_takes_3_10_subprocess_test",
+    srcs = ["cross_version_test.py"],
+    data = [":version_3_10"],
+    env = {
+        "SUBPROCESS_VERSION_CHECK": "3.10",
+        "SUBPROCESS_VERSION_PY_BINARY": "$(rootpath :version_3_10)",
+        "VERSION_CHECK": "3.9",
+    },
+    main = "cross_version_test.py",
+)
+
+py_test_3_10(
+    name = "version_3_10_takes_3_9_subprocess_test",
+    srcs = ["cross_version_test.py"],
+    data = [":version_3_9"],
+    env = {
+        "SUBPROCESS_VERSION_CHECK": "3.9",
+        "SUBPROCESS_VERSION_PY_BINARY": "$(rootpath :version_3_9)",
+        "VERSION_CHECK": "3.10",
+    },
+    main = "cross_version_test.py",
+)
+
+sh_test(
+    name = "version_test_binary_default",
+    srcs = ["version_test.sh"],
+    data = [":version_default"],
+    env = {
+        "VERSION_CHECK": "3.9",  # The default defined in the WORKSPACE.
+        "VERSION_PY_BINARY": "$(rootpath :version_default)",
+    },
+)
+
+sh_test(
+    name = "version_test_binary_3_9",
+    srcs = ["version_test.sh"],
+    data = [":version_3_9"],
+    env = {
+        "VERSION_CHECK": "3.9",
+        "VERSION_PY_BINARY": "$(rootpath :version_3_9)",
+    },
+)
+
+sh_test(
+    name = "version_test_binary_3_10",
+    srcs = ["version_test.sh"],
+    data = [":version_3_10"],
+    env = {
+        "VERSION_CHECK": "3.10",
+        "VERSION_PY_BINARY": "$(rootpath :version_3_10)",
+    },
+)

examples/bzlmod/tests/cross_version_test.py

@@ -0,0 +1,39 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import sys
+
+process = subprocess.run(
+    [os.getenv("SUBPROCESS_VERSION_PY_BINARY")],
+    stdout=subprocess.PIPE,
+    universal_newlines=True,
+)
+
+subprocess_current = process.stdout.strip()
+subprocess_expected = os.getenv("SUBPROCESS_VERSION_CHECK")
+
+if subprocess_current != subprocess_expected:
+    print(
+        f"expected subprocess version '{subprocess_expected}' is different than returned '{subprocess_current}'"
+    )
+    sys.exit(1)
+
+expected = os.getenv("VERSION_CHECK")
+current = f"{sys.version_info.major}.{sys.version_info.minor}"
+
+if current != expected:
+    print(f"expected version '{expected}' is different than returned '{current}'")
+    sys.exit(1)

examples/bzlmod/tests/version.py

@@ -0,0 +1,17 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+print(f"{sys.version_info.major}.{sys.version_info.minor}")