feat: provide access to arbitrary interpreters

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/interpreter,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/interpreter,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

docs/toolchains.md

@@ -313,7 +313,7 @@ provide `Python.h`.
 
 This is typically implemented using {obj}`py_cc_toolchain()`, which provides
 {obj}`ToolchainInfo` with the field `py_cc_toolchain` set, which is a
-{obj}`PyCcToolchainInfo` provider instance. 
+{obj}`PyCcToolchainInfo` provider instance.
 
 This toolchain type is intended to hold only _target configuration_ values
 relating to the C/C++ information for the Python runtime. As such, when defining
@@ -467,3 +467,35 @@ Currently the following flags are used to influence toolchain selection:
 * {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant.
 * {obj}`--@rules_python//python/config_settings:py_freethreaded` for selecting
   the freethreaded experimental Python builds available from `3.13.0` onwards.
+
+## Accessing the underlying interpreter
+
+To access the interpreter that bazel manages, you can use the
+`@rules_python//python/bin:interpreter` target. This is a binary target with
+the executable pointing at the `python3` binary plus the relevent runfiles.
+
+```
+$ bazel run @rules_python//python/bin:interpreter
+Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>>
+$ bazel run @rules_python//python/bin:interpreter --@rules_python//python/config_settings:python_version=3.12
+Python 3.12.0 (main, Oct  3 2023, 01:27:23) [Clang 17.0.1 ] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>>
+```
+
+You can also access a specific binary's interpreter this way by using the
+`@rules_python//python/bin:interpreter_src` target.
+
+```
+$ bazel run @rules_python//python/bin:interpreter --@rules_python//python/bin:interpreter_src=//path/to:bin
+Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>>
+```
+
+:::{note}
+The interpreter target does not provide access to any modules from `py_*`
+targets on its own. Work is ongoing to support that.
+:::

python/BUILD.bazel

@@ -35,6 +35,7 @@ filegroup(
     name = "distribution",
     srcs = glob(["**"]) + [
         "//python/api:distribution",
+        "//python/bin:distribution",
         "//python/cc:distribution",
         "//python/config_settings:distribution",
         "//python/constraints:distribution",

python/bin/BUILD.bazel

@@ -0,0 +1,22 @@
+load("//python:defs.bzl", "py_binary")
+load(":interpreter.bzl", "interpreter")
+
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "distribution",
+    srcs = glob(["**"]),
+    visibility = ["//python:__pkg__"],
+)
+
+interpreter(
+    name = "interpreter",
+    binary = ":interpreter_src",
+)
+
+# The user can modify this flag to source different interpreters for the
+# `interpreter` target above.
+label_flag(
+    name = "interpreter_src",
+    build_setting_default = "//python:none",
+)

python/bin/interpreter.bzl

@@ -0,0 +1,40 @@
+load("@bazel_skylib//lib:paths.bzl", "paths")
+load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
+load("//python:py_runtime_info.bzl", "PyRuntimeInfo")
+load("//python/private:sentinel.bzl", "SentinelInfo")
+load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE")
+
+def _interpreter_impl(ctx):
+    if SentinelInfo in ctx.attr.binary:
+        toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE]
+        runtime = toolchain.py3_runtime
+    else:
+        runtime = ctx.attr.binary[PyRuntimeInfo]
+
+    # NOTE: We name the output filename after the underlying file name
+    # because of things like pyenv: they use $0 to determine what to
+    # re-exec. If it's not a recognized name, then they fail.
+    if runtime.interpreter:
+        executable = ctx.actions.declare_file(runtime.interpreter.basename)
+        ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True)
+    else:
+        executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path))
+        ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path)
+
+    return [
+        DefaultInfo(
+            executable = executable,
+            runfiles = ctx.runfiles([executable], transitive_files = runtime.files),
+        ),
+    ]
+
+interpreter = rule(
+    implementation = _interpreter_impl,
+    toolchains = [TARGET_TOOLCHAIN_TYPE],
+    executable = True,
+    attrs = {
+        "binary": attr.label(
+            mandatory = True,
+        ),
+    },
+)

tests/integration/BUILD.bazel

@@ -120,6 +120,11 @@ rules_python_integration_test(
     py_main = "custom_commands_test.py",
 )
 
+rules_python_integration_test(
+    name = "interpreter_test",
+    py_main = "interpreter_test.py",
+)
+
 py_library(
     name = "runner_lib",
     srcs = ["runner.py"],

tests/integration/interpreter/BUILD.bazel

@@ -0,0 +1,13 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/integration/interpreter/MODULE.bazel

@@ -0,0 +1,21 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module(name = "module_under_test")
+
+bazel_dep(name = "rules_python", version = "0.0.0")
+local_path_override(
+    module_name = "rules_python",
+    path = "../../..",
+)

tests/integration/interpreter_test.py

@@ -0,0 +1,74 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import unittest
+
+from tests.integration import runner
+
+
+class InterpreterTest(runner.TestCase):
+    def _run_version_test(self, expected_version):
+        """Validates that we can successfully execute arbitrary code from the CLI."""
+        result = self.run_bazel(
+            "run",
+            f"--@rules_python//python/config_settings:python_version={expected_version}",
+            "@rules_python//python/bin:interpreter",
+            input = "\r".join([
+                "import sys",
+                "v = sys.version_info",
+                "print(f'version: {v.major}.{v.minor}')",
+            ]),
+        )
+        self.assert_result_matches(result, f"version: {expected_version}")
+
+    def test_run_interpreter_3_10(self):
+        self._run_version_test("3.10")
+
+    def test_run_interpreter_3_11(self):
+        self._run_version_test("3.11")
+
+    def test_run_interpreter_3_12(self):
+        self._run_version_test("3.12")
+
+    def _run_module_test(self, version):
+        """Validates that we can successfully invoke a module from the CLI."""
+        # Pass unformatted JSON to the json.tool module.
+        result = self.run_bazel(
+            "run",
+            f"--@rules_python//python/config_settings:python_version={version}",
+            "@rules_python//python/bin:interpreter",
+            "--",
+            "-m",
+            "json.tool",
+            input = '{"json":"obj"}',
+        )
+        # Validate that we get formatted JSON back.
+        self.assert_result_matches(result, r'{\n    "json": "obj"\n}')
+
+    def test_run_module_3_10(self):
+        self._run_module_test("3.10")
+
+    def test_run_module_3_11(self):
+        self._run_module_test("3.11")
+
+    def test_run_module_3_12(self):
+        self._run_module_test("3.12")
+
+
+
+if __name__ == "__main__":
+    # Enabling this makes the runner log subprocesses as the test goes along.
+    # logging.basicConfig(level = "INFO")
+    unittest.main()

tests/integration/runner.py

@@ -86,7 +86,7 @@ def setUp(self):
             "RUNFILES_DIR": os.environ["TEST_SRCDIR"]
         }
 
-    def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult:
+    def run_bazel(self, *args: str, input=None, check: bool = True) -> ExecuteResult:
         """Run a bazel invocation.
 
         Args:
@@ -104,6 +104,7 @@ def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult:
         proc_result = subprocess.run(
             args=args,
             text=True,
+            input=input,
             capture_output=True,
             cwd=cwd,
             env=env,