diff --git a/docs/api/rules_python/python/bin/index.md b/docs/api/rules_python/python/bin/index.md new file mode 100644 index 0000000000..ad6a4e7ed5 --- /dev/null +++ b/docs/api/rules_python/python/bin/index.md @@ -0,0 +1,41 @@ +:::{default-domain} bzl +::: +:::{bzl:currentfile} //python/bin:BUILD.bazel +::: + +# //python/bin + +:::{bzl:target} python + +A target to directly run a Python interpreter. + +By default, it uses the Python version that toolchain resolution matches +(typically the one marked `is_default=True` in `MODULE.bazel`). + +This runs a Python interpreter in a similar manner as when running `python3` +on the command line. It can be invoked using `bazel run`. Remember that in +order to pass flags onto the program `--` must be specified to separate +Bazel flags from the program flags. + +An example that will run Python 3.12 and have it print the version + +``` +bazel run @rules_python//python/bin:python \ + `--@rule_python//python/config_settings:python_verion=3.12 \ + -- \ + --version +``` + +::::{seealso} +The {flag}`--python_src` flag for using the intepreter a binary/test uses. +:::: + +::::{versionadded} VERSION_NEXT_FEATURE +:::: +::: + +:::{bzl:flag} python_src + +The target (one providing `PyRuntimeInfo`) whose python interpreter to use for +{obj}`:python`. +::: diff --git a/docs/toolchains.md b/docs/toolchains.md index 6eaa244b1f..3294c1732a 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -396,7 +396,7 @@ provide `Python.h`. This is typically implemented using {obj}`py_cc_toolchain()`, which provides {obj}`ToolchainInfo` with the field `py_cc_toolchain` set, which is a -{obj}`PyCcToolchainInfo` provider instance. +{obj}`PyCcToolchainInfo` provider instance. This toolchain type is intended to hold only _target configuration_ values relating to the C/C++ information for the Python runtime. As such, when defining @@ -556,4 +556,45 @@ of available toolchains. Currently the following flags are used to influence toolchain selection: * {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant. * {obj}`--@rules_python//python/config_settings:py_freethreaded` for selecting - the freethreaded experimental Python builds available from `3.13.0` onwards. \ No newline at end of file + the freethreaded experimental Python builds available from `3.13.0` onwards. + +## Running the underlying interpreter + +To run the interpreter that Bazel will use, you can use the +`@rules_python//python/bin:python` target. This is a binary target with +the executable pointing at the `python3` binary plus its relevent runfiles. + +```console +$ bazel run @rules_python//python/bin:python +Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +$ bazel run @rules_python//python/bin:python --@rules_python//python/config_settings:python_version=3.12 +Python 3.12.0 (main, Oct 3 2023, 01:27:23) [Clang 17.0.1 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` + +You can also access a specific binary's interpreter this way by using the +`@rules_python//python/bin:python_src` target. In the example below, it is +assumed that the `@rules_python//tools/publish:twine` binary is fixed at Python +3.11. + +```console +$ bazel run @rules_python//python/bin:python --@rules_python//python/bin:interpreter_src=@rules_python//tools/publish:twine +Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +$ bazel run @rules_python//python/bin:python --@rules_python//python/bin:interpreter_src=@rules_python//tools/publish:twine --@rules_python//python/config_settings:python_version=3.12 +Python 3.11.1 (main, Jan 16 2023, 22:41:20) [Clang 15.0.7 ] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` +Despite setting the Python version explicitly to 3.12 in the example above, the +interpreter comes from the `@rules_python//tools/publish:twine` binary. That is +a fixed version. + +:::{note} +The `python` target does not provide access to any modules from `py_*` +targets on its own. Please file a feature request if this is desired. +::: diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 5c6c6a4175..c52e772666 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -35,6 +35,7 @@ filegroup( name = "distribution", srcs = glob(["**"]) + [ "//python/api:distribution", + "//python/bin:distribution", "//python/cc:distribution", "//python/config_settings:distribution", "//python/constraints:distribution", diff --git a/python/bin/BUILD.bazel b/python/bin/BUILD.bazel new file mode 100644 index 0000000000..57bee34378 --- /dev/null +++ b/python/bin/BUILD.bazel @@ -0,0 +1,24 @@ +load("//python/private:interpreter.bzl", _interpreter_binary = "interpreter_binary") + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//:__subpackages__"], +) + +_interpreter_binary( + name = "python", + binary = ":python_src", + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], +) + +# The user can modify this flag to source different interpreters for the +# `python` target above. +label_flag( + name = "python_src", + build_setting_default = "//python:none", +) diff --git a/python/private/common.bzl b/python/private/common.bzl index b6a54532d3..137f0d23f3 100644 --- a/python/private/common.bzl +++ b/python/private/common.bzl @@ -543,3 +543,20 @@ def target_platform_has_any_constraint(ctx, constraints): if ctx.target_platform_has_constraint(constraint_value): return True return False + +def runfiles_root_path(ctx, short_path): + """Compute a runfiles-root relative path from `File.short_path` + + Args: + ctx: current target ctx + short_path: str, a main-repo relative path from `File.short_path` + + Returns: + {type}`str`, a runflies-root relative path + """ + + # The ../ comes from short_path is for files in other repos. + if short_path.startswith("../"): + return short_path[3:] + else: + return "{}/{}".format(ctx.workspace_name, short_path) diff --git a/python/private/interpreter.bzl b/python/private/interpreter.bzl new file mode 100644 index 0000000000..c66d3dc21e --- /dev/null +++ b/python/private/interpreter.bzl @@ -0,0 +1,82 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of the rules to access the underlying Python interpreter.""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("//python:py_runtime_info.bzl", "PyRuntimeInfo") +load(":common.bzl", "runfiles_root_path") +load(":sentinel.bzl", "SentinelInfo") +load(":toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") + +def _interpreter_binary_impl(ctx): + if SentinelInfo in ctx.attr.binary: + toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE] + runtime = toolchain.py3_runtime + else: + runtime = ctx.attr.binary[PyRuntimeInfo] + + # NOTE: We name the output filename after the underlying file name + # because of things like pyenv: they use $0 to determine what to + # re-exec. If it's not a recognized name, then they fail. + if runtime.interpreter: + # In order for this to work both locally and remotely, we create a + # shell script here that re-exec's into the real interpreter. Ideally, + # we'd just use a symlink, but that breaks under certain conditions. If + # we use a ctx.actions.symlink(target=...) then it fails under remote + # execution. If we use ctx.actions.symlink(target_path=...) then it + # behaves differently inside the runfiles tree and outside the runfiles + # tree. + # + # This currently does not work on Windows. Need to find a way to enable + # that. + executable = ctx.actions.declare_file(runtime.interpreter.basename) + ctx.actions.expand_template( + template = ctx.file._template, + output = executable, + substitutions = { + "%target_file%": runfiles_root_path(ctx, runtime.interpreter.short_path), + }, + is_executable = True, + ) + else: + executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) + ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) + + return [ + DefaultInfo( + executable = executable, + runfiles = ctx.runfiles([executable], transitive_files = runtime.files).merge_all([ + ctx.attr._bash_runfiles[DefaultInfo].default_runfiles, + ]), + ), + ] + +interpreter_binary = rule( + implementation = _interpreter_binary_impl, + toolchains = [TARGET_TOOLCHAIN_TYPE], + executable = True, + attrs = { + "binary": attr.label( + mandatory = True, + ), + "_bash_runfiles": attr.label( + default = "@bazel_tools//tools/bash/runfiles", + ), + "_template": attr.label( + default = "//python/private:interpreter_tmpl.sh", + allow_single_file = True, + ), + }, +) diff --git a/python/private/interpreter_tmpl.sh b/python/private/interpreter_tmpl.sh new file mode 100644 index 0000000000..cfe85ec1be --- /dev/null +++ b/python/private/interpreter_tmpl.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +# shellcheck disable=SC1090 +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- + +set +e # allow us to check for errors more easily +readonly TARGET_FILE="%target_file%" +MAIN_BIN=$(rlocation "$TARGET_FILE") + +if [[ -z "$MAIN_BIN" || ! -e "$MAIN_BIN" ]]; then + echo "ERROR: interpreter executable not found: $MAIN_BIN (from $TARGET_FILE)" + exit 1 +fi +exec "${MAIN_BIN}" "$@" diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 2b2bf6636a..a2ccdc65f3 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -48,6 +48,7 @@ load( "filter_to_py_srcs", "get_imports", "is_bool", + "runfiles_root_path", "target_platform_has_any_constraint", "union_attrs", ) @@ -447,7 +448,7 @@ def _create_executable( ) def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): - python_binary = _runfiles_root_path(ctx, venv.interpreter.short_path) + python_binary = runfiles_root_path(ctx, venv.interpreter.short_path) python_binary_actual = venv.interpreter_actual_path # The location of this file doesn't really matter. It's added to @@ -522,7 +523,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): if not venvs_use_declare_symlink_enabled: if runtime.interpreter: - interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) + interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) else: interpreter_actual_path = runtime.interpreter_path @@ -543,11 +544,11 @@ def _create_venv(ctx, output_prefix, imports, runtime_details): # may choose to write what symlink() points to instead. interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename)) - interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path) + interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) rel_path = relative_path( # dirname is necessary because a relative symlink is relative to # the directory the symlink resides within. - from_ = paths.dirname(_runfiles_root_path(ctx, interpreter.short_path)), + from_ = paths.dirname(runfiles_root_path(ctx, interpreter.short_path)), to = interpreter_actual_path, ) @@ -646,23 +647,6 @@ def _create_stage2_bootstrap( ) return output -def _runfiles_root_path(ctx, short_path): - """Compute a runfiles-root relative path from `File.short_path` - - Args: - ctx: current target ctx - short_path: str, a main-repo relative path from `File.short_path` - - Returns: - {type}`str`, a runflies-root relative path - """ - - # The ../ comes from short_path is for files in other repos. - if short_path.startswith("../"): - return short_path[3:] - else: - return "{}/{}".format(ctx.workspace_name, short_path) - def _create_stage1_bootstrap( ctx, *, @@ -676,7 +660,7 @@ def _create_stage1_bootstrap( runtime = runtime_details.effective_runtime if venv: - python_binary_path = _runfiles_root_path(ctx, venv.interpreter.short_path) + python_binary_path = runfiles_root_path(ctx, venv.interpreter.short_path) else: python_binary_path = runtime_details.executable_interpreter_path diff --git a/python/private/site_init_template.py b/python/private/site_init_template.py index dcbd799909..40fb4e4139 100644 --- a/python/private/site_init_template.py +++ b/python/private/site_init_template.py @@ -163,7 +163,9 @@ def _maybe_add_path(path): if cov_tool: _print_verbose_coverage(f"Using toolchain coverage_tool {cov_tool}") elif cov_tool := os.environ.get("PYTHON_COVERAGE"): - _print_verbose_coverage(f"Using env var coverage: PYTHON_COVERAGE={cov_tool}") + _print_verbose_coverage( + f"Using env var coverage: PYTHON_COVERAGE={cov_tool}" + ) if cov_tool: if os.path.isabs(cov_tool): diff --git a/tests/interpreter/BUILD.bazel b/tests/interpreter/BUILD.bazel new file mode 100644 index 0000000000..5d89ede28a --- /dev/null +++ b/tests/interpreter/BUILD.bazel @@ -0,0 +1,52 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":interpreter_tests.bzl", "PYTHON_VERSIONS_TO_TEST", "py_reconfig_interpreter_tests") + +# For this test the interpreter is sourced from the current configuration. That +# means both the interpreter and the test itself are expected to run under the +# same Python version. +py_reconfig_interpreter_tests( + name = "interpreter_version_test", + srcs = ["interpreter_test.py"], + data = [ + "//python/bin:python", + ], + env = { + "PYTHON_BIN": "$(rootpath //python/bin:python)", + }, + main = "interpreter_test.py", + python_versions = PYTHON_VERSIONS_TO_TEST, +) + +# For this test the interpreter is sourced from a binary pinned at a specific +# Python version. That means the interpreter and the test itself can run +# different Python versions. +py_reconfig_interpreter_tests( + name = "python_src_test", + srcs = ["interpreter_test.py"], + data = [ + "//python/bin:python", + ], + env = { + # Since we're grabbing the interpreter from a binary with a fixed + # version, we expect to always see that version. It doesn't matter what + # Python version the test itself is running with. + "EXPECTED_INTERPRETER_VERSION": "3.11", + "PYTHON_BIN": "$(rootpath //python/bin:python)", + }, + main = "interpreter_test.py", + python_src = "//tools/publish:twine", + python_versions = PYTHON_VERSIONS_TO_TEST, +) diff --git a/tests/interpreter/interpreter_test.py b/tests/interpreter/interpreter_test.py new file mode 100644 index 0000000000..0971fa2eba --- /dev/null +++ b/tests/interpreter/interpreter_test.py @@ -0,0 +1,80 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys +import unittest + + +class InterpreterTest(unittest.TestCase): + def setUp(self): + super().setUp() + self.interpreter = os.environ["PYTHON_BIN"] + + v = sys.version_info + self.version = f"{v.major}.{v.minor}" + + def test_self_version(self): + """Performs a sanity check on the Python version used for this test.""" + expected_version = os.environ["EXPECTED_SELF_VERSION"] + self.assertEqual(expected_version, self.version) + + def test_interpreter_version(self): + """Validates that we can successfully execute arbitrary code from the CLI.""" + expected_version = os.environ.get("EXPECTED_INTERPRETER_VERSION", self.version) + + try: + result = subprocess.check_output( + [self.interpreter], + text=True, + stderr=subprocess.STDOUT, + input="\r".join( + [ + "import sys", + "v = sys.version_info", + "print(f'version: {v.major}.{v.minor}')", + ] + ), + ).strip() + except subprocess.CalledProcessError as error: + print("OUTPUT:", error.stdout) + raise + + self.assertEqual(result, f"version: {expected_version}") + + def test_json_tool(self): + """Validates that we can successfully invoke a module from the CLI.""" + # Pass unformatted JSON to the json.tool module. + try: + result = subprocess.check_output( + [ + self.interpreter, + "-m", + "json.tool", + ], + text=True, + stderr=subprocess.STDOUT, + input='{"json":"obj"}', + ).strip() + except subprocess.CalledProcessError as error: + print("OUTPUT:", error.stdout) + raise + + # Validate that we get formatted JSON back. + self.assertEqual(result, '{\n "json": "obj"\n}') + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/interpreter/interpreter_tests.bzl b/tests/interpreter/interpreter_tests.bzl new file mode 100644 index 0000000000..ad94f43423 --- /dev/null +++ b/tests/interpreter/interpreter_tests.bzl @@ -0,0 +1,54 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This file contains helpers for testing the interpreter rule.""" + +load("//tests/support:sh_py_run_test.bzl", "py_reconfig_test") + +# The versions of Python that we want to run the interpreter tests against. +PYTHON_VERSIONS_TO_TEST = ( + "3.10", + "3.11", + "3.12", +) + +def py_reconfig_interpreter_tests(name, python_versions, env = {}, **kwargs): + """Runs the specified test against each of the specified Python versions. + + One test gets generated for each Python version. The following environment + variable gets set for the test: + + EXPECTED_SELF_VERSION: Contains the Python version that the test itself + is running under. + + Args: + name: Name of the test. + python_versions: A list of Python versions to test. + env: The environment to set on the test. + **kwargs: Passed to the underlying py_reconfig_test targets. + """ + for python_version in python_versions: + py_reconfig_test( + name = "{}_{}".format(name, python_version), + env = env | { + "EXPECTED_SELF_VERSION": python_version, + }, + python_version = python_version, + **kwargs + ) + + native.test_suite( + name = name, + tests = [":{}_{}".format(name, python_version) for python_version in python_versions], + ) diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index a1da285864..d116f0403f 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -35,12 +35,15 @@ def _perform_transition_impl(input_settings, attr, base_impl): settings["//python/config_settings:bootstrap_impl"] = attr.bootstrap_impl if attr.extra_toolchains: settings["//command_line_option:extra_toolchains"] = attr.extra_toolchains + if attr.python_src: + settings["//python/bin:python_src"] = attr.python_src if attr.venvs_use_declare_symlink: settings["//python/config_settings:venvs_use_declare_symlink"] = attr.venvs_use_declare_symlink return settings _RECONFIG_INPUTS = [ "//python/config_settings:bootstrap_impl", + "//python/bin:python_src", "//command_line_option:extra_toolchains", "//python/config_settings:venvs_use_declare_symlink", ] @@ -62,6 +65,7 @@ to make the RBE presubmits happy, which disable auto-detection of a CC toolchain. """, ), + "python_src": attr.label(), "venvs_use_declare_symlink": attr.string(), }