From 58ec109c07a38d8c362a72de4ec4010bb8e3daef Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 24 Apr 2025 22:08:53 +0900 Subject: [PATCH 1/3] fix(pypi): call python --version before marker eval `bzlmod` has the full python version information statically and we don't need to call Python to get its version, but for `WORKSPACE` that is not the case and we have to call it before evaluating the markers in universal requirements files. Fixes #2818 --- .../requirements/requirements.in | 2 +- .../requirements/requirements_lock_3_10.txt | 4 +-- .../requirements/requirements_lock_3_11.txt | 4 +-- .../requirements/requirements_lock_3_9.txt | 2 +- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/evaluate_markers.bzl | 7 +++--- python/private/pypi/pip_repository.bzl | 25 +++++++++++++++++-- 7 files changed, 34 insertions(+), 11 deletions(-) diff --git a/examples/multi_python_versions/requirements/requirements.in b/examples/multi_python_versions/requirements/requirements.in index 14774b465e..4d1474b9a2 100644 --- a/examples/multi_python_versions/requirements/requirements.in +++ b/examples/multi_python_versions/requirements/requirements.in @@ -1 +1 @@ -websockets +websockets ; python_full_version > "3.9.1" diff --git a/examples/multi_python_versions/requirements/requirements_lock_3_10.txt b/examples/multi_python_versions/requirements/requirements_lock_3_10.txt index 4910d13844..795b530f99 100644 --- a/examples/multi_python_versions/requirements/requirements_lock_3_10.txt +++ b/examples/multi_python_versions/requirements/requirements_lock_3_10.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # bazel run //requirements:requirements_3_10.update # -websockets==11.0.3 \ +websockets==11.0.3 ; python_full_version > "3.9.1" \ --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ diff --git a/examples/multi_python_versions/requirements/requirements_lock_3_11.txt b/examples/multi_python_versions/requirements/requirements_lock_3_11.txt index 35666b54b1..6f598e1155 100644 --- a/examples/multi_python_versions/requirements/requirements_lock_3_11.txt +++ b/examples/multi_python_versions/requirements/requirements_lock_3_11.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # bazel run //requirements:requirements_3_11.update # -websockets==11.0.3 \ +websockets==11.0.3 ; python_full_version > "3.9.1" \ --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ diff --git a/examples/multi_python_versions/requirements/requirements_lock_3_9.txt b/examples/multi_python_versions/requirements/requirements_lock_3_9.txt index 0001f88d48..3c696a865e 100644 --- a/examples/multi_python_versions/requirements/requirements_lock_3_9.txt +++ b/examples/multi_python_versions/requirements/requirements_lock_3_9.txt @@ -4,7 +4,7 @@ # # bazel run //requirements:requirements_3_9.update # -websockets==11.0.3 \ +websockets==11.0.3 ; python_full_version > "3.9.1" \ --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index a758b3f153..8e97283469 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -280,6 +280,7 @@ bzl_library( ":evaluate_markers_bzl", ":parse_requirements_bzl", ":pip_repository_attrs_bzl", + ":pypi_repo_utils_bzl", ":render_pkg_aliases_bzl", ":whl_config_setting_bzl", "//python/private:normalize_name_bzl", diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index a0223abdc8..f966aa32be 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -19,11 +19,12 @@ load(":pep508_evaluate.bzl", "evaluate") load(":pep508_platform.bzl", "platform_from_str") load(":pep508_requirement.bzl", "requirement") -def evaluate_markers(requirements): +def evaluate_markers(requirements, python_version = None): """Return the list of supported platforms per requirements line. Args: - requirements: dict[str, list[str]] of the requirement file lines to evaluate. + requirements: {type}`dict[str, list[str]]` of the requirement file lines to evaluate. + python_version: {type}`str | None` the version that can be used when evaluating the markers. Returns: dict of string lists with target platforms @@ -32,7 +33,7 @@ def evaluate_markers(requirements): for req_string, platforms in requirements.items(): req = requirement(req_string) for platform in platforms: - if evaluate(req.marker, env = env(platform_from_str(platform, None))): + if evaluate(req.marker, env = env(platform_from_str(platform, python_version))): ret.setdefault(req_string, []).append(platform) return ret diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 01a541cf2f..b7ed1659d1 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -16,11 +16,12 @@ load("@bazel_skylib//lib:sets.bzl", "sets") load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load("//python/private:text_util.bzl", "render") load(":evaluate_markers.bzl", "evaluate_markers") load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load(":pip_repository_attrs.bzl", "ATTRS") +load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":render_pkg_aliases.bzl", "render_pkg_aliases") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") @@ -70,7 +71,27 @@ package(default_visibility = ["//visibility:public"]) exports_files(["requirements.bzl"]) """ +def _evaluate_markers(rctx, requirements, logger = None): + python_interpreter = _get_python_interpreter_attr(rctx) + stdout = pypi_repo_utils.execute_checked_stdout( + rctx, + op = "GetPythonVersionForMarkerEval", + python = python_interpreter, + arguments = [ + # Run the interpreter in isolated mode, this options implies -E, -P and -s. + # Ensures environment variables are ignored that are set in userspace, such as PYTHONPATH, + # which may interfere with this invocation. + "-I", + "-c", + "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}.{sys.version_info[2]}', end='')", + ], + srcs = [], + logger = logger, + ) + return evaluate_markers(requirements, python_version = stdout) + def _pip_repository_impl(rctx): + logger = repo_utils.logger(rctx) requirements_by_platform = parse_requirements( rctx, requirements_by_platform = requirements_files_by_platform( @@ -82,7 +103,7 @@ def _pip_repository_impl(rctx): extra_pip_args = rctx.attr.extra_pip_args, ), extra_pip_args = rctx.attr.extra_pip_args, - evaluate_markers = evaluate_markers, + evaluate_markers = lambda requirements: _evaluate_markers(rctx, requirements, logger), ) selected_requirements = {} options = None From 0968a9e5892057a3ad294b7db62712e7d61d0ff8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 24 Apr 2025 22:19:39 +0900 Subject: [PATCH 2/3] also fix the transition for legacy pip_compile --- .../requirements/requirements_lock_3_10.txt | 2 +- .../requirements/requirements_lock_3_11.txt | 2 +- python/private/pypi/pip_compile.bzl | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/multi_python_versions/requirements/requirements_lock_3_10.txt b/examples/multi_python_versions/requirements/requirements_lock_3_10.txt index 795b530f99..3a8453223f 100644 --- a/examples/multi_python_versions/requirements/requirements_lock_3_10.txt +++ b/examples/multi_python_versions/requirements/requirements_lock_3_10.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # bazel run //requirements:requirements_3_10.update diff --git a/examples/multi_python_versions/requirements/requirements_lock_3_11.txt b/examples/multi_python_versions/requirements/requirements_lock_3_11.txt index 6f598e1155..f1fa8f56f5 100644 --- a/examples/multi_python_versions/requirements/requirements_lock_3_11.txt +++ b/examples/multi_python_versions/requirements/requirements_lock_3_11.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # bazel run //requirements:requirements_3_11.update diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index e5b62c4ab0..9782d3ce21 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -160,6 +160,7 @@ def pip_compile( py_binary( name = name + ".update", env = env, + python_version = kwargs.get("python_version", None), **attrs ) From 340ff1e13a330467e8b89116ed3944f25f260c53 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 24 Apr 2025 22:28:03 +0900 Subject: [PATCH 3/3] doc: changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 984af8bad2..8fc00ca25f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,6 +151,10 @@ END_UNRELEASED_TEMPLATE * (pypi) Correctly handle `METADATA` entries when `python_full_version` is used in the environment marker. Fixes [#2319](https://github.com/bazel-contrib/rules_python/issues/2319). +* (pypi) Correctly handle `python_version` parameter and transition the requirement + locking to the right interpreter version when using + {obj}`compile_pip_requirements` rule. + See [#2819](https://github.com/bazel-contrib/rules_python/pull/2819). {#1-4-0-added} ### Added