refactor(pypi): implement PEP508 compliant marker evaluation

python/private/pypi/BUILD.bazel

@@ -75,7 +75,9 @@ bzl_library(
     name = "evaluate_markers_bzl",
     srcs = ["evaluate_markers.bzl"],
     deps = [
-        ":pypi_repo_utils_bzl",
+        ":pep508_env_bzl",
+        ":pep508_evaluate_bzl",
+        ":pep508_req_bzl",
     ],
 )
 
@@ -208,6 +210,37 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "pep508_bzl",
+    srcs = ["pep508.bzl"],
+    deps = [
+        ":pep508_env_bzl",
+        ":pep508_evaluate_bzl",
+    ],
+)
+
+bzl_library(
+    name = "pep508_env_bzl",
+    srcs = ["pep508_env.bzl"],
+)
+
+bzl_library(
+    name = "pep508_evaluate_bzl",
+    srcs = ["pep508_evaluate.bzl"],
+    deps = [
+        "//python/private:enum_bzl",
+        "//python/private:semver_bzl",
+    ],
+)
+
+bzl_library(
+    name = "pep508_req_bzl",
+    srcs = ["pep508_req.bzl"],
+    deps = [
+        "//python/private:normalize_name_bzl",
+    ],
+)
+
 bzl_library(
     name = "pip_bzl",
     srcs = ["pip.bzl"],

python/private/pypi/evaluate_markers.bzl

@@ -14,65 +14,24 @@
 
 """A simple function that evaluates markers using a python interpreter."""
 
-load(":deps.bzl", "record_files")
-load(":pypi_repo_utils.bzl", "pypi_repo_utils")
+load(":pep508_env.bzl", "env", _platform_from_str = "platform_from_str")
+load(":pep508_evaluate.bzl", "evaluate")
+load(":pep508_req.bzl", _req = "requirement")
 
-# Used as a default value in a rule to ensure we fetch the dependencies.
-SRCS = [
-    # When the version, or any of the files in `packaging` package changes,
-    # this file will change as well.
-    record_files["pypi__packaging"],
-    Label("//python/private/pypi/requirements_parser:resolve_target_platforms.py"),
-    Label("//python/private/pypi/whl_installer:platform.py"),
-]
-
-def evaluate_markers(mrctx, *, requirements, python_interpreter, python_interpreter_target, srcs, logger = None):
+def evaluate_markers(requirements):
     """Return the list of supported platforms per requirements line.
 
     Args:
-        mrctx: repository_ctx or module_ctx.
-        requirements: list[str] of the requirement file lines to evaluate.
-        python_interpreter: str, path to the python_interpreter to use to
-            evaluate the env markers in the given requirements files. It will
-            be only called if the requirements files have env markers. This
-            should be something that is in your PATH or an absolute path.
-        python_interpreter_target: Label, same as python_interpreter, but in a
-            label format.
-        srcs: list[Label], the value of SRCS passed from the `rctx` or `mctx` to this function.
-        logger: repo_utils.logger or None, a simple struct to log diagnostic
-            messages. Defaults to None.
+        requirements: dict[str, list[str]] of the requirement file lines to evaluate.
 
     Returns:
         dict of string lists with target platforms
     """
-    if not requirements:
-        return {}
-
-    in_file = mrctx.path("requirements_with_markers.in.json")
-    out_file = mrctx.path("requirements_with_markers.out.json")
-    mrctx.file(in_file, json.encode(requirements))
-
-    pypi_repo_utils.execute_checked(
-        mrctx,
-        op = "ResolveRequirementEnvMarkers({})".format(in_file),
-        python = pypi_repo_utils.resolve_python_interpreter(
-            mrctx,
-            python_interpreter = python_interpreter,
-            python_interpreter_target = python_interpreter_target,
-        ),
-        arguments = [
-            "-m",
-            "python.private.pypi.requirements_parser.resolve_target_platforms",
-            in_file,
-            out_file,
-        ],
-        srcs = srcs,
-        environment = {
-            "PYTHONPATH": [
-                Label("@pypi__packaging//:BUILD.bazel"),
-                Label("//:BUILD.bazel"),
-            ],
-        },
-        logger = logger,
-    )
-    return json.decode(mrctx.read(out_file))
+    ret = {}
+    for req_string, platforms in requirements.items():
+        req = _req(req_string)
+        for platform in platforms:
+            if evaluate(req.marker, env = env(_platform_from_str(platform, None))):
+                ret.setdefault(req_string, []).append(platform)
+
+    return ret

python/private/pypi/extension.bzl

@@ -22,7 +22,7 @@ load("//python/private:repo_utils.bzl", "repo_utils")
 load("//python/private:semver.bzl", "semver")
 load("//python/private:version_label.bzl", "version_label")
 load(":attrs.bzl", "use_isolated")
-load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS")
+load(":evaluate_markers.bzl", "evaluate_markers")
 load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
 load(":parse_requirements.bzl", "parse_requirements")
 load(":parse_whl_name.bzl", "parse_whl_name")
@@ -166,28 +166,10 @@ def _create_whl_repos(
         ),
         extra_pip_args = pip_attr.extra_pip_args,
         get_index_urls = get_index_urls,
-        # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either
-        # in the PATH or if specified as a label. We will configure the env
-        # markers when evaluating the requirement lines based on the output
-        # from the `requirements_files_by_platform` which should have something
-        # similar to:
-        # {
-        #    "//:requirements.txt": ["cp311_linux_x86_64", ...]
-        # }
-        #
-        # We know the target python versions that we need to evaluate the
-        # markers for and thus we don't need to use multiple python interpreter
-        # instances to perform this manipulation. This function should be executed
-        # only once by the underlying code to minimize the overhead needed to
-        # spin up a Python interpreter.
-        evaluate_markers = lambda module_ctx, requirements: evaluate_markers(
-            module_ctx,
-            requirements = requirements,
-            python_interpreter = pip_attr.python_interpreter,
-            python_interpreter_target = python_interpreter_target,
-            srcs = pip_attr._evaluate_markers_srcs,
-            logger = logger,
-        ),
+        # NOTE @aignas 2025-02-24: we will use the "cp3xx_os_arch" platform labels
+        # for converting to the PEP508 environment and will evaluate them in starlark
+        # without involving the interpreter at all.
+        evaluate_markers = evaluate_markers,
         logger = logger,
     )
 
@@ -764,13 +746,6 @@ a corresponding `python.toolchain()` configured.
             doc = """\
 A dict of labels to wheel names that is typically generated by the whl_modifications.
 The labels are JSON config files describing the modifications.
-""",
-        ),
-        "_evaluate_markers_srcs": attr.label_list(
-            default = EVALUATE_MARKERS_SRCS,
-            doc = """\
-The list of labels to use as SRCS for the marker evaluation code. This ensures that the
-code will be re-evaluated when any of files in the default changes.
 """,
         ),
     }, **ATTRS)

python/private/pypi/parse_requirements.bzl

@@ -67,10 +67,10 @@ def parse_requirements(
             of the distribution URLs from a PyPI index. Accepts ctx and
             distribution names to query.
         evaluate_markers: A function to use to evaluate the requirements.
-            Accepts the ctx and a dict where keys are requirement lines to
-            evaluate against the platforms stored as values in the input dict.
-            Returns the same dict, but with values being platforms that are
-            compatible with the requirements line.
+            Accepts a dict where keys are requirement lines to evaluate against
+            the platforms stored as values in the input dict. Returns the same
+            dict, but with values being platforms that are compatible with the
+            requirements line.
         logger: repo_utils.logger or None, a simple struct to log diagnostic messages.
 
     Returns:
@@ -93,7 +93,7 @@ def parse_requirements(
 
         The second element is extra_pip_args should be passed to `whl_library`.
     """
-    evaluate_markers = evaluate_markers or (lambda *_: {})
+    evaluate_markers = evaluate_markers or (lambda _: {})
     options = {}
     requirements = {}
     for file, plats in requirements_by_platform.items():
@@ -168,7 +168,7 @@ def parse_requirements(
     # to do, we could use Python to parse the requirement lines and infer the
     # URL of the files to download things from. This should be important for
     # VCS package references.
-    env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers)
+    env_marker_target_platforms = evaluate_markers(reqs_with_env_markers)
     if logger:
         logger.debug(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format(
             reqs_with_env_markers,

python/private/pypi/pep508.bzl

@@ -0,0 +1,23 @@
+# Copyright 2025 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module is for implementing PEP508 in starlark as FeatureFlagInfo
+"""
+
+load(":pep508_env.bzl", _env = "env")
+load(":pep508_evaluate.bzl", _evaluate = "evaluate", _to_string = "to_string")
+
+to_string = _to_string
+evaluate = _evaluate
+env = _env

python/private/pypi/pep508_env.bzl

@@ -0,0 +1,109 @@
+# Copyright 2025 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This module is for implementing PEP508 environment definition.
+"""
+
+_platform_machine_values = {
+    "aarch64": "arm64",
+    "ppc": "ppc64le",
+    "s390x": "s390x",
+    "x86_32": "i386",
+    "x86_64": "x86_64",
+}
+_platform_system_values = {
+    "linux": "Linux",
+    "osx": "Darwin",
+    "windows": "Windows",
+}
+_sys_platform_values = {
+    "linux": "posix",
+    "osx": "darwin",
+    "windows": "win32",
+}
+_os_name_values = {
+    "linux": "posix",
+    "osx": "posix",
+    "windows": "nt",
+}
+
+def env(target_platform, *, extra = None):
+    """Return an env target platform
+
+    Args:
+        target_platform: {type}`str` the target platform identifier, e.g.
+            `cp33_linux_aarch64`
+        extra: {type}`str` the extra value to be added into the env.
+
+    Returns:
+        A dict that can be used as `env` in the marker evaluation.
+    """
+
+    # TODO @aignas 2025-02-13: consider moving this into config settings.
+
+    env = {"extra": extra} if extra != None else {}
+    env = env | {
+        "implementation_name": "cpython",
+        "platform_python_implementation": "CPython",
+        "platform_release": "",
+        "platform_version": "",
+    }
+    if target_platform.abi:
+        minor_version, _, micro_version = target_platform.abi[3:].partition(".")
+        micro_version = micro_version or "0"
+        env = env | {
+            "implementation_version": "3.{}.{}".format(minor_version, micro_version),
+            "python_full_version": "3.{}.{}".format(minor_version, micro_version),
+            "python_version": "3.{}".format(minor_version),
+        }
+    if target_platform.os and target_platform.arch:
+        os = target_platform.os
+        arch = target_platform.arch
+        env = env | {
+            "os_name": _os_name_values.get(os, ""),
+            "platform_machine": "aarch64" if (os, arch) == ("linux", "aarch64") else _platform_machine_values.get(arch, ""),
+            "platform_system": _platform_system_values.get(os, ""),
+            "sys_platform": _sys_platform_values.get(os, ""),
+        }
+
+    # This is split by topic
+    return env
+
+def _platform(*, abi = None, os = None, arch = None):
+    return struct(
+        abi = abi,
+        os = os,
+        arch = arch,
+    )
+
+def platform_from_str(p, python_version):
+    """Return a platform from a string.
+
+    Args:
+        p: {type}`str` the actual string.
+        python_version: {type}`str` the python version to add to platform if needed.
+
+    Returns:
+        A struct that is returned by the `_platform` function.
+    """
+    if p.startswith("cp"):
+        abi, _, p = p.partition("_")
+    elif python_version:
+        major, _, tail = python_version.partition(".")
+        abi = "cp{}{}".format(major, tail)
+    else:
+        abi = None
+
+    os, _, arch = p.partition("_")
+    return _platform(abi = abi, os = os or None, arch = arch or None)