revert(pypi): use Python for marker eval and METADATA parsing

Summary:
- Revert to using Python for marker evaluation during parsing of
  requirements (partial revert of #2692).
- Use Python to parse whl METADATA.

Fixes #2830

Author: aignas

python/private/pypi/evaluate_markers.bzl

@@ -18,6 +18,17 @@ load(":pep508_env.bzl", "env")
 load(":pep508_evaluate.bzl", "evaluate")
 load(":pep508_platform.bzl", "platform_from_str")
 load(":pep508_requirement.bzl", "requirement")
+load(":deps.bzl", "record_files")
+load(":pypi_repo_utils.bzl", "pypi_repo_utils")
+
+# Used as a default value in a rule to ensure we fetch the dependencies.
+SRCS = [
+    # When the version, or any of the files in `packaging` package changes,
+    # this file will change as well.
+    record_files["pypi__packaging"],
+    Label("//python/private/pypi/requirements_parser:resolve_target_platforms.py"),
+    Label("//python/private/pypi/whl_installer:platform.py"),
+]
 
 def evaluate_markers(requirements, python_version = None):
     """Return the list of supported platforms per requirements line.
@@ -37,3 +48,54 @@ def evaluate_markers(requirements, python_version = None):
                 ret.setdefault(req_string, []).append(platform)
 
     return ret
+
+def evaluate_markers_py(mrctx, *, requirements, python_interpreter, python_interpreter_target, srcs, logger = None):
+    """Return the list of supported platforms per requirements line.
+
+    Args:
+        mrctx: repository_ctx or module_ctx.
+        requirements: list[str] of the requirement file lines to evaluate.
+        python_interpreter: str, path to the python_interpreter to use to
+            evaluate the env markers in the given requirements files. It will
+            be only called if the requirements files have env markers. This
+            should be something that is in your PATH or an absolute path.
+        python_interpreter_target: Label, same as python_interpreter, but in a
+            label format.
+        srcs: list[Label], the value of SRCS passed from the `rctx` or `mctx` to this function.
+        logger: repo_utils.logger or None, a simple struct to log diagnostic
+            messages. Defaults to None.
+
+    Returns:
+        dict of string lists with target platforms
+    """
+    if not requirements:
+        return {}
+
+    in_file = mrctx.path("requirements_with_markers.in.json")
+    out_file = mrctx.path("requirements_with_markers.out.json")
+    mrctx.file(in_file, json.encode(requirements))
+
+    pypi_repo_utils.execute_checked(
+        mrctx,
+        op = "ResolveRequirementEnvMarkers({})".format(in_file),
+        python = pypi_repo_utils.resolve_python_interpreter(
+            mrctx,
+            python_interpreter = python_interpreter,
+            python_interpreter_target = python_interpreter_target,
+        ),
+        arguments = [
+            "-m",
+            "python.private.pypi.requirements_parser.resolve_target_platforms",
+            in_file,
+            out_file,
+        ],
+        srcs = srcs,
+        environment = {
+            "PYTHONPATH": [
+                Label("@pypi__packaging//:BUILD.bazel"),
+                Label("//:BUILD.bazel"),
+            ],
+        },
+        logger = logger,
+    )
+    return json.decode(mrctx.read(out_file))

python/private/pypi/extension.bzl

@@ -24,7 +24,7 @@ load("//python/private:repo_utils.bzl", "repo_utils")
 load("//python/private:semver.bzl", "semver")
 load("//python/private:version_label.bzl", "version_label")
 load(":attrs.bzl", "use_isolated")
-load(":evaluate_markers.bzl", "evaluate_markers")
+load(":evaluate_markers.bzl", "evaluate_markers_py", EVALUATE_MARKERS_SRCS = "SRCS")
 load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
 load(":parse_requirements.bzl", "parse_requirements")
 load(":parse_whl_name.bzl", "parse_whl_name")
@@ -172,7 +172,28 @@ def _create_whl_repos(
         ),
         extra_pip_args = pip_attr.extra_pip_args,
         get_index_urls = get_index_urls,
-        evaluate_markers = evaluate_markers,
+        # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either
+        # in the PATH or if specified as a label. We will configure the env
+        # markers when evaluating the requirement lines based on the output
+        # from the `requirements_files_by_platform` which should have something
+        # similar to:
+        # {
+        #    "//:requirements.txt": ["cp311_linux_x86_64", ...]
+        # }
+        #
+        # We know the target python versions that we need to evaluate the
+        # markers for and thus we don't need to use multiple python interpreter
+        # instances to perform this manipulation. This function should be executed
+        # only once by the underlying code to minimize the overhead needed to
+        # spin up a Python interpreter.
+        evaluate_markers = lambda module_ctx, requirements: evaluate_markers_py(
+            module_ctx,
+            requirements = requirements,
+            python_interpreter = pip_attr.python_interpreter,
+            python_interpreter_target = python_interpreter_target,
+            srcs = pip_attr._evaluate_markers_srcs,
+            logger = logger,
+        ),
         logger = logger,
     )
 
@@ -193,6 +214,7 @@ def _create_whl_repos(
             enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,
             environment = pip_attr.environment,
             envsubst = pip_attr.envsubst,
+            experimental_target_platforms = pip_attr.experimental_target_platforms,
             group_deps = group_deps,
             group_name = group_name,
             pip_data_exclude = pip_attr.pip_data_exclude,
@@ -281,6 +303,7 @@ def _whl_repos(*, requirement, whl_library_args, download_only, netrc, auth_patt
         args["urls"] = [distribution.url]
         args["sha256"] = distribution.sha256
         args["filename"] = distribution.filename
+        args["experimental_target_platforms"] = requirement.target_platforms
 
         # Pure python wheels or sdists may need to have a platform here
         target_platforms = None
@@ -775,6 +798,13 @@ EXPERIMENTAL: this may be removed without notice.
             doc = """\
 A dict of labels to wheel names that is typically generated by the whl_modifications.
 The labels are JSON config files describing the modifications.
+""",
+        ),
+        "_evaluate_markers_srcs": attr.label_list(
+            default = EVALUATE_MARKERS_SRCS,
+            doc = """\
+The list of labels to use as SRCS for the marker evaluation code. This ensures that the
+code will be re-evaluated when any of files in the default changes.
 """,
         ),
     }, **ATTRS)

python/private/pypi/generate_whl_library_build_bazel.bzl

@@ -21,6 +21,8 @@ _RENDER = {
     "copy_files": render.dict,
     "data": render.list,
     "data_exclude": render.list,
+    "dependencies": render.list,
+    "dependencies_by_platform": lambda x: render.dict(x, value_repr = render.list),
     "entry_points": render.dict,
     "extras": render.list,
     "group_deps": render.list,
@@ -37,7 +39,7 @@ _TEMPLATE = """\
 
 package(default_visibility = ["//visibility:public"])
 
-whl_library_targets_from_requires(
+whl_library_targets(
 {kwargs}
 )
 """
@@ -60,16 +62,8 @@ def generate_whl_library_build_bazel(
     """
 
     loads = [
-        """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires")""",
+        """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets")""",
     ]
-    if not kwargs.setdefault("target_platforms", None):
-        dep_template = kwargs["dep_template"]
-        loads.append(
-            "load(\"{}\", \"{}\")".format(
-                dep_template.format(name = "", target = "config.bzl"),
-                "target_platforms",
-            ),
-        )
 
     additional_content = []
     if annotation:

python/private/pypi/parse_requirements.bzl

@@ -80,7 +80,7 @@ def parse_requirements(
 
         The second element is extra_pip_args should be passed to `whl_library`.
     """
-    evaluate_markers = evaluate_markers or (lambda _: {})
+    evaluate_markers = evaluate_markers or (lambda _ctx, _requirements: {})
     options = {}
     requirements = {}
     for file, plats in requirements_by_platform.items():
@@ -156,7 +156,7 @@ def parse_requirements(
     # to do, we could use Python to parse the requirement lines and infer the
     # URL of the files to download things from. This should be important for
     # VCS package references.
-    env_marker_target_platforms = evaluate_markers(reqs_with_env_markers)
+    env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers)
     if logger:
         logger.debug(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format(
             reqs_with_env_markers,

python/private/pypi/pip_repository.bzl

@@ -18,7 +18,7 @@ load("@bazel_skylib//lib:sets.bzl", "sets")
 load("//python/private:normalize_name.bzl", "normalize_name")
 load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
 load("//python/private:text_util.bzl", "render")
-load(":evaluate_markers.bzl", "evaluate_markers")
+load(":evaluate_markers.bzl", "evaluate_markers_py")
 load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
 load(":pip_repository_attrs.bzl", "ATTRS")
 load(":pypi_repo_utils.bzl", "pypi_repo_utils")
@@ -71,25 +71,6 @@ package(default_visibility = ["//visibility:public"])
 exports_files(["requirements.bzl"])
 """
 
-def _evaluate_markers(rctx, requirements, logger = None):
-    python_interpreter = _get_python_interpreter_attr(rctx)
-    stdout = pypi_repo_utils.execute_checked_stdout(
-        rctx,
-        op = "GetPythonVersionForMarkerEval",
-        python = python_interpreter,
-        arguments = [
-            # Run the interpreter in isolated mode, this options implies -E, -P and -s.
-            # Ensures environment variables are ignored that are set in userspace, such as PYTHONPATH,
-            # which may interfere with this invocation.
-            "-I",
-            "-c",
-            "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}.{sys.version_info[2]}', end='')",
-        ],
-        srcs = [],
-        logger = logger,
-    )
-    return evaluate_markers(requirements, python_version = stdout)
-
 def _pip_repository_impl(rctx):
     logger = repo_utils.logger(rctx)
     requirements_by_platform = parse_requirements(
@@ -103,7 +84,13 @@ def _pip_repository_impl(rctx):
             extra_pip_args = rctx.attr.extra_pip_args,
         ),
         extra_pip_args = rctx.attr.extra_pip_args,
-        evaluate_markers = lambda requirements: _evaluate_markers(rctx, requirements, logger),
+        evaluate_markers = lambda rctx, requirements: evaluate_markers_py(
+            rctx,
+            requirements = requirements,
+            python_interpreter = rctx.attr.python_interpreter,
+            python_interpreter_target = rctx.attr.python_interpreter_target,
+            srcs = rctx.attr._evaluate_markers_srcs,
+        ),
     )
     selected_requirements = {}
     options = None

python/private/pypi/whl_library.bzl

@@ -15,18 +15,16 @@
 ""
 
 load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth")
-load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
 load("//python/private:envsubst.bzl", "envsubst")
 load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter")
 load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
 load(":attrs.bzl", "ATTRS", "use_isolated")
 load(":deps.bzl", "all_repo_names", "record_files")
 load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
-load(":parse_requirements.bzl", "host_platform")
+load(":parse_whl_name.bzl", "parse_whl_name")
 load(":patch_whl.bzl", "patch_whl")
-load(":pep508_requirement.bzl", "requirement")
 load(":pypi_repo_utils.bzl", "pypi_repo_utils")
-load(":whl_metadata.bzl", "whl_metadata")
+load(":whl_target_platforms.bzl", "whl_target_platforms")
 
 _CPPFLAGS = "CPPFLAGS"
 _COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools"
@@ -342,14 +340,29 @@ def _whl_library_impl(rctx):
                 timeout = rctx.attr.timeout,
             )
 
+    target_platforms = rctx.attr.experimental_target_platforms or []
+    if target_platforms:
+        parsed_whl = parse_whl_name(whl_path.basename)
+
+        # NOTE @aignas 2023-12-04: if the wheel is a platform specific wheel, we
+        # only include deps for that target platform
+        if parsed_whl.platform_tag != "any":
+            target_platforms = [
+                p.target_platform
+                for p in whl_target_platforms(
+                    platform_tag = parsed_whl.platform_tag,
+                    abi_tag = parsed_whl.abi_tag.strip("tm"),
+                )
+            ]
+
     pypi_repo_utils.execute_checked(
         rctx,
         op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path),
         python = python_interpreter,
         arguments = args + [
             "--whl-file",
             whl_path,
-        ],
+        ] + ["--platform={}".format(p) for p in target_platforms],
         srcs = rctx.attr._python_srcs,
         environment = environment,
         quiet = rctx.attr.quiet,
@@ -384,45 +397,21 @@ def _whl_library_impl(rctx):
         )
         entry_points[entry_point_without_py] = entry_point_script_name
 
-    if BZLMOD_ENABLED:
-        # The following attributes are unset on bzlmod and we pass data through
-        # the hub via load statements.
-        default_python_version = None
-        target_platforms = []
-    else:
-        # NOTE @aignas 2025-04-16: if BZLMOD_ENABLED, we should use
-        # DEFAULT_PYTHON_VERSION since platforms always come with the actual
-        # python version otherwise we should use the version of the interpreter
-        # here. In WORKSPACE `multi_pip_parse` is using an interpreter for each
-        # `pip_parse` invocation, so we will have the host target platform
-        # only. Even if somebody would change the code to support
-        # `experimental_target_platforms`, they would be for a single python
-        # version. Hence, using the `default_python_version` that we get from the
-        # interpreter is correct. Hence, we unset the argument if we are on bzlmod.
-        default_python_version = metadata["python_version"]
-        target_platforms = rctx.attr.experimental_target_platforms or [host_platform(rctx)]
-
-    metadata = whl_metadata(
-        install_dir = rctx.path("site-packages"),
-        read_fn = rctx.read,
-        logger = logger,
-    )
-
     build_file_contents = generate_whl_library_build_bazel(
         name = whl_path.basename,
-        metadata_name = metadata.name,
-        metadata_version = metadata.version,
-        requires_dist = metadata.requires_dist,
         dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
         entry_points = entry_points,
-        target_platforms = target_platforms,
-        default_python_version = default_python_version,
         # TODO @aignas 2025-04-14: load through the hub:
+        dependencies = metadata["deps"],
+        dependencies_by_platform = metadata["deps_by_platform"],
         annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
         data_exclude = rctx.attr.pip_data_exclude,
-        extras = requirement(rctx.attr.requirement).extras,
         group_deps = rctx.attr.group_deps,
         group_name = rctx.attr.group_name,
+        tags = [
+            "pypi_name={}".format(metadata["name"]),
+            "pypi_version={}".format(metadata["version"]),
+        ],
     )
     rctx.file("BUILD.bazel", build_file_contents)