Reapply "use starlark to eval markers"

This reverts commit 301f99f.

Author: aignas

python/private/pypi/evaluate_markers.bzl

@@ -15,7 +15,9 @@
 """A simple function that evaluates markers using a python interpreter."""
 
 load(":deps.bzl", "record_files")
-load(":pypi_repo_utils.bzl", "pypi_repo_utils")
+load(":pep508_env.bzl", "env", _platform_from_str = "platform_from_str")
+load(":pep508_evaluate.bzl", "evaluate")
+load(":pep508_req.bzl", _req = "requirement")
 
 # Used as a default value in a rule to ensure we fetch the dependencies.
 SRCS = [
@@ -26,53 +28,20 @@ SRCS = [
     Label("//python/private/pypi/whl_installer:platform.py"),
 ]
 
-def evaluate_markers(mrctx, *, requirements, python_interpreter, python_interpreter_target, srcs, logger = None):
+def evaluate_markers(requirements):
     """Return the list of supported platforms per requirements line.
 
     Args:
-        mrctx: repository_ctx or module_ctx.
-        requirements: list[str] of the requirement file lines to evaluate.
-        python_interpreter: str, path to the python_interpreter to use to
-            evaluate the env markers in the given requirements files. It will
-            be only called if the requirements files have env markers. This
-            should be something that is in your PATH or an absolute path.
-        python_interpreter_target: Label, same as python_interpreter, but in a
-            label format.
-        srcs: list[Label], the value of SRCS passed from the `rctx` or `mctx` to this function.
-        logger: repo_utils.logger or None, a simple struct to log diagnostic
-            messages. Defaults to None.
+        requirements: dict[str, list[str]] of the requirement file lines to evaluate.
 
     Returns:
         dict of string lists with target platforms
     """
-    if not requirements:
-        return {}
-
-    in_file = mrctx.path("requirements_with_markers.in.json")
-    out_file = mrctx.path("requirements_with_markers.out.json")
-    mrctx.file(in_file, json.encode(requirements))
-
-    pypi_repo_utils.execute_checked(
-        mrctx,
-        op = "ResolveRequirementEnvMarkers({})".format(in_file),
-        arguments = [
-            pypi_repo_utils.resolve_python_interpreter(
-                mrctx,
-                python_interpreter = python_interpreter,
-                python_interpreter_target = python_interpreter_target,
-            ),
-            "-m",
-            "python.private.pypi.requirements_parser.resolve_target_platforms",
-            in_file,
-            out_file,
-        ],
-        srcs = srcs,
-        environment = {
-            "PYTHONPATH": [
-                Label("@pypi__packaging//:BUILD.bazel"),
-                Label("//:BUILD.bazel"),
-            ],
-        },
-        logger = logger,
-    )
-    return json.decode(mrctx.read(out_file))
+    ret = {}
+    for req_string, platforms in requirements.items():
+        req = _req(req_string)
+        for platform in platforms:
+            if evaluate(req.marker, env = env(_platform_from_str(platform, None))):
+                ret.setdefault(req_string, []).append(platform)
+
+    return ret

python/private/pypi/extension.bzl

@@ -22,7 +22,7 @@ load("//python/private:repo_utils.bzl", "repo_utils")
 load("//python/private:semver.bzl", "semver")
 load("//python/private:version_label.bzl", "version_label")
 load(":attrs.bzl", "use_isolated")
-load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS")
+load(":evaluate_markers.bzl", "evaluate_markers")
 load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
 load(":parse_requirements.bzl", "parse_requirements")
 load(":parse_whl_name.bzl", "parse_whl_name")
@@ -180,14 +180,7 @@ def _create_whl_repos(
         # instances to perform this manipulation. This function should be executed
         # only once by the underlying code to minimize the overhead needed to
         # spin up a Python interpreter.
-        evaluate_markers = lambda module_ctx, requirements: evaluate_markers(
-            module_ctx,
-            requirements = requirements,
-            python_interpreter = pip_attr.python_interpreter,
-            python_interpreter_target = python_interpreter_target,
-            srcs = pip_attr._evaluate_markers_srcs,
-            logger = logger,
-        ),
+        evaluate_markers = lambda r: evaluate_markers(r),
         logger = logger,
     )
 
@@ -759,13 +752,6 @@ a corresponding `python.toolchain()` configured.
             doc = """\
 A dict of labels to wheel names that is typically generated by the whl_modifications.
 The labels are JSON config files describing the modifications.
-""",
-        ),
-        "_evaluate_markers_srcs": attr.label_list(
-            default = EVALUATE_MARKERS_SRCS,
-            doc = """\
-The list of labels to use as SRCS for the marker evaluation code. This ensures that the
-code will be re-evaluated when any of files in the default changes.
 """,
         ),
     }, **ATTRS)

python/private/pypi/parse_requirements.bzl

@@ -67,10 +67,10 @@ def parse_requirements(
             of the distribution URLs from a PyPI index. Accepts ctx and
             distribution names to query.
         evaluate_markers: A function to use to evaluate the requirements.
-            Accepts the ctx and a dict where keys are requirement lines to
-            evaluate against the platforms stored as values in the input dict.
-            Returns the same dict, but with values being platforms that are
-            compatible with the requirements line.
+            Accepts a dict where keys are requirement lines to evaluate against
+            the platforms stored as values in the input dict. Returns the same
+            dict, but with values being platforms that are compatible with the
+            requirements line.
         logger: repo_utils.logger or None, a simple struct to log diagnostic messages.
 
     Returns:
@@ -93,7 +93,7 @@ def parse_requirements(
 
         The second element is extra_pip_args should be passed to `whl_library`.
     """
-    evaluate_markers = evaluate_markers or (lambda *_: {})
+    evaluate_markers = evaluate_markers or (lambda _: {})
     options = {}
     requirements = {}
     for file, plats in requirements_by_platform.items():
@@ -168,7 +168,7 @@ def parse_requirements(
     # to do, we could use Python to parse the requirement lines and infer the
     # URL of the files to download things from. This should be important for
     # VCS package references.
-    env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers)
+    env_marker_target_platforms = evaluate_markers(reqs_with_env_markers)
     if logger:
         logger.debug(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format(
             reqs_with_env_markers,

python/private/pypi/pep508_env.bzl

@@ -114,10 +114,10 @@ def deps(name, *, requires_dist, platforms = [], extras = [], host_python_versio
     reqs = [r for r in reqs if r.name != name]
 
     platforms = [
-        _platform_from_str(p, python_version = host_python_version)
+        platform_from_str(p, python_version = host_python_version)
         for p in platforms
     ] or [
-        _platform_from_str("", python_version = host_python_version),
+        platform_from_str("", python_version = host_python_version),
     ]
 
     abis = sorted({p.abi: True for p in platforms if p.abi})
@@ -157,7 +157,16 @@ def _platform(*, abi = None, os = None, arch = None):
         arch = arch,
     )
 
-def _platform_from_str(p, python_version):
+def platform_from_str(p, python_version):
+    """Return a platform from a string.
+
+    Args:
+        p: {type}`str` the actual string.
+        python_version: {type}`str` the python version to add to platform if needed.
+
+    Returns:
+        A struct that is returned by the `_platform` function.
+    """
     if p.startswith("cp"):
         abi, _, p = p.partition("_")
     elif python_version:

python/private/pypi/pep508_evaluate.bzl

@@ -114,7 +114,7 @@ def tokenize(marker):
             elif char in _WSP:
                 state = _STATE.NONE
             else:
-                fail("BUG: Cannot parse '{}' in {}".format(char, state))
+                fail("BUG: Cannot parse '{}' in {} ({})".format(char, state, marker))
         else:
             token += char
 

python/private/pypi/pep508_req.bzl

@@ -26,11 +26,13 @@ def requirement(spec):
     Returns:
         A struct with the information.
     """
-    requires, _, marker = spec.partition(";")
+    requires, _, maybe_hashes = spec.partition(";")
+    marker, _, _ = maybe_hashes.partition("--hash")
     requires, _, extras_unparsed = requires.partition("[")
     requires, _, _ = requires.partition("(")
     requires, _, _ = requires.partition(" ")
     extras = extras_unparsed.strip("]").split(",")
+
     return struct(
         name = normalize_name(requires.strip(" ")),
         marker = marker.strip(" "),

python/private/pypi/pip_repository.bzl

@@ -18,7 +18,7 @@ load("@bazel_skylib//lib:sets.bzl", "sets")
 load("//python/private:normalize_name.bzl", "normalize_name")
 load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR")
 load("//python/private:text_util.bzl", "render")
-load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS")
+load(":evaluate_markers.bzl", "evaluate_markers")
 load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
 load(":pip_repository_attrs.bzl", "ATTRS")
 load(":render_pkg_aliases.bzl", "render_pkg_aliases")
@@ -82,12 +82,8 @@ def _pip_repository_impl(rctx):
             extra_pip_args = rctx.attr.extra_pip_args,
         ),
         extra_pip_args = rctx.attr.extra_pip_args,
-        evaluate_markers = lambda rctx, requirements: evaluate_markers(
-            rctx,
+        evaluate_markers = lambda requirements: evaluate_markers(
             requirements = requirements,
-            python_interpreter = rctx.attr.python_interpreter,
-            python_interpreter_target = rctx.attr.python_interpreter_target,
-            srcs = rctx.attr._evaluate_markers_srcs,
         ),
     )
     selected_requirements = {}
@@ -234,13 +230,6 @@ file](https://github.com/bazelbuild/rules_python/blob/main/examples/pip_reposito
         _template = attr.label(
             default = ":requirements.bzl.tmpl.workspace",
         ),
-        _evaluate_markers_srcs = attr.label_list(
-            default = EVALUATE_MARKERS_SRCS,
-            doc = """\
-The list of labels to use as SRCS for the marker evaluation code. This ensures that the
-code will be re-evaluated when any of files in the default changes.
-""",
-        ),
         **ATTRS
     ),
     doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within.