Merge branch 'main' into fix/2434-repro

Author: aignas

CHANGELOG.md

@@ -57,7 +57,7 @@ Unreleased changes template.
   `exec_interpreter` now also forwards the `ToolchainInfo` provider. This is
   for increased compatibility with the `RBE` setups where access to the `exec`
   configuration interpreter is needed.
-* (toolchains) Use the latest astrahl-sh toolchain release [20250317] for Python versions:
+* (toolchains) Use the latest astral-sh toolchain release [20250317] for Python versions:
     * 3.9.21
     * 3.10.16
     * 3.11.11
@@ -84,6 +84,14 @@ Unreleased changes template.
 
 {#v0-0-0-added}
 ### Added
+* (pypi) From now on `sha256` values in the `requirements.txt` is no longer
+  mandatory when enabling {attr}`pip.parse.experimental_index_url` feature.
+  This means that `rules_python` will attempt to fetch metadata for all
+  packages through SimpleAPI unless they are pulled through direct URL
+  references. Fixes [#2023](https://github.com/bazel-contrib/rules_python/issues/2023).
+  In case you see issues with `rules_python` being too eager to fetch the SimpleAPI
+  metadata, you can use the newly added {attr}`pip.parse.experimental_skip_sources`
+  to skip metadata fetching for those packages.
 * (uv) A {obj}`lock` rule that is the replacement for the
   {obj}`compile_pip_requirements`. This may still have rough corners
   so please report issues with it in the

docs/pypi-dependencies.md

@@ -386,11 +386,13 @@ This does not mean that `rules_python` is fetching the wheels eagerly, but it
 rather means that it is calling the PyPI server to get the Simple API response
 to get the list of all available source and wheel distributions. Once it has
 got all of the available distributions, it will select the right ones depending
-on the `sha256` values in your `requirements_lock.txt` file. The compatible
-distribution URLs will be then written to the `MODULE.bazel.lock` file. Currently
-users wishing to use the lock file with `rules_python` with this feature have
-to set an environment variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will
-become default in the next release.
+on the `sha256` values in your `requirements_lock.txt` file. If `sha256` hashes
+are not present in the requirements file, we will fallback to matching by version
+specified in the lock file. The compatible distribution URLs will be then
+written to the `MODULE.bazel.lock` file. Currently users wishing to use the
+lock file with `rules_python` with this feature have to set an environment
+variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will become default in the
+next release.
 
 Fetching the distribution information from the PyPI allows `rules_python` to
 know which `whl` should be used on which target platform and it will determine

docs/requirements.txt

@@ -18,9 +18,9 @@ babel==2.17.0 \
     --hash=sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d \
     --hash=sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2
     # via sphinx
-certifi==2024.8.30 \
-    --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \
-    --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9
+certifi==2025.1.31 \
+    --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \
+    --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe
     # via requests
 charset-normalizer==3.4.0 \
     --hash=sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621 \
@@ -236,15 +236,15 @@ myst-parser==4.0.0 \
     --hash=sha256:851c9dfb44e36e56d15d05e72f02b80da21a9e0d07cba96baf5e2d476bb91531 \
     --hash=sha256:b9317997552424448c6096c2558872fdb6f81d3ecb3a40ce84a7518798f3f28d
     # via rules-python-docs (docs/pyproject.toml)
-packaging==24.1 \
-    --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \
-    --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124
+packaging==24.2 \
+    --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \
+    --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f
     # via
     #   readthedocs-sphinx-ext
     #   sphinx
-pygments==2.18.0 \
-    --hash=sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199 \
-    --hash=sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a
+pygments==2.19.1 \
+    --hash=sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f \
+    --hash=sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c
     # via sphinx
 pyyaml==6.0.2 \
     --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \
@@ -328,9 +328,9 @@ sphinx-autodoc2==0.5.0 \
     --hash=sha256:7d76044aa81d6af74447080182b6868c7eb066874edc835e8ddf810735b6565a \
     --hash=sha256:e867013b1512f9d6d7e6f6799f8b537d6884462acd118ef361f3f619a60b5c9e
     # via rules-python-docs (docs/pyproject.toml)
-sphinx-reredirects==0.1.5 \
-    --hash=sha256:444ae1438fba4418242ca76d6a6de3eaee82aaf0d8f2b0cac71a15d32ce6eba2 \
-    --hash=sha256:cfa753b441020a22708ce8eb17d4fd553a28fc87a609330092917ada2a6da0d8
+sphinx-reredirects==0.1.6 \
+    --hash=sha256:c491cba545f67be9697508727818d8626626366245ae64456fe29f37e9bbea64 \
+    --hash=sha256:efd50c766fbc5bf40cd5148e10c00f2c00d143027de5c5e48beece93cc40eeea
     # via rules-python-docs (docs/pyproject.toml)
 sphinx-rtd-theme==3.0.1 \
     --hash=sha256:921c0ece75e90633ee876bd7b148cfaad136b481907ad154ac3669b6fc957916 \

python/private/BUILD.bazel

@@ -72,7 +72,6 @@ bzl_library(
         ":py_internal_bzl",
         ":reexports_bzl",
         ":rules_cc_srcs_bzl",
-        ":semantics_bzl",
         "@bazel_skylib//rules:common_settings",
     ],
 )
@@ -131,7 +130,6 @@ bzl_library(
         ":py_internal_bzl",
         ":reexports_bzl",
         ":rules_cc_srcs_bzl",
-        ":semantics_bzl",
         "@bazel_skylib//lib:paths",
     ],
 )
@@ -302,7 +300,6 @@ bzl_library(
         ":attributes_bzl",
         ":py_executable_bzl",
         ":rule_builders_bzl",
-        ":semantics_bzl",
         "@bazel_skylib//lib:dicts",
     ],
 )
@@ -537,7 +534,6 @@ bzl_library(
         ":common_bzl",
         ":py_executable_bzl",
         ":rule_builders_bzl",
-        ":semantics_bzl",
         "@bazel_skylib//lib:dicts",
     ],
 )
@@ -677,11 +673,6 @@ bzl_library(
     ],
 )
 
-bzl_library(
-    name = "semantics_bzl",
-    srcs = ["semantics.bzl"],
-)
-
 # Needed to define bzl_library targets for docgen. (We don't define the
 # bzl_library target here because it'd give our users a transitive dependency
 # on Skylib.)

python/private/attributes.bzl

@@ -23,11 +23,6 @@ load(":py_info.bzl", "PyInfo")
 load(":py_internal.bzl", "py_internal")
 load(":reexports.bzl", "BuiltinPyInfo")
 load(":rule_builders.bzl", "ruleb")
-load(
-    ":semantics.bzl",
-    "DEPS_ATTR_ALLOW_RULES",
-    "SRCS_ATTR_ALLOW_FILES",
-)
 
 _PackageSpecificationInfo = getattr(py_internal, "PackageSpecificationInfo", None)
 
@@ -250,9 +245,6 @@ PY_SRCS_ATTRS = dicts.add(
                 [PyInfo],
                 [CcInfo],
             ] + _MaybeBuiltinPyInfo,
-            # TODO(b/228692666): Google-specific; remove these allowances once
-            # the depot is cleaned up.
-            allow_rules = DEPS_ATTR_ALLOW_RULES,
             doc = """
 List of additional libraries to be linked in to the target.
 See comments about
@@ -359,8 +351,7 @@ as part of a runnable program (packaging rules may include them, however).
             allow_files = True,
         ),
         "srcs": lambda: attrb.LabelList(
-            # Google builds change the set of allowed files.
-            allow_files = SRCS_ATTR_ALLOW_FILES,
+            allow_files = [".py", ".py3"],
             # Necessary for --compile_one_dependency to work.
             flags = ["DIRECT_COMPILE_TIME_INPUT"],
             doc = """

python/private/py_executable.bzl

@@ -59,13 +59,6 @@ load(":py_internal.bzl", "py_internal")
 load(":py_runtime_info.bzl", "DEFAULT_STUB_SHEBANG", "PyRuntimeInfo")
 load(":reexports.bzl", "BuiltinPyInfo", "BuiltinPyRuntimeInfo")
 load(":rule_builders.bzl", "ruleb")
-load(
-    ":semantics.bzl",
-    "ALLOWED_MAIN_EXTENSIONS",
-    "BUILD_DATA_SYMLINK_PATH",
-    "IS_BAZEL",
-    "PY_RUNTIME_ATTR_NAME",
-)
 load(
     ":toolchain_types.bzl",
     "EXEC_TOOLS_TOOLCHAIN_TYPE",
@@ -1116,19 +1109,12 @@ def _get_runtime_details(ctx, semantics):
     #
     # TOOD(bazelbuild/bazel#7901): Remove this once --python_path flag is removed.
 
-    if IS_BAZEL:
-        flag_interpreter_path = ctx.fragments.bazel_py.python_path
-        toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx)
-        if not effective_runtime:
-            # Clear these just in case
-            toolchain_runtime = None
-            effective_runtime = None
-
-    else:  # Google code path
-        flag_interpreter_path = None
-        toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx)
-        if not effective_runtime:
-            fail("Unable to find Python runtime")
+    flag_interpreter_path = ctx.fragments.bazel_py.python_path
+    toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx)
+    if not effective_runtime:
+        # Clear these just in case
+        toolchain_runtime = None
+        effective_runtime = None
 
     if effective_runtime:
         direct = []  # List of files
@@ -1207,7 +1193,7 @@ def _maybe_get_runtime_from_ctx(ctx):
         effective_runtime = toolchain_runtime
     else:
         toolchain_runtime = None
-        attr_target = getattr(ctx.attr, PY_RUNTIME_ATTR_NAME)
+        attr_target = ctx.attr._py_interpreter
 
         # In Bazel, --python_top is null by default.
         if attr_target and PyRuntimeInfo in attr_target:
@@ -1335,9 +1321,9 @@ def _create_runfiles_with_build_data(
         central_uncachable_version_file,
         extra_write_build_data_env,
     )
-    build_data_runfiles = ctx.runfiles(symlinks = {
-        BUILD_DATA_SYMLINK_PATH: build_data_file,
-    })
+    build_data_runfiles = ctx.runfiles(files = [
+        build_data_file,
+    ])
     return build_data_file, build_data_runfiles
 
 def _write_build_data(ctx, central_uncachable_version_file, extra_write_build_data_env):
@@ -1552,7 +1538,7 @@ def determine_main(ctx):
     """
     if ctx.attr.main:
         proposed_main = ctx.attr.main.label.name
-        if not proposed_main.endswith(tuple(ALLOWED_MAIN_EXTENSIONS)):
+        if not proposed_main.endswith(".py"):
             fail("main must end in '.py'")
     else:
         if ctx.label.name.endswith(".py"):

python/private/pypi/BUILD.bazel

@@ -75,7 +75,9 @@ bzl_library(
     name = "evaluate_markers_bzl",
     srcs = ["evaluate_markers.bzl"],
     deps = [
-        ":pypi_repo_utils_bzl",
+        ":pep508_env_bzl",
+        ":pep508_evaluate_bzl",
+        ":pep508_req_bzl",
     ],
 )
 
@@ -209,6 +211,37 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "pep508_bzl",
+    srcs = ["pep508.bzl"],
+    deps = [
+        ":pep508_env_bzl",
+        ":pep508_evaluate_bzl",
+    ],
+)
+
+bzl_library(
+    name = "pep508_env_bzl",
+    srcs = ["pep508_env.bzl"],
+)
+
+bzl_library(
+    name = "pep508_evaluate_bzl",
+    srcs = ["pep508_evaluate.bzl"],
+    deps = [
+        "//python/private:enum_bzl",
+        "//python/private:semver_bzl",
+    ],
+)
+
+bzl_library(
+    name = "pep508_req_bzl",
+    srcs = ["pep508_req.bzl"],
+    deps = [
+        "//python/private:normalize_name_bzl",
+    ],
+)
+
 bzl_library(
     name = "pip_bzl",
     srcs = ["pip.bzl"],

python/private/pypi/evaluate_markers.bzl

@@ -14,65 +14,24 @@
 
 """A simple function that evaluates markers using a python interpreter."""
 
-load(":deps.bzl", "record_files")
-load(":pypi_repo_utils.bzl", "pypi_repo_utils")
+load(":pep508_env.bzl", "env", _platform_from_str = "platform_from_str")
+load(":pep508_evaluate.bzl", "evaluate")
+load(":pep508_req.bzl", _req = "requirement")
 
-# Used as a default value in a rule to ensure we fetch the dependencies.
-SRCS = [
-    # When the version, or any of the files in `packaging` package changes,
-    # this file will change as well.
-    record_files["pypi__packaging"],
-    Label("//python/private/pypi/requirements_parser:resolve_target_platforms.py"),
-    Label("//python/private/pypi/whl_installer:platform.py"),
-]
-
-def evaluate_markers(mrctx, *, requirements, python_interpreter, python_interpreter_target, srcs, logger = None):
+def evaluate_markers(requirements):
     """Return the list of supported platforms per requirements line.
 
     Args:
-        mrctx: repository_ctx or module_ctx.
-        requirements: list[str] of the requirement file lines to evaluate.
-        python_interpreter: str, path to the python_interpreter to use to
-            evaluate the env markers in the given requirements files. It will
-            be only called if the requirements files have env markers. This
-            should be something that is in your PATH or an absolute path.
-        python_interpreter_target: Label, same as python_interpreter, but in a
-            label format.
-        srcs: list[Label], the value of SRCS passed from the `rctx` or `mctx` to this function.
-        logger: repo_utils.logger or None, a simple struct to log diagnostic
-            messages. Defaults to None.
+        requirements: dict[str, list[str]] of the requirement file lines to evaluate.
 
     Returns:
         dict of string lists with target platforms
     """
-    if not requirements:
-        return {}
-
-    in_file = mrctx.path("requirements_with_markers.in.json")
-    out_file = mrctx.path("requirements_with_markers.out.json")
-    mrctx.file(in_file, json.encode(requirements))
-
-    pypi_repo_utils.execute_checked(
-        mrctx,
-        op = "ResolveRequirementEnvMarkers({})".format(in_file),
-        python = pypi_repo_utils.resolve_python_interpreter(
-            mrctx,
-            python_interpreter = python_interpreter,
-            python_interpreter_target = python_interpreter_target,
-        ),
-        arguments = [
-            "-m",
-            "python.private.pypi.requirements_parser.resolve_target_platforms",
-            in_file,
-            out_file,
-        ],
-        srcs = srcs,
-        environment = {
-            "PYTHONPATH": [
-                Label("@pypi__packaging//:BUILD.bazel"),
-                Label("//:BUILD.bazel"),
-            ],
-        },
-        logger = logger,
-    )
-    return json.decode(mrctx.read(out_file))
+    ret = {}
+    for req_string, platforms in requirements.items():
+        req = _req(req_string)
+        for platform in platforms:
+            if evaluate(req.marker, env = env(_platform_from_str(platform, None))):
+                ret.setdefault(req_string, []).append(platform)
+
+    return ret