add an escape hatch and documentation if users encounter errors

Author: aignas

CHANGELOG.md

@@ -84,6 +84,9 @@ Unreleased changes template.
   This means that `rules_python` will attempt to fetch metadata for all
   packages through SimpleAPI unless they are pulled through direct URL
   references. Fixes [#2023](https://github.com/bazel-contrib/rules_python/issues/2023).
+  In case you see issues with `rules_python` being too eager to fetch the SimpleAPI
+  metadata, you can use the newly added {attr}`pip.parse.experimental_skip_sources`
+  to skip metadata fetching for those packages.
 * (uv) A {obj}`lock` rule that is the replacement for the
   {obj}`compile_pip_requirements`. This may still have rough corners
   so please report issues with it in the

python/private/pypi/extension.bzl

@@ -467,13 +467,21 @@ You cannot use both the additive_build_content and additive_build_content_file a
             get_index_urls = None
             if pip_attr.experimental_index_url:
                 is_reproducible = False
+                skip_sources = [
+                    normalize_name(s)
+                    for s in pip_attr.simpleapi_skip
+                ]
                 get_index_urls = lambda ctx, distributions: simpleapi_download(
                     ctx,
                     attr = struct(
                         index_url = pip_attr.experimental_index_url,
                         extra_index_urls = pip_attr.experimental_extra_index_urls or [],
                         index_url_overrides = pip_attr.experimental_index_url_overrides or {},
-                        sources = distributions,
+                        sources = [
+                            d
+                            for d in distributions
+                            if normalize_name(d) not in skip_sources
+                        ],
                         envsubst = pip_attr.envsubst,
                         # Auth related info
                         netrc = pip_attr.netrc,
@@ -690,6 +698,11 @@ This is equivalent to `--index-url` `pip` option.
 If {attr}`download_only` is set, then `sdist` archives will be discarded and `pip.parse` will
 operate in wheel-only mode.
 :::
+
+:::{versionchanged} VERSION_NEXT_FEATURE
+Index metadata will be used to deduct `sha256` values for packages even if the
+`sha256` values are not present in the requirements.txt lock file.
+:::
 """,
         ),
         "experimental_index_url_overrides": attr.string_dict(
@@ -757,6 +770,18 @@ The Python version the dependencies are targetting, in Major.Minor format
 If an interpreter isn't explicitly provided (using `python_interpreter` or
 `python_interpreter_target`), then the version specified here must have
 a corresponding `python.toolchain()` configured.
+""",
+        ),
+        "simpleapi_skip": attr.string_list(
+            doc = """\
+The list of packages to skip fetching metadata for from SimpleAPI index. You should
+normally not need this attribute, but in case you do, please report this as a bug
+to `rules_python` and use this attribute until the bug is fixed.
+
+EXPERIMENTAL: this may be removed without notice.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
 """,
         ),
         "whl_modifications": attr.label_keyed_string_dict(

python/private/pypi/parse_requirements.bzl

@@ -318,7 +318,10 @@ def _add_dists(*, requirement, index_urls, logger = None):
     # First try to find distributions by SHA256 if provided
     shas_to_use = requirement.srcs.shas
     if not shas_to_use:
-        shas_to_use = index_urls.sha256s_by_version.get(requirement.srcs.version, [])
+        version = requirement.srcs.version
+        shas_to_use = index_urls.sha256s_by_version.get(version, [])
+        if logger:
+            logger.warn(lambda: "requirement file has been generated without hashes, will use all hashes for the given version {} that could find on the index:\n    {}".format(version, shas_to_use))
 
     for sha256 in shas_to_use:
         # For now if the artifact is marked as yanked we just ignore it.

python/private/pypi/simpleapi_download.bzl

@@ -127,10 +127,17 @@ def simpleapi_download(
 
     failed_sources = [pkg for pkg in attr.sources if pkg not in found_on_index]
     if failed_sources:
-        _fail("Failed to download metadata for {} for from urls: {}".format(
-            failed_sources,
-            index_urls,
-        ))
+        _fail(
+            "\n".join([
+                "Failed to download metadata for {} for from urls: {}.".format(
+                    failed_sources,
+                    index_urls,
+                ),
+                "If you would like to skip downloading metadata for these packages please add 'simpleapi_skip={}' to your 'pip.parse' call.".format(
+                    render.list(failed_sources),
+                ),
+            ]),
+        )
         return None
 
     if warn_overrides:

tests/pypi/extension/extension_tests.bzl

@@ -100,6 +100,7 @@ def _parse(
         requirements_linux = None,
         requirements_lock = None,
         requirements_windows = None,
+        simpleapi_skip = [],
         timeout = 600,
         whl_modifications = {},
         **kwargs):
@@ -136,6 +137,7 @@ def _parse(
         experimental_extra_index_urls = [],
         parallel_download = False,
         experimental_index_url_overrides = {},
+        simpleapi_skip = simpleapi_skip,
         **kwargs
     )
 

tests/pypi/simpleapi_download/simpleapi_download_tests.bzl

@@ -110,7 +110,10 @@ def _test_fail(env):
     )
 
     env.expect.that_collection(fails).contains_exactly([
-        """Failed to download metadata for ["foo"] for from urls: ["main", "extra"]""",
+        """\
+Failed to download metadata for ["foo"] for from urls: ["main", "extra"].
+If you would like to skip downloading metadata for these packages please add 'simpleapi_skip=["foo"]' to your 'pip.parse' call.\
+""",
     ])
     env.expect.that_collection(calls).contains_exactly([
         "extra/foo/",