feat(pypi/parse_requirements): get dists by version when no hash provided

Modify _add_dists to fetch files by version if no sha256 hashes are available,
and add corresponding unit tests.

- Updated _add_dists to check for the presence of hashes before fetching by hash.
- Added an `else` condition that iterates through index URLs to find wheels and source distributions matching the requirement's version when no hashes are provided.
- Added unit tests in //tests/pypi/parse_requirements to verify version-based fetching.

Author: Yanpei-Wang

CHANGELOG.md

@@ -61,7 +61,7 @@ Unreleased changes template.
 
 {#v0-0-0-added}
 ### Added
-* Nothing added.
+* (pypi) Added version-based fetching in `_add_dists` when SHA256 hashes are unavailable, with unit tests in `//tests/pypi/parse_requirements`[#2023](https://github.com/bazel-contrib/rules_python/issues/2023).
 
 {#v0-0-0-removed}
 ### Removed

python/private/pypi/parse_requirements.bzl

@@ -315,26 +315,46 @@ def _add_dists(*, requirement, index_urls, logger = None):
     whls = []
     sdist = None
 
-    # TODO @aignas 2024-05-22: it is in theory possible to add all
-    # requirements by version instead of by sha256. This may be useful
-    # for some projects.
-    for sha256 in requirement.srcs.shas:
-        # For now if the artifact is marked as yanked we just ignore it.
-        #
-        # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
-
-        maybe_whl = index_urls.whls.get(sha256)
-        if maybe_whl and not maybe_whl.yanked:
-            whls.append(maybe_whl)
-            continue
-
-        maybe_sdist = index_urls.sdists.get(sha256)
-        if maybe_sdist and not maybe_sdist.yanked:
-            sdist = maybe_sdist
-            continue
-
-        if logger:
-            logger.warn(lambda: "Could not find a whl or an sdist with sha256={}".format(sha256))
+    # First try to find distributions by SHA256 if provided
+    if requirement.srcs.shas:
+        for sha256 in requirement.srcs.shas:
+            # For now if the artifact is marked as yanked we just ignore it.
+            #
+            # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
+
+            maybe_whl = index_urls.whls.get(sha256)
+            if maybe_whl and not maybe_whl.yanked:
+                whls.append(maybe_whl)
+                continue
+
+            maybe_sdist = index_urls.sdists.get(sha256)
+            if maybe_sdist and not maybe_sdist.yanked:
+                sdist = maybe_sdist
+                continue
+
+            if logger:
+                logger.warn(lambda: "Could not find a whl or an sdist with sha256={}".format(sha256))
+    else:
+        # If no SHA256s provided, try to find distributions by version
+        version = requirement.srcs.version
+        if version:
+            # Look for wheels matching the version
+            for whl in index_urls.whls.values():
+                # Extract package name from wheel filename (format: package_name-version-python_tag-abi_tag-platform_tag.whl)
+                whl_name = whl.filename.split("-")[0]
+                if whl_name == requirement.distribution and whl.version == version and not whl.yanked:
+                    whls.append(whl)
+
+            # Look for source distributions matching the version
+            for sdist_dist in index_urls.sdists.values():
+                # Extract package name from sdist filename (format: package_name-version.tar.gz or package_name-version.zip)
+                sdist_name = sdist_dist.filename.split("-")[0]
+                if sdist_name == requirement.distribution and sdist_dist.version == version and not sdist_dist.yanked:
+                    sdist = sdist_dist
+                    break
+
+            if not whls and not sdist and logger:
+                logger.warn(lambda: "Could not find any distributions for version={}".format(version))
 
     yanked = {}
     for dist in whls + [sdist]:

tests/pypi/parse_requirements/parse_requirements_tests.bzl

@@ -61,6 +61,10 @@ foo[extra]==0.0.1 --hash=sha256:deadbeef
         "requirements_marker": """\
 foo[extra]==0.0.1 ;marker --hash=sha256:deadbeef
 bar==0.0.1 --hash=sha256:deadbeef
+""",
+        "requirements_optional_hash": """
+foo==0.0.4 @ https://example.org/foo-0.0.4.whl
+foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef
 """,
         "requirements_osx": """\
 foo==0.0.3 --hash=sha256:deadbaaf
@@ -563,6 +567,62 @@ def _test_different_package_version(env):
 
 _tests.append(_test_different_package_version)
 
+def _test_optional_hash(env):
+    got = parse_requirements(
+        ctx = _mock_ctx(),
+        requirements_by_platform = {
+            "requirements_optional_hash": ["linux_x86_64"],
+        },
+    )
+    env.expect.that_dict(got).contains_exactly({
+        "foo": [
+            struct(
+                distribution = "foo",
+                extra_pip_args = [],
+                sdist = None,
+                is_exposed = True,
+                srcs = struct(
+                    marker = "",
+                    requirement = "foo==0.0.4 @ https://example.org/foo-0.0.4.whl",
+                    requirement_line = "foo==0.0.4 @ https://example.org/foo-0.0.4.whl",
+                    shas = [],
+                    version = "0.0.4",
+                    url = "https://example.org/foo-0.0.4.whl",
+                ),
+                target_platforms = ["linux_x86_64"],
+                whls = [struct(
+                    url = "https://example.org/foo-0.0.4.whl",
+                    filename = "foo-0.0.4.whl",
+                    sha256 = "",
+                    yanked = False,
+                )],
+            ),
+            struct(
+                distribution = "foo",
+                extra_pip_args = [],
+                sdist = None,
+                is_exposed = True,
+                srcs = struct(
+                    marker = "",
+                    requirement = "foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef",
+                    requirement_line = "foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef",
+                    shas = ["deadbeef"],
+                    version = "0.0.5",
+                    url = "https://example.org/foo-0.0.5.whl",
+                ),
+                target_platforms = ["linux_x86_64"],
+                whls = [struct(
+                    url = "https://example.org/foo-0.0.5.whl",
+                    filename = "foo-0.0.5.whl",
+                    sha256 = "deadbeef",
+                    yanked = False,
+                )],
+            ),
+        ],
+    })
+
+_tests.append(_test_optional_hash)
+
 def parse_requirements_test_suite(name):
     """Create the test suite.