diff --git a/CHANGELOG.md b/CHANGELOG.md
index cc742e6160..f34b4f3140 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -81,6 +81,14 @@ Unreleased changes template.
{#v0-0-0-added}
### Added
+* (pypi) From now on `sha256` values in the `requirements.txt` is no longer
+ mandatory when enabling {attr}`pip.parse.experimental_index_url` feature.
+ This means that `rules_python` will attempt to fetch metadata for all
+ packages through SimpleAPI unless they are pulled through direct URL
+ references. Fixes [#2023](https://github.com/bazel-contrib/rules_python/issues/2023).
+ In case you see issues with `rules_python` being too eager to fetch the SimpleAPI
+ metadata, you can use the newly added {attr}`pip.parse.experimental_skip_sources`
+ to skip metadata fetching for those packages.
* (uv) A {obj}`lock` rule that is the replacement for the
{obj}`compile_pip_requirements`. This may still have rough corners
so please report issues with it in the
diff --git a/docs/pypi-dependencies.md b/docs/pypi-dependencies.md
index 039200dfd4..6cc0da6cb4 100644
--- a/docs/pypi-dependencies.md
+++ b/docs/pypi-dependencies.md
@@ -386,11 +386,13 @@ This does not mean that `rules_python` is fetching the wheels eagerly, but it
rather means that it is calling the PyPI server to get the Simple API response
to get the list of all available source and wheel distributions. Once it has
got all of the available distributions, it will select the right ones depending
-on the `sha256` values in your `requirements_lock.txt` file. The compatible
-distribution URLs will be then written to the `MODULE.bazel.lock` file. Currently
-users wishing to use the lock file with `rules_python` with this feature have
-to set an environment variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will
-become default in the next release.
+on the `sha256` values in your `requirements_lock.txt` file. If `sha256` hashes
+are not present in the requirements file, we will fallback to matching by version
+specified in the lock file. The compatible distribution URLs will be then
+written to the `MODULE.bazel.lock` file. Currently users wishing to use the
+lock file with `rules_python` with this feature have to set an environment
+variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will become default in the
+next release.
Fetching the distribution information from the PyPI allows `rules_python` to
know which `whl` should be used on which target platform and it will determine
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
index be3067d04a..3b13667d3b 100644
--- a/python/private/pypi/extension.bzl
+++ b/python/private/pypi/extension.bzl
@@ -477,13 +477,21 @@ You cannot use both the additive_build_content and additive_build_content_file a
get_index_urls = None
if pip_attr.experimental_index_url:
is_reproducible = False
+ skip_sources = [
+ normalize_name(s)
+ for s in pip_attr.simpleapi_skip
+ ]
get_index_urls = lambda ctx, distributions: simpleapi_download(
ctx,
attr = struct(
index_url = pip_attr.experimental_index_url,
extra_index_urls = pip_attr.experimental_extra_index_urls or [],
index_url_overrides = pip_attr.experimental_index_url_overrides or {},
- sources = distributions,
+ sources = [
+ d
+ for d in distributions
+ if normalize_name(d) not in skip_sources
+ ],
envsubst = pip_attr.envsubst,
# Auth related info
netrc = pip_attr.netrc,
@@ -700,6 +708,11 @@ This is equivalent to `--index-url` `pip` option.
If {attr}`download_only` is set, then `sdist` archives will be discarded and `pip.parse` will
operate in wheel-only mode.
:::
+
+:::{versionchanged} VERSION_NEXT_FEATURE
+Index metadata will be used to deduct `sha256` values for packages even if the
+`sha256` values are not present in the requirements.txt lock file.
+:::
""",
),
"experimental_index_url_overrides": attr.string_dict(
@@ -767,6 +780,18 @@ The Python version the dependencies are targetting, in Major.Minor format
If an interpreter isn't explicitly provided (using `python_interpreter` or
`python_interpreter_target`), then the version specified here must have
a corresponding `python.toolchain()` configured.
+""",
+ ),
+ "simpleapi_skip": attr.string_list(
+ doc = """\
+The list of packages to skip fetching metadata for from SimpleAPI index. You should
+normally not need this attribute, but in case you do, please report this as a bug
+to `rules_python` and use this attribute until the bug is fixed.
+
+EXPERIMENTAL: this may be removed without notice.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
""",
),
"whl_modifications": attr.label_keyed_string_dict(
diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl
index dbff44ecb3..72f38997a1 100644
--- a/python/private/pypi/parse_requirements.bzl
+++ b/python/private/pypi/parse_requirements.bzl
@@ -184,7 +184,7 @@ def parse_requirements(
req.distribution: None
for reqs in requirements_by_platform.values()
for req in reqs.values()
- if req.srcs.shas
+ if not req.srcs.url
}),
)
@@ -315,10 +315,15 @@ def _add_dists(*, requirement, index_urls, logger = None):
whls = []
sdist = None
- # TODO @aignas 2024-05-22: it is in theory possible to add all
- # requirements by version instead of by sha256. This may be useful
- # for some projects.
- for sha256 in requirement.srcs.shas:
+ # First try to find distributions by SHA256 if provided
+ shas_to_use = requirement.srcs.shas
+ if not shas_to_use:
+ version = requirement.srcs.version
+ shas_to_use = index_urls.sha256s_by_version.get(version, [])
+ if logger:
+ logger.warn(lambda: "requirement file has been generated without hashes, will use all hashes for the given version {} that could find on the index:\n {}".format(version, shas_to_use))
+
+ for sha256 in shas_to_use:
# For now if the artifact is marked as yanked we just ignore it.
#
# See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
diff --git a/python/private/pypi/parse_simpleapi_html.bzl b/python/private/pypi/parse_simpleapi_html.bzl
index e549e76181..8c6f739fe3 100644
--- a/python/private/pypi/parse_simpleapi_html.bzl
+++ b/python/private/pypi/parse_simpleapi_html.bzl
@@ -26,6 +26,7 @@ def parse_simpleapi_html(*, url, content):
Returns:
A list of structs with:
* filename: The filename of the artifact.
+ * version: The version of the artifact.
* url: The URL to download the artifact.
* sha256: The sha256 of the artifact.
* metadata_sha256: The whl METADATA sha256 if we can download it. If this is
@@ -51,8 +52,11 @@ def parse_simpleapi_html(*, url, content):
# Each line follows the following pattern
# filename
+ sha256_by_version = {}
for line in lines[1:]:
dist_url, _, tail = line.partition("#sha256=")
+ dist_url = _absolute_url(url, dist_url)
+
sha256, _, tail = tail.partition("\"")
# See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
@@ -60,6 +64,8 @@ def parse_simpleapi_html(*, url, content):
head, _, _ = tail.rpartition("")
maybe_metadata, _, filename = head.rpartition(">")
+ version = _version(filename)
+ sha256_by_version.setdefault(version, []).append(sha256)
metadata_sha256 = ""
metadata_url = ""
@@ -75,7 +81,8 @@ def parse_simpleapi_html(*, url, content):
if filename.endswith(".whl"):
whls[sha256] = struct(
filename = filename,
- url = _absolute_url(url, dist_url),
+ version = version,
+ url = dist_url,
sha256 = sha256,
metadata_sha256 = metadata_sha256,
metadata_url = _absolute_url(url, metadata_url) if metadata_url else "",
@@ -84,7 +91,8 @@ def parse_simpleapi_html(*, url, content):
else:
sdists[sha256] = struct(
filename = filename,
- url = _absolute_url(url, dist_url),
+ version = version,
+ url = dist_url,
sha256 = sha256,
metadata_sha256 = "",
metadata_url = "",
@@ -94,8 +102,31 @@ def parse_simpleapi_html(*, url, content):
return struct(
sdists = sdists,
whls = whls,
+ sha256_by_version = sha256_by_version,
)
+_SDIST_EXTS = [
+ ".tar", # handles any compression
+ ".zip",
+]
+
+def _version(filename):
+ # See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format
+
+ _, _, tail = filename.partition("-")
+ version, _, _ = tail.partition("-")
+ if version != tail:
+ # The format is {name}-{version}-{whl_specifiers}.whl
+ return version
+
+ # NOTE @aignas 2025-03-29: most of the files are wheels, so this is not the common path
+
+ # {name}-{version}.{ext}
+ for ext in _SDIST_EXTS:
+ version, _, _ = version.partition(ext) # build or name
+
+ return version
+
def _get_root_directory(url):
scheme_end = url.find("://")
if scheme_end == -1:
diff --git a/python/private/pypi/simpleapi_download.bzl b/python/private/pypi/simpleapi_download.bzl
index ef39fb8723..e8d7d0941a 100644
--- a/python/private/pypi/simpleapi_download.bzl
+++ b/python/private/pypi/simpleapi_download.bzl
@@ -127,10 +127,17 @@ def simpleapi_download(
failed_sources = [pkg for pkg in attr.sources if pkg not in found_on_index]
if failed_sources:
- _fail("Failed to download metadata for {} for from urls: {}".format(
- failed_sources,
- index_urls,
- ))
+ _fail(
+ "\n".join([
+ "Failed to download metadata for {} for from urls: {}.".format(
+ failed_sources,
+ index_urls,
+ ),
+ "If you would like to skip downloading metadata for these packages please add 'simpleapi_skip={}' to your 'pip.parse' call.".format(
+ render.list(failed_sources),
+ ),
+ ]),
+ )
return None
if warn_overrides:
diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl
index 38ac9dcd92..2904f85f1b 100644
--- a/python/private/pypi/whl_library.bzl
+++ b/python/private/pypi/whl_library.bzl
@@ -270,6 +270,12 @@ def _whl_library_impl(rctx):
sha256 = rctx.attr.sha256,
auth = get_auth(rctx, urls),
)
+ if not rctx.attr.sha256:
+ # this is only seen when there is a direct URL reference without sha256
+ logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format(
+ rctx.attr.requirement,
+ result.sha256,
+ ))
if not result.success:
fail("could not download the '{}' from {}:\n{}".format(filename, urls, result))
diff --git a/python/private/pypi/whl_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl
index 48bbd1a9b2..02a7c8142c 100644
--- a/python/private/pypi/whl_repo_name.bzl
+++ b/python/private/pypi/whl_repo_name.bzl
@@ -32,11 +32,19 @@ def whl_repo_name(filename, sha256):
if not filename.endswith(".whl"):
# Then the filename is basically foo-3.2.1.
- parts.append(normalize_name(filename.rpartition("-")[0]))
- parts.append("sdist")
+ name, _, tail = filename.rpartition("-")
+ parts.append(normalize_name(name))
+ if sha256:
+ parts.append("sdist")
+ version = ""
+ else:
+ for ext in [".tar", ".zip"]:
+ tail, _, _ = tail.partition(ext)
+ version = tail.replace(".", "_").replace("!", "_")
else:
parsed = parse_whl_name(filename)
name = normalize_name(parsed.distribution)
+ version = parsed.version.replace(".", "_").replace("!", "_")
python_tag, _, _ = parsed.python_tag.partition(".")
abi_tag, _, _ = parsed.abi_tag.partition(".")
platform_tag, _, _ = parsed.platform_tag.partition(".")
@@ -46,7 +54,10 @@ def whl_repo_name(filename, sha256):
parts.append(abi_tag)
parts.append(platform_tag)
- parts.append(sha256[:8])
+ if sha256:
+ parts.append(sha256[:8])
+ elif version:
+ parts.insert(1, version)
return "_".join(parts)
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl
index 1b18d2a339..c603a29219 100644
--- a/tests/pypi/extension/extension_tests.bzl
+++ b/tests/pypi/extension/extension_tests.bzl
@@ -101,6 +101,7 @@ def _parse(
requirements_linux = None,
requirements_lock = None,
requirements_windows = None,
+ simpleapi_skip = [],
timeout = 600,
whl_modifications = {},
**kwargs):
@@ -137,6 +138,7 @@ def _parse(
experimental_extra_index_urls = [],
parallel_download = False,
experimental_index_url_overrides = {},
+ simpleapi_skip = simpleapi_skip,
**kwargs
)
@@ -635,6 +637,21 @@ def _test_simple_get_index(env):
),
},
),
+ "some_other_pkg": struct(
+ whls = {
+ "deadb33f": struct(
+ yanked = False,
+ filename = "some-other-pkg-0.0.1-py3-none-any.whl",
+ sha256 = "deadb33f",
+ url = "example2.org/index/some_other_pkg/",
+ ),
+ },
+ sdists = {},
+ sha256s_by_version = {
+ "0.0.1": ["deadb33f"],
+ "0.0.3": ["deadbeef"],
+ },
+ ),
}
pypi = _parse_modules(
@@ -659,7 +676,11 @@ def _test_simple_get_index(env):
simple==0.0.1 \
--hash=sha256:deadbeef \
--hash=sha256:deadb00f
-some_pkg==0.0.1
+some_pkg==0.0.1 @ example-direct.org/some_pkg-0.0.1-py3-none-any.whl \
+ --hash=sha256:deadbaaf
+direct_without_sha==0.0.1 @ example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl
+some_other_pkg==0.0.1
+pip_fallback==0.0.1
""",
}[x],
),
@@ -670,42 +691,91 @@ some_pkg==0.0.1
)
pypi.is_reproducible().equals(False)
- pypi.exposed_packages().contains_exactly({"pypi": ["simple", "some_pkg"]})
+ pypi.exposed_packages().contains_exactly({"pypi": ["direct_without_sha", "pip_fallback", "simple", "some_other_pkg", "some_pkg"]})
pypi.hub_group_map().contains_exactly({"pypi": {}})
pypi.hub_whl_map().contains_exactly({
"pypi": {
+ "direct_without_sha": {
+ "pypi_315_direct_without_sha_0_0_1_py3_none_any": [
+ struct(
+ config_setting = None,
+ filename = "direct_without_sha-0.0.1-py3-none-any.whl",
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
+ },
+ "pip_fallback": {
+ "pypi_315_pip_fallback": [
+ struct(
+ config_setting = None,
+ filename = None,
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
+ },
"simple": {
"pypi_315_simple_py3_none_any_deadb00f": [
- whl_config_setting(
+ struct(
+ config_setting = None,
filename = "simple-0.0.1-py3-none-any.whl",
+ target_platforms = None,
version = "3.15",
),
],
"pypi_315_simple_sdist_deadbeef": [
- whl_config_setting(
+ struct(
+ config_setting = None,
filename = "simple-0.0.1.tar.gz",
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
+ },
+ "some_other_pkg": {
+ "pypi_315_some_py3_none_any_deadb33f": [
+ struct(
+ config_setting = None,
+ filename = "some-other-pkg-0.0.1-py3-none-any.whl",
+ target_platforms = None,
version = "3.15",
),
],
},
"some_pkg": {
- "pypi_315_some_pkg": [whl_config_setting(version = "3.15")],
+ "pypi_315_some_pkg_py3_none_any_deadbaaf": [
+ struct(
+ config_setting = None,
+ filename = "some_pkg-0.0.1-py3-none-any.whl",
+ target_platforms = None,
+ version = "3.15",
+ ),
+ ],
},
},
})
pypi.whl_libraries().contains_exactly({
+ "pypi_315_direct_without_sha_0_0_1_py3_none_any": {
+ "dep_template": "@pypi//{name}:{target}",
+ "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
+ "filename": "direct_without_sha-0.0.1-py3-none-any.whl",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "direct_without_sha==0.0.1 @ example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl",
+ "sha256": "",
+ "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"],
+ },
+ "pypi_315_pip_fallback": {
+ "dep_template": "@pypi//{name}:{target}",
+ "extra_pip_args": ["--extra-args-for-sdist-building"],
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "pip_fallback==0.0.1",
+ },
"pypi_315_simple_py3_none_any_deadb00f": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": [
- "cp315_linux_aarch64",
- "cp315_linux_arm",
- "cp315_linux_ppc",
- "cp315_linux_s390x",
- "cp315_linux_x86_64",
- "cp315_osx_aarch64",
- "cp315_osx_x86_64",
- "cp315_windows_x86_64",
- ],
+ "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
"filename": "simple-0.0.1-py3-none-any.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"repo": "pypi_315",
@@ -715,16 +785,7 @@ some_pkg==0.0.1
},
"pypi_315_simple_sdist_deadbeef": {
"dep_template": "@pypi//{name}:{target}",
- "experimental_target_platforms": [
- "cp315_linux_aarch64",
- "cp315_linux_arm",
- "cp315_linux_ppc",
- "cp315_linux_s390x",
- "cp315_linux_x86_64",
- "cp315_osx_aarch64",
- "cp315_osx_x86_64",
- "cp315_windows_x86_64",
- ],
+ "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
"extra_pip_args": ["--extra-args-for-sdist-building"],
"filename": "simple-0.0.1.tar.gz",
"python_interpreter_target": "unit_test_interpreter_target",
@@ -733,29 +794,43 @@ some_pkg==0.0.1
"sha256": "deadbeef",
"urls": ["example.org"],
},
- # We are falling back to regular `pip`
- "pypi_315_some_pkg": {
+ "pypi_315_some_pkg_py3_none_any_deadbaaf": {
"dep_template": "@pypi//{name}:{target}",
- "extra_pip_args": ["--extra-args-for-sdist-building"],
+ "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
+ "filename": "some_pkg-0.0.1-py3-none-any.whl",
+ "python_interpreter_target": "unit_test_interpreter_target",
+ "repo": "pypi_315",
+ "requirement": "some_pkg==0.0.1 @ example-direct.org/some_pkg-0.0.1-py3-none-any.whl --hash=sha256:deadbaaf",
+ "sha256": "deadbaaf",
+ "urls": ["example-direct.org/some_pkg-0.0.1-py3-none-any.whl"],
+ },
+ "pypi_315_some_py3_none_any_deadb33f": {
+ "dep_template": "@pypi//{name}:{target}",
+ "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"],
+ "filename": "some-other-pkg-0.0.1-py3-none-any.whl",
"python_interpreter_target": "unit_test_interpreter_target",
"repo": "pypi_315",
- "requirement": "some_pkg==0.0.1",
+ "requirement": "some_other_pkg==0.0.1",
+ "sha256": "deadb33f",
+ "urls": ["example2.org/index/some_other_pkg/"],
},
})
pypi.whl_mods().contains_exactly({})
- env.expect.that_dict(got_simpleapi_download_kwargs).contains_exactly({
- "attr": struct(
- auth_patterns = {},
- envsubst = {},
- extra_index_urls = [],
- index_url = "pypi.org",
- index_url_overrides = {},
- netrc = None,
- sources = ["simple"],
- ),
- "cache": {},
- "parallel_download": False,
- })
+ env.expect.that_dict(got_simpleapi_download_kwargs).contains_exactly(
+ {
+ "attr": struct(
+ auth_patterns = {},
+ envsubst = {},
+ extra_index_urls = [],
+ index_url = "pypi.org",
+ index_url_overrides = {},
+ netrc = None,
+ sources = ["simple", "pip_fallback", "some_other_pkg"],
+ ),
+ "cache": {},
+ "parallel_download": False,
+ },
+ )
_tests.append(_test_simple_get_index)
diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
index 8edc2689bf..723bb605ce 100644
--- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl
+++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl
@@ -61,6 +61,10 @@ foo[extra]==0.0.1 --hash=sha256:deadbeef
"requirements_marker": """\
foo[extra]==0.0.1 ;marker --hash=sha256:deadbeef
bar==0.0.1 --hash=sha256:deadbeef
+""",
+ "requirements_optional_hash": """
+foo==0.0.4 @ https://example.org/foo-0.0.4.whl
+foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef
""",
"requirements_osx": """\
foo==0.0.3 --hash=sha256:deadbaaf
@@ -563,6 +567,62 @@ def _test_different_package_version(env):
_tests.append(_test_different_package_version)
+def _test_optional_hash(env):
+ got = parse_requirements(
+ ctx = _mock_ctx(),
+ requirements_by_platform = {
+ "requirements_optional_hash": ["linux_x86_64"],
+ },
+ )
+ env.expect.that_dict(got).contains_exactly({
+ "foo": [
+ struct(
+ distribution = "foo",
+ extra_pip_args = [],
+ sdist = None,
+ is_exposed = True,
+ srcs = struct(
+ marker = "",
+ requirement = "foo==0.0.4 @ https://example.org/foo-0.0.4.whl",
+ requirement_line = "foo==0.0.4 @ https://example.org/foo-0.0.4.whl",
+ shas = [],
+ version = "0.0.4",
+ url = "https://example.org/foo-0.0.4.whl",
+ ),
+ target_platforms = ["linux_x86_64"],
+ whls = [struct(
+ url = "https://example.org/foo-0.0.4.whl",
+ filename = "foo-0.0.4.whl",
+ sha256 = "",
+ yanked = False,
+ )],
+ ),
+ struct(
+ distribution = "foo",
+ extra_pip_args = [],
+ sdist = None,
+ is_exposed = True,
+ srcs = struct(
+ marker = "",
+ requirement = "foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef",
+ requirement_line = "foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef",
+ shas = ["deadbeef"],
+ version = "0.0.5",
+ url = "https://example.org/foo-0.0.5.whl",
+ ),
+ target_platforms = ["linux_x86_64"],
+ whls = [struct(
+ url = "https://example.org/foo-0.0.5.whl",
+ filename = "foo-0.0.5.whl",
+ sha256 = "deadbeef",
+ yanked = False,
+ )],
+ ),
+ ],
+ })
+
+_tests.append(_test_optional_hash)
+
def parse_requirements_test_suite(name):
"""Create the test suite.
diff --git a/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl
index d3c42a8864..abaa7a6a49 100644
--- a/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl
+++ b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl
@@ -52,13 +52,14 @@ def _test_sdist(env):
'data-requires-python=">=3.7"',
],
filename = "foo-0.0.1.tar.gz",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.1.tar.gz",
sha256 = "deadbeefasource",
url = "https://example.org/full-url/foo-0.0.1.tar.gz",
yanked = False,
+ version = "0.0.1",
),
),
(
@@ -68,12 +69,13 @@ def _test_sdist(env):
'data-requires-python=">=3.7"',
],
filename = "foo-0.0.1.tar.gz",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.1.tar.gz",
sha256 = "deadbeefasource",
url = "https://example.org/full-url/foo-0.0.1.tar.gz",
+ version = "0.0.1",
yanked = False,
),
),
@@ -94,12 +96,14 @@ def _test_sdist(env):
sha256 = subjects.str,
url = subjects.str,
yanked = subjects.bool,
+ version = subjects.str,
),
)
actual.filename().equals(want.filename)
actual.sha256().equals(want.sha256)
actual.url().equals(want.url)
actual.yanked().equals(want.yanked)
+ actual.version().equals(want.version)
_tests.append(_test_sdist)
@@ -115,7 +119,7 @@ def _test_whls(env):
'data-core-metadata="sha256=deadb00f"',
],
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
@@ -123,6 +127,7 @@ def _test_whls(env):
metadata_url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata",
sha256 = "deadbeef",
url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ version = "0.0.2",
yanked = False,
),
),
@@ -135,7 +140,7 @@ def _test_whls(env):
'data-core-metadata="sha256=deadb00f"',
],
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
@@ -143,6 +148,7 @@ def _test_whls(env):
metadata_url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata",
sha256 = "deadbeef",
url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ version = "0.0.2",
yanked = False,
),
),
@@ -154,13 +160,14 @@ def _test_whls(env):
'data-core-metadata="sha256=deadb00f"',
],
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
metadata_sha256 = "deadb00f",
metadata_url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata",
sha256 = "deadbeef",
+ version = "0.0.2",
url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
yanked = False,
),
@@ -173,13 +180,14 @@ def _test_whls(env):
'data-dist-info-metadata="sha256=deadb00f"',
],
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
metadata_sha256 = "deadb00f",
metadata_url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata",
sha256 = "deadbeef",
+ version = "0.0.2",
url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
yanked = False,
),
@@ -191,7 +199,7 @@ def _test_whls(env):
'data-requires-python=">=3.7"',
],
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
- url = "ignored",
+ url = "foo",
),
struct(
filename = "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
@@ -199,6 +207,7 @@ def _test_whls(env):
metadata_url = "",
sha256 = "deadbeef",
url = "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+ version = "0.0.2",
yanked = False,
),
),
@@ -217,6 +226,7 @@ def _test_whls(env):
metadata_sha256 = "deadb00f",
metadata_url = "https://example.org/python-wheels/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata",
sha256 = "deadbeef",
+ version = "0.0.2",
url = "https://example.org/python-wheels/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
yanked = False,
),
@@ -235,6 +245,7 @@ def _test_whls(env):
metadata_url = "",
sha256 = "deadbeef",
url = "https://download.pytorch.org/whl/torch-2.0.0-cp38-cp38-manylinux2014_aarch64.whl",
+ version = "2.0.0",
yanked = False,
),
),
@@ -252,6 +263,7 @@ def _test_whls(env):
metadata_url = "",
sha256 = "notdeadbeef",
url = "http://download.pytorch.org/whl/torch-2.0.0-cp38-cp38-manylinux2014_aarch64.whl",
+ version = "2.0.0",
yanked = False,
),
),
@@ -267,6 +279,7 @@ def _test_whls(env):
filename = "mypy_extensions-1.0.0-py3-none-any.whl",
metadata_sha256 = "",
metadata_url = "",
+ version = "1.0.0",
sha256 = "deadbeef",
url = "https://example.org/simple/mypy_extensions/1.0.0/mypy_extensions-1.0.0-py3-none-any.whl",
yanked = False,
@@ -285,6 +298,7 @@ def _test_whls(env):
metadata_sha256 = "",
metadata_url = "",
sha256 = "deadbeef",
+ version = "1.0.0",
url = "https://example.org/simple/mypy_extensions/unknown://example.com/mypy_extensions-1.0.0-py3-none-any.whl",
yanked = False,
),
@@ -308,6 +322,7 @@ def _test_whls(env):
sha256 = subjects.str,
url = subjects.str,
yanked = subjects.bool,
+ version = subjects.str,
),
)
actual.filename().equals(want.filename)
@@ -316,6 +331,7 @@ def _test_whls(env):
actual.sha256().equals(want.sha256)
actual.url().equals(want.url)
actual.yanked().equals(want.yanked)
+ actual.version().equals(want.version)
_tests.append(_test_whls)
diff --git a/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl b/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl
index 964d3e25ea..ce214d6e34 100644
--- a/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl
+++ b/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl
@@ -110,7 +110,10 @@ def _test_fail(env):
)
env.expect.that_collection(fails).contains_exactly([
- """Failed to download metadata for ["foo"] for from urls: ["main", "extra"]""",
+ """\
+Failed to download metadata for ["foo"] for from urls: ["main", "extra"].
+If you would like to skip downloading metadata for these packages please add 'simpleapi_skip=["foo"]' to your 'pip.parse' call.\
+""",
])
env.expect.that_collection(calls).contains_exactly([
"extra/foo/",
diff --git a/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl
index 000941b55b..f0d1d059e1 100644
--- a/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl
+++ b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl
@@ -25,12 +25,24 @@ def _test_simple(env):
_tests.append(_test_simple)
+def _test_simple_no_sha(env):
+ got = whl_repo_name("foo-1.2.3-py3-none-any.whl", "")
+ env.expect.that_str(got).equals("foo_1_2_3_py3_none_any")
+
+_tests.append(_test_simple_no_sha)
+
def _test_sdist(env):
got = whl_repo_name("foo-1.2.3.tar.gz", "deadbeef000deadbeef")
env.expect.that_str(got).equals("foo_sdist_deadbeef")
_tests.append(_test_sdist)
+def _test_sdist_no_sha(env):
+ got = whl_repo_name("foo-1.2.3.tar.gz", "")
+ env.expect.that_str(got).equals("foo_1_2_3")
+
+_tests.append(_test_sdist_no_sha)
+
def _test_platform_whl(env):
got = whl_repo_name(
"foo-1.2.3-cp39.cp310-abi3-manylinux1_x86_64.manylinux_2_17_x86_64.whl",