fix(pypi): support properly installing sdists via pypi without index

This fixes the subtle bug introduced in #2871, where we were dropping
the URL from the requirement, because we can download the sdist
directly. We cannot add `--no-index` because sdists in general may
require extra build dependencies and we had already issues previously
(see 0.36 release notes).

Fixes #2363

Author: aignas

python/private/pypi/index_sources.bzl

@@ -93,12 +93,12 @@ def index_sources(line):
                 is_known_ext = True
                 break
 
-        if is_known_ext:
+        requirement = requirement_line
+        if filename.endswith(".whl"):
             requirement = maybe_requirement.strip()
-        else:
+        elif not is_known_ext:
             # could not detect filename from the URL
             filename = ""
-            requirement = requirement_line
 
     return struct(
         requirement = requirement,

tests/pypi/extension/extension_tests.bzl

@@ -934,7 +934,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef
             "extra_pip_args": ["--extra-args-for-sdist-building"],
             "filename": "any-name.tar.gz",
             "python_interpreter_target": "unit_test_interpreter_target",
-            "requirement": "direct_sdist_without_sha",
+            "requirement": "direct_sdist_without_sha @ some-archive/any-name.tar.gz",
             "sha256": "",
             "urls": ["some-archive/any-name.tar.gz"],
         },

tests/pypi/index_sources/index_sources_tests.bzl

@@ -73,7 +73,7 @@ def _test_no_simple_api_sources(env):
             filename = "package.whl",
         ),
         "foo[extra] @ https://example.org/foo-1.0.tar.gz --hash=sha256:deadbe0f": struct(
-            requirement = "foo[extra]",
+            requirement = "foo[extra] @ https://example.org/foo-1.0.tar.gz --hash=sha256:deadbe0f",
             requirement_line = "foo[extra] @ https://example.org/foo-1.0.tar.gz --hash=sha256:deadbe0f",
             marker = "",
             url = "https://example.org/foo-1.0.tar.gz",

tests/pypi/parse_requirements/parse_requirements_tests.bzl

@@ -27,6 +27,9 @@ foo==0.0.1 \
 """,
         "requirements_direct": """\
 foo[extra] @ https://some-url/package.whl
+""",
+        "requirements_direct_sdist": """
+foo @ https://github.com/org/foo/downloads/foo-1.1.tar.gz
 """,
         "requirements_extra_args": """\
 --index-url=example.org
@@ -131,22 +134,33 @@ def _test_direct_urls_integration(env):
         ctx = _mock_ctx(),
         requirements_by_platform = {
             "requirements_direct": ["linux_x86_64"],
+            "requirements_direct_sdist": ["osx_x86_64"],
         },
     )
     env.expect.that_collection(got).contains_exactly([
         struct(
             name = "foo",
             is_exposed = True,
-            is_multiple_versions = False,
+            is_multiple_versions = True,
             srcs = [
                 struct(
                     distribution = "foo",
                     extra_pip_args = [],
+                    filename = "foo-1.1.tar.gz",
+                    requirement_line = "foo @ https://github.com/org/foo/downloads/foo-1.1.tar.gz",
+                    sha256 = "",
+                    target_platforms = ["osx_x86_64"],
+                    url = "https://github.com/org/foo/downloads/foo-1.1.tar.gz",
+                    yanked = False,
+                ),
+                struct(
+                    distribution = "foo",
+                    extra_pip_args = [],
+                    filename = "package.whl",
                     requirement_line = "foo[extra]",
+                    sha256 = "",
                     target_platforms = ["linux_x86_64"],
                     url = "https://some-url/package.whl",
-                    filename = "package.whl",
-                    sha256 = "",
                     yanked = False,
                 ),
             ],