fix(pypi): expose pypi packages only common to all python versions (#3107)

Closes #2921

For a single pypi repo with multiple python versions, `all_requirements`
fail when a pypi package supports Python version A but not version B. In
this case, the pypi package would be included only in requirements lock
file for version A, not in one for version B. However, the failure
occurs since the package is included in `all_requirements` even for
Python version B.
(Minimal reproduction:
https://github.com/dotoleeoak/rules-python-2921-repro)

This happens since `packages` parameter for `hub_repository` targets are
including all packages across all requirement lock files. Instead of
union of packages, intersection of packages for requirement files should
be passed to `packages` and exposed to `all_requirements` macro, so that
those packages are compatible with all Python versions.

---------

Co-authored-by: Ignas Anikevicius <[email protected]>

Author: dotoleeoak

CHANGELOG.md

@@ -91,6 +91,8 @@ END_UNRELEASED_TEMPLATE
 * Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle.
 * (toolchains) `local_runtime_repo` works with multiarch Debian with Python 3.8
   ([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)).
+* (pypi) Expose pypi packages only common to all Python versions in `all_requirements`
+  ([#2921](https://github.com/bazel-contrib/rules_python/issues/2921)).
 
 {#v0-0-0-added}
 ### Added

python/private/pypi/extension.bzl

@@ -601,7 +601,15 @@ You cannot use both the additive_build_content and additive_build_content_file a
             extra_aliases.setdefault(hub_name, {})
             for whl_name, aliases in out.extra_aliases.items():
                 extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases)
-            exposed_packages.setdefault(hub_name, {}).update(out.exposed_packages)
+            if hub_name not in exposed_packages:
+                exposed_packages[hub_name] = out.exposed_packages
+            else:
+                intersection = {}
+                for pkg in out.exposed_packages:
+                    if pkg not in exposed_packages[hub_name]:
+                        continue
+                    intersection[pkg] = None
+                exposed_packages[hub_name] = intersection
             whl_libraries.update(out.whl_libraries)
 
             # TODO @aignas 2024-04-05: how do we support different requirement

tests/pypi/extension/extension_tests.bzl

@@ -285,6 +285,104 @@ def _test_simple_multiple_requirements(env):
 
 _tests.append(_test_simple_multiple_requirements)
 
+def _test_simple_multiple_python_versions(env):
+    pypi = _parse_modules(
+        env,
+        module_ctx = _mock_mctx(
+            _mod(
+                name = "rules_python",
+                parse = [
+                    _parse(
+                        hub_name = "pypi",
+                        python_version = "3.15",
+                        requirements_lock = "requirements_3_15.txt",
+                    ),
+                    _parse(
+                        hub_name = "pypi",
+                        python_version = "3.16",
+                        requirements_lock = "requirements_3_16.txt",
+                    ),
+                ],
+            ),
+            read = lambda x: {
+                "requirements_3_15.txt": """
+simple==0.0.1 --hash=sha256:deadbeef
+old-package==0.0.1 --hash=sha256:deadbaaf
+""",
+                "requirements_3_16.txt": """
+simple==0.0.2 --hash=sha256:deadb00f
+new-package==0.0.1 --hash=sha256:deadb00f2
+""",
+            }[x],
+        ),
+        available_interpreters = {
+            "python_3_15_host": "unit_test_interpreter_target",
+            "python_3_16_host": "unit_test_interpreter_target",
+        },
+        minor_mapping = {
+            "3.15": "3.15.19",
+            "3.16": "3.16.9",
+        },
+    )
+
+    pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
+    pypi.hub_group_map().contains_exactly({"pypi": {}})
+    pypi.hub_whl_map().contains_exactly({
+        "pypi": {
+            "new_package": {
+                "pypi_316_new_package": [
+                    whl_config_setting(
+                        version = "3.16",
+                    ),
+                ],
+            },
+            "old_package": {
+                "pypi_315_old_package": [
+                    whl_config_setting(
+                        version = "3.15",
+                    ),
+                ],
+            },
+            "simple": {
+                "pypi_315_simple": [
+                    whl_config_setting(
+                        version = "3.15",
+                    ),
+                ],
+                "pypi_316_simple": [
+                    whl_config_setting(
+                        version = "3.16",
+                    ),
+                ],
+            },
+        },
+    })
+    pypi.whl_libraries().contains_exactly({
+        "pypi_315_old_package": {
+            "dep_template": "@pypi//{name}:{target}",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "requirement": "old-package==0.0.1 --hash=sha256:deadbaaf",
+        },
+        "pypi_315_simple": {
+            "dep_template": "@pypi//{name}:{target}",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "requirement": "simple==0.0.1 --hash=sha256:deadbeef",
+        },
+        "pypi_316_new_package": {
+            "dep_template": "@pypi//{name}:{target}",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "requirement": "new-package==0.0.1 --hash=sha256:deadb00f2",
+        },
+        "pypi_316_simple": {
+            "dep_template": "@pypi//{name}:{target}",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "requirement": "simple==0.0.2 --hash=sha256:deadb00f",
+        },
+    })
+    pypi.whl_mods().contains_exactly({})
+
+_tests.append(_test_simple_multiple_python_versions)
+
 def _test_simple_with_markers(env):
     pypi = _parse_modules(
         env,