wip

Author: aignas

python/private/pypi/extension.bzl

@@ -16,7 +16,9 @@
 
 load("@bazel_features//:features.bzl", "bazel_features")
 load("@pythons_hub//:interpreters.bzl", "INTERPRETER_LABELS")
+load("@pythons_hub//:versions.bzl", "MINOR_MAPPING")
 load("//python/private:auth.bzl", "AUTH_ATTRS")
+load("//python/private:full_version.bzl", "full_version")
 load("//python/private:normalize_name.bzl", "normalize_name")
 load("//python/private:repo_utils.bzl", "repo_utils")
 load("//python/private:semver.bzl", "semver")
@@ -173,6 +175,7 @@ def _create_whl_repos(
         logger = logger,
     )
 
+    platforms = {}
     for whl_name, requirements in requirements_by_platform.items():
         group_name = whl_group_mapping.get(whl_name)
         group_deps = requirement_cycles.get(group_name, [])
@@ -233,6 +236,10 @@ def _create_whl_repos(
                     ))
 
                 whl_libraries[repo_name] = args
+
+                # TODO @aignas 2025-03-23: make this more efficient
+                for p in args.pop("experimental_target_platforms", []):
+                    platforms[p] = None
                 whl_map.setdefault(whl_name, {})[config_setting] = repo_name
 
     return struct(
@@ -244,6 +251,7 @@ def _create_whl_repos(
         },
         extra_aliases = extra_aliases,
         whl_libraries = whl_libraries,
+        platforms = platforms,
     )
 
 def _whl_repos(*, requirement, whl_library_args, download_only, netrc, auth_patterns, multiple_requirements_for_whl = False, python_version):
@@ -408,6 +416,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
     exposed_packages = {}
     extra_aliases = {}
     whl_libraries = {}
+    platforms = {}
 
     is_reproducible = True
 
@@ -485,6 +494,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
                 extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases)
             exposed_packages.setdefault(hub_name, {}).update(out.exposed_packages)
             whl_libraries.update(out.whl_libraries)
+            platforms.setdefault(hub_name, {}).update(out.platforms)
 
             # TODO @aignas 2024-04-05: how do we support different requirement
             # cycles for different abis/oses? For now we will need the users to
@@ -521,6 +531,17 @@ You cannot use both the additive_build_content and additive_build_content_file a
             }
             for hub_name, extra_whl_aliases in extra_aliases.items()
         },
+        platforms = {
+            hub_name: sorted(p)
+            for hub_name, p in platforms.items()
+        },
+        python_versions = {
+            hub_name: sorted({
+                full_version(version = v, minor_mapping = MINOR_MAPPING): None
+                for v in m.python_versions
+            })
+            for hub_name, m in pip_hub_map.items()
+        },
         whl_libraries = {
             k: dict(sorted(args.items()))
             for k, args in sorted(whl_libraries.items())
@@ -612,6 +633,8 @@ def _pip_impl(module_ctx):
                 for key, values in whl_map.items()
             },
             packages = mods.exposed_packages.get(hub_name, []),
+            python_versions = mods.python_versions[hub_name],
+            platforms = mods.platforms.get(hub_name, ["host"]),
             groups = mods.hub_group_map.get(hub_name),
         )
 

python/private/pypi/generate_whl_library_build_bazel.bzl

@@ -14,6 +14,7 @@
 
 """Generate the BUILD.bazel contents for a repo defined by a whl_library."""
 
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
 load("//python/private:text_util.bzl", "render")
 
 _RENDER = {
@@ -26,7 +27,8 @@ _RENDER = {
     "entry_points": render.dict,
     "extras": render.list,
     "group_deps": render.list,
-    "platforms": render.list,
+    "host_python_version": str,
+    "platforms": str,
     "requires_dist": render.list,
     "srcs_exclude": render.list,
     "tags": render.list,
@@ -36,7 +38,7 @@ _RENDER = {
 # this repository can be publicly visible without the need for
 # export_files
 _TEMPLATE = """\
-load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets")
+{loads}
 
 package(default_visibility = ["//visibility:public"])
 
@@ -61,6 +63,17 @@ def generate_whl_library_build_bazel(
     """
 
     additional_content = []
+    loads = {
+        "@rules_python//python/private/pypi:whl_library_targets.bzl": ('"whl_library_targets"',),
+    }
+    if BZLMOD_ENABLED:
+        dep_template = kwargs["dep_template"]
+        loads[dep_template.format(
+            name = "",
+            target = "requirements.bzl",
+        )] = ("hub_settings = \"private\"",)
+        kwargs["platforms"] = "hub_settings.platforms"
+        kwargs["host_python_version"] = "hub_settings.python_versions[0]"
     if annotation:
         kwargs["data"] = annotation.data
         kwargs["copy_files"] = annotation.copy_files
@@ -73,6 +86,13 @@ def generate_whl_library_build_bazel(
     contents = "\n".join(
         [
             _TEMPLATE.format(
+                loads = "\n".join([
+                    "load({}, {})".format(
+                        repr(path),
+                        ", ".join([s for s in symbols]),
+                    )
+                    for path, symbols in loads.items()
+                ]),
                 kwargs = render.indent("\n".join([
                     "{} = {},".format(k, _RENDER.get(k, repr)(v))
                     for k, v in sorted(kwargs.items())

python/private/pypi/hub_repository.bzl

@@ -15,6 +15,7 @@
 ""
 
 load("//python/private:text_util.bzl", "render")
+load(":parse_requirements.bzl", "host_platform")
 load(":render_pkg_aliases.bzl", "render_multiplatform_pkg_aliases")
 load(":whl_config_setting.bzl", "whl_config_setting")
 
@@ -44,6 +45,11 @@ def _impl(rctx):
     # `requirement`, et al. macros.
     macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name)
 
+    platforms = [
+        host_platform(rctx) if p == "host" else p
+        for p in rctx.attr.platforms
+    ]
+
     rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
     rctx.template("requirements.bzl", rctx.attr._template, substitutions = {
         "%%ALL_DATA_REQUIREMENTS%%": render.list([
@@ -59,6 +65,8 @@ def _impl(rctx):
             for p in bzl_packages
         }),
         "%%MACRO_TMPL%%": macro_tmpl,
+        "%%PLATFORMS%%": render.indent(render.list(sorted(platforms))).lstrip(),
+        "%%PYTHON_VERSIONS%%": render.indent(render.list(sorted(rctx.attr.python_versions))).lstrip(),
     })
 
 hub_repository = repository_rule(
@@ -74,6 +82,24 @@ hub_repository = repository_rule(
             mandatory = False,
             doc = """\
 The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys.
+""",
+        ),
+        # TODO @aignas 2025-03-23: get the `platforms` and `python_versions`
+        # from the aliases? This requires us to only create aliases for the
+        # target platforms and python_versions we care about. Python versions
+        # probably still need to be passed in.
+        "platforms": attr.string_list(
+            mandatory = True,
+            doc = """\
+The list of target platforms that are supported in this hub repository. This
+can contain 'abi_os_arch' tuples or 'host' to keep the lock files os/arch
+agnostic.
+""",
+        ),
+        "python_versions": attr.string_list(
+            mandatory = True,
+            doc = """\
+The list of python versions that are supported in this hub repository.
 """,
         ),
         "repo_name": attr.string(

python/private/pypi/requirements.bzl.tmpl.bzlmod

@@ -13,6 +13,15 @@ all_whl_requirements = all_whl_requirements_by_package.values()
 
 all_data_requirements = %%ALL_DATA_REQUIREMENTS%%
 
+# Internal constants used for passing information to `whl_library`
+# repositories without causing re-fetching. This is not intended to be
+# a stable API and may break. If you would like to depend on it, please raise
+# a ticket describing your use-case.
+private = struct(
+    python_versions = %%PYTHON_VERSIONS%%,
+    platforms = %%PLATFORMS%%,
+)
+
 def requirement(name):
     return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg")
 

python/private/pypi/whl_library.bzl

@@ -21,7 +21,6 @@ load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
 load(":attrs.bzl", "ATTRS", "use_isolated")
 load(":deps.bzl", "all_repo_names", "record_files")
 load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel")
-load(":parse_requirements.bzl", "host_platform")
 load(":parse_whl_name.bzl", "parse_whl_name")
 load(":patch_whl.bzl", "patch_whl")
 load(":pypi_repo_utils.bzl", "pypi_repo_utils")
@@ -354,6 +353,7 @@ def _whl_library_impl(rctx):
         entry_points[entry_point_without_py] = entry_point_script_name
 
     build_file_contents = generate_whl_library_build_bazel(
+        name = metadata["name"],
         whl_name = whl_path.basename,
         # TODO @aignas 2025-03-23: load the dep_template from the hub repository
         dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
@@ -369,25 +369,12 @@ def _whl_library_impl(rctx):
         entry_points = entry_points,
         # TODO @aignas 2025-03-23: store the annotation in the hub repo.
         annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
-        name = metadata["name"],
         requires_dist = metadata["requires_dist"],
-        # target the host platform if the target platform is not specified in the rule.
-        # TODO @aignas 2025-03-23: we should materialize this inside the
-        # hub_repository `requirements.bzl` file as `TARGET_PLATFORMS` with a
-        # note, that this is internal and will be only for usage of the
-        # `whl_library`
-        platforms = target_platforms or [
-            "{}_{}".format(metadata["abi"], host_platform(rctx)),
-        ],
-        # TODO @aignas 2025-03-23: we should expose the requested extras via a
-        # dict in `requirements.bzl` `EXTRAS` where the key is the package name
-        # and the value is the list of requested extras. like the above, for
-        # internal usage only.
+        # TODO @aignas 2025-03-23: we should just generate targets for all of the
+        # exposed extras and ask users to depend on this by `@pypi//foo:foo__extra`.
+        # Since all of the dependencies go through the `hub` repository, the extras
+        # that include packages that we don't
         extras = metadata["extras"],
-        # TODO @aignas 2025-03-23: we should expose full python version via the
-        # TARGET_PYTHON_VERSIONS list so that we can correctly calculate the
-        # deps. This would be again, internal only stuff.
-        host_python_version = metadata["python_version"],
     )
     rctx.file("BUILD.bazel", build_file_contents)
 

python/private/pypi/whl_library_targets.bzl

@@ -58,6 +58,7 @@ def whl_library_targets(
             copy_file = copy_file,
             py_binary = py_binary,
             py_library = py_library,
+            deps = deps,
         )):
     """Create all of the whl_library targets.
 
@@ -101,7 +102,7 @@ def whl_library_targets(
     # the analysis phase. This means that we need to get the target platform abi
     # from the python version/versions we are setting the package up for. We can
     # potentially get this from the python toolchain interpreter.
-    package_deps = deps(
+    package_deps = rules.deps(
         name = name,
         requires_dist = requires_dist,
         platforms = platforms,
@@ -112,11 +113,6 @@ def whl_library_targets(
     dependencies = package_deps.deps
     dependencies_by_platform = package_deps.deps_select
 
-    dependencies = sorted([normalize_name(d) for d in dependencies])
-    dependencies_by_platform = {
-        platform: sorted([normalize_name(d) for d in deps])
-        for platform, deps in dependencies_by_platform.items()
-    }
     tags = sorted(tags)
     data = [] + data