Merge branch 'main' of https://github.com/bazel-contrib/rules_python into feat.custom.toolchain.override

Author: rickeylev

MODULE.bazel

@@ -102,7 +102,6 @@ use_repo(
     internal_dev_deps,
     "buildkite_config",
     "rules_python_runtime_env_tc_info",
-    "wheel_for_testing",
 )
 
 # Add gazelle plugin so that we can run the gazelle example as an e2e integration

WORKSPACE

@@ -78,7 +78,7 @@ python_register_multi_toolchains(
     python_versions = PYTHON_VERSIONS,
 )
 
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 # Used for Bazel CI
 http_archive(
@@ -155,14 +155,3 @@ pip_parse(
 load("@dev_pip//:requirements.bzl", docs_install_deps = "install_deps")
 
 docs_install_deps()
-
-# This wheel is purely here to validate the wheel extraction code. It's not
-# intended for anything else.
-http_file(
-    name = "wheel_for_testing",
-    downloaded_file_path = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
-    sha256 = "0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2",
-    urls = [
-        "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
-    ],
-)

python/config_settings/BUILD.bazel

@@ -217,6 +217,15 @@ string_flag(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "is_venvs_site_packages",
+    flag_values = {
+        ":venvs_site_packages": VenvsSitePackages.YES,
+    },
+    # NOTE: Only public because it is used in whl_library repos.
+    visibility = ["//visibility:public"],
+)
+
 define_pypi_internal_flags(
     name = "define_pypi_internal_flags",
 )

python/private/internal_dev_deps.bzl

@@ -14,23 +14,11 @@
 """Module extension for internal dev_dependency=True setup."""
 
 load("@bazel_ci_rules//:rbe_repo.bzl", "rbe_preconfig")
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
 load(":runtime_env_repo.bzl", "runtime_env_repo")
 
 def _internal_dev_deps_impl(mctx):
     _ = mctx  # @unused
 
-    # This wheel is purely here to validate the wheel extraction code. It's not
-    # intended for anything else.
-    http_file(
-        name = "wheel_for_testing",
-        downloaded_file_path = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
-        sha256 = "0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2",
-        urls = [
-            "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
-        ],
-    )
-
     # Creates a default toolchain config for RBE.
     # Use this as is if you are using the rbe_ubuntu16_04 container,
     # otherwise refer to RBE docs.

python/private/pypi/BUILD.bazel

@@ -18,6 +18,11 @@ package(default_visibility = ["//:__subpackages__"])
 
 licenses(["notice"])
 
+exports_files(
+    srcs = ["namespace_pkg_tmpl.py"],
+    visibility = ["//visibility:public"],
+)
+
 filegroup(
     name = "distribution",
     srcs = glob(

python/private/pypi/namespace_pkg_tmpl.py

@@ -0,0 +1,2 @@
+# __path__ manipulation added by bazel-contrib/rules_python to support namespace pkgs.
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

python/private/pypi/namespace_pkgs.bzl

@@ -0,0 +1,83 @@
+"""Utilities to get where we should write namespace pkg paths."""
+
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+
+_ext = struct(
+    py = ".py",
+    pyd = ".pyd",
+    so = ".so",
+    pyc = ".pyc",
+)
+
+_TEMPLATE = Label("//python/private/pypi:namespace_pkg_tmpl.py")
+
+def _add_all(dirname, dirs):
+    dir_path = "."
+    for dir_name in dirname.split("/"):
+        dir_path = "{}/{}".format(dir_path, dir_name)
+        dirs[dir_path[2:]] = None
+
+def get_files(*, srcs, ignored_dirnames = [], root = None):
+    """Get the list of filenames to write the namespace pkg files.
+
+    Args:
+        srcs: {type}`src` a list of files to be passed to {bzl:obj}`py_library`
+            as `srcs` and `data`. This is usually a result of a {obj}`glob`.
+        ignored_dirnames: {type}`str` a list of patterns to ignore.
+        root: {type}`str` the prefix to use as the root.
+
+    Returns:
+        {type}`src` a list of paths to write the namespace pkg `__init__.py` file.
+    """
+    dirs = {}
+    ignored = {i: None for i in ignored_dirnames}
+
+    if root:
+        _add_all(root, ignored)
+
+    for file in srcs:
+        dirname, _, filename = file.rpartition("/")
+
+        if filename == "__init__.py":
+            ignored[dirname] = None
+            dirname, _, _ = dirname.rpartition("/")
+        elif filename.endswith(_ext.py):
+            pass
+        elif filename.endswith(_ext.pyc):
+            pass
+        elif filename.endswith(_ext.pyd):
+            pass
+        elif filename.endswith(_ext.so):
+            pass
+        else:
+            continue
+
+        if dirname in dirs or not dirname:
+            continue
+
+        _add_all(dirname, dirs)
+
+    return sorted([d for d in dirs if d not in ignored])
+
+def create_inits(**kwargs):
+    """Create init files and return the list to be included `py_library` srcs.
+
+    Args:
+        **kwargs: passed to {obj}`get_files`.
+
+    Returns:
+        {type}`list[str]` to be included as part of `py_library`.
+    """
+    srcs = []
+    for out in get_files(**kwargs):
+        src = "{}/__init__.py".format(out)
+        srcs.append(srcs)
+
+        copy_file(
+            name = "_cp_{}_namespace".format(out),
+            src = _TEMPLATE,
+            out = src,
+            **kwargs
+        )
+
+    return srcs

python/private/pypi/whl_installer/arguments.py

@@ -57,11 +57,6 @@ def parser(**kwargs: Any) -> argparse.ArgumentParser:
         action="store",
         help="Additional data exclusion parameters to add to the pip packages BUILD file.",
     )
-    parser.add_argument(
-        "--enable_implicit_namespace_pkgs",
-        action="store_true",
-        help="Disables conversion of implicit namespace packages into pkg-util style packages.",
-    )
     parser.add_argument(
         "--environment",
         action="store",

python/private/pypi/whl_installer/wheel_installer.py

@@ -27,7 +27,7 @@
 
 from pip._vendor.packaging.utils import canonicalize_name
 
-from python.private.pypi.whl_installer import arguments, namespace_pkgs, wheel
+from python.private.pypi.whl_installer import arguments, wheel
 
 
 def _configure_reproducible_wheels() -> None:
@@ -77,35 +77,10 @@ def _parse_requirement_for_extra(
     return None, None
 
 
-def _setup_namespace_pkg_compatibility(wheel_dir: str) -> None:
-    """Converts native namespace packages to pkgutil-style packages
-
-    Namespace packages can be created in one of three ways. They are detailed here:
-    https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package
-
-    'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but
-    'native namespace packages' (1) do not.
-
-    We ensure compatibility with Bazel of method 1 by converting them into method 2.
-
-    Args:
-        wheel_dir: the directory of the wheel to convert
-    """
-
-    namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages(
-        wheel_dir,
-        ignored_dirnames=["%s/bin" % wheel_dir],
-    )
-
-    for ns_pkg_dir in namespace_pkg_dirs:
-        namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir)
-
-
 def _extract_wheel(
     wheel_file: str,
     extras: Dict[str, Set[str]],
     enable_pipstar: bool,
-    enable_implicit_namespace_pkgs: bool,
     platforms: List[wheel.Platform],
     installation_dir: Path = Path("."),
 ) -> None:
@@ -116,15 +91,11 @@ def _extract_wheel(
         installation_dir: the destination directory for installation of the wheel.
         extras: a list of extras to add as dependencies for the installed wheel
         enable_pipstar: if true, turns off certain operations.
-        enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is
     """
 
     whl = wheel.Wheel(wheel_file)
     whl.unzip(installation_dir)
 
-    if not enable_implicit_namespace_pkgs:
-        _setup_namespace_pkg_compatibility(installation_dir)
-
     metadata = {
         "entry_points": [
             {
@@ -168,7 +139,6 @@ def main() -> None:
             wheel_file=whl,
             extras=extras,
             enable_pipstar=args.enable_pipstar,
-            enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs,
             platforms=arguments.get_platforms(args),
         )
         return

python/private/pypi/whl_library.bzl

@@ -173,9 +173,6 @@ def _parse_optional_attrs(rctx, args, extra_pip_args = None):
             json.encode(struct(arg = rctx.attr.pip_data_exclude)),
         ]
 
-    if rctx.attr.enable_implicit_namespace_pkgs:
-        args.append("--enable_implicit_namespace_pkgs")
-
     env = {}
     if rctx.attr.environment != None:
         for key, value in rctx.attr.environment.items():
@@ -389,6 +386,8 @@ def _whl_library_impl(rctx):
             metadata_name = metadata.name,
             metadata_version = metadata.version,
             requires_dist = metadata.requires_dist,
+            # TODO @aignas 2025-05-17: maybe have a build flag for this instead
+            enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs,
             # TODO @aignas 2025-04-14: load through the hub:
             annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
             data_exclude = rctx.attr.pip_data_exclude,
@@ -457,6 +456,8 @@ def _whl_library_impl(rctx):
             name = whl_path.basename,
             dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format(rctx.attr.repo_prefix),
             entry_points = entry_points,
+            # TODO @aignas 2025-05-17: maybe have a build flag for this instead
+            enable_implicit_namespace_pkgs = rctx.attr.enable_implicit_namespace_pkgs,
             # TODO @aignas 2025-04-14: load through the hub:
             dependencies = metadata["deps"],
             dependencies_by_platform = metadata["deps_by_platform"],
@@ -580,7 +581,6 @@ attr makes `extra_pip_args` and `download_only` ignored.""",
             Label("//python/private/pypi/whl_installer:wheel.py"),
             Label("//python/private/pypi/whl_installer:wheel_installer.py"),
             Label("//python/private/pypi/whl_installer:arguments.py"),
-            Label("//python/private/pypi/whl_installer:namespace_pkgs.py"),
         ] + record_files.values(),
     ),
     "_rule_name": attr.string(default = "whl_library"),