feat: allow populating binary's venv site-packages with symlinks

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma
 
 test --test_output=errors
 

CHANGELOG.md

@@ -91,6 +91,10 @@ Unreleased changes template.
   please check the {obj}`uv.configure` tag class.
 * Add support for riscv64 linux platform.
 * (toolchains) Add python 3.13.2 and 3.12.9 toolchains
+* (providers) (experimental) {obj}`PyInfo.site_packages_symlinks` field added to
+  allow specifying links to create within the venv site packages (only
+  applicable with {obj}`--bootstrap_impl=script`)
+  ([#2156](https://github.com/bazelbuild/rules_python/issues/2156)).
 
 {#v0-0-0-removed}
 ### Removed

MODULE.bazel

@@ -85,6 +85,7 @@ bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True)
 bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True)
 bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
 bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True)
+bazel_dep(name = "other", version = "0", dev_dependency = True)
 
 # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests.
 # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides.
@@ -106,6 +107,11 @@ local_path_override(
     path = "gazelle",
 )
 
+local_path_override(
+    module_name = "other",
+    path = "tests/modules/other",
+)
+
 dev_python = use_extension(
     "//python/extensions:python.bzl",
     "python",

docs/BUILD.bazel

@@ -87,6 +87,7 @@ sphinx_stardocs(
     name = "bzl_api_docs",
     srcs = [
         "//python:defs_bzl",
+        "//python:features_bzl",
         "//python:packaging_bzl",
         "//python:pip_bzl",
         "//python:py_binary_bzl",

docs/_includes/experimental_api.md

@@ -0,0 +1,5 @@
+:::{warning}
+
+**Experimental API.** This API is still under development and may change or be
+removed without notice.
+:::

docs/api/rules_python/python/config_settings/index.md

@@ -213,6 +213,23 @@ Values:
 ::::
 
 
+::::
+
+:::{flag} venvs_site_packages
+
+Determines if libraries use a site-packages layout for their files.
+
+Note this flag only affects PyPI dependencies of `--bootstrap_impl=script` binaries
+
+:::{include} /_includes/experimental_api.md
+:::
+
+
+Values:
+* `no` (default): Make libraries importable by adding to `sys.path`
+* `yes`: Make libraries importable by creating paths in a binary's site-packages directory.
+::::
+
 ::::{bzl:flag} bootstrap_impl
 Determine how programs implement their startup process.
 

internal_dev_deps.bzl

@@ -15,6 +15,7 @@
 """Dependencies that are needed for development and testing of rules_python itself."""
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive", _http_file = "http_file")
+load("@bazel_tools//tools/build_defs/repo:local.bzl", "local_repository")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 load("//python/private:internal_config_repo.bzl", "internal_config_repo")  # buildifier: disable=bzl-visibility
 
@@ -42,6 +43,11 @@ def rules_python_internal_deps():
     """
     internal_config_repo(name = "rules_python_internal")
 
+    local_repository(
+        name = "other",
+        path = "tests/modules/other",
+    )
+
     http_archive(
         name = "bazel_skylib",
         sha256 = "bc283cdfcd526a52c3201279cda4bc298652efa898b10b4db0837dc51652756f",

python/BUILD.bazel

@@ -79,6 +79,9 @@ bzl_library(
 bzl_library(
     name = "features_bzl",
     srcs = ["features.bzl"],
+    deps = [
+        "@rules_python_internal//:rules_python_config_bzl",
+    ],
 )
 
 bzl_library(

python/config_settings/BUILD.bazel

@@ -9,6 +9,7 @@ load(
     "LibcFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
+    "VenvsSitePackages",
     "VenvsUseDeclareSymlinkFlag",
 )
 load(
@@ -195,6 +196,13 @@ string_flag(
     visibility = ["//visibility:public"],
 )
 
+string_flag(
+    name = "venvs_site_packages",
+    build_setting_default = VenvsSitePackages.NO,
+    # NOTE: Only public because it is used in pip hub repos.
+    visibility = ["//visibility:public"],
+)
+
 define_pypi_internal_flags(
     name = "define_pypi_internal_flags",
 )

python/features.bzl

@@ -19,8 +19,49 @@ load("@rules_python_internal//:rules_python_config.bzl", "config")
 # See https://git-scm.com/docs/git-archive/2.29.0#Documentation/git-archive.txt-export-subst
 _VERSION_PRIVATE = "$Format:%(describe:tags=true)$"
 
+def _features_typedef():
+    """Information about features rules_python has implemented.
+
+    ::::{field} precompile
+    :type: bool
+
+    True if the precompile attributes are available.
+
+    :::{versionadded} 0.33.0
+    :::
+    ::::
+
+    ::::{field} py_info_site_packages_symlinks
+
+    True if the `PyInfo.site_packages_symlinks` field is available.
+
+    :::{versionadded} VERSION_NEXT_FEATURE
+    :::
+    ::::
+
+    ::::{field} uses_builtin_rules
+    :type: bool
+
+    True if the rules are using the Bazel-builtin implementation.
+
+    :::{versionadded} 1.1.0
+    :::
+    ::::
+
+    ::::{field} version
+    :type: str
+
+    The rules_python version. This is a semver format, e.g. `X.Y.Z` with
+    optional trailing `-rcN`. For unreleased versions, it is an empty string.
+    :::{versionadded} 0.38.0
+    ::::
+    """
+
 features = struct(
-    version = _VERSION_PRIVATE if "$Format" not in _VERSION_PRIVATE else "",
+    TYPEDEF = _features_typedef,
+    # keep sorted
     precompile = True,
+    py_info_site_packages_symlinks = True,
     uses_builtin_rules = not config.enable_pystar,
+    version = _VERSION_PRIVATE if "$Format" not in _VERSION_PRIVATE else "",
 )

python/private/attributes.bzl

@@ -262,6 +262,17 @@ These are typically `py_library` rules.
 
 Targets that only provide data files used at runtime belong in the `data`
 attribute.
+
+:::{note}
+The order of this list can matter because it affects the order that information
+from dependencies is merged in, which can be relevant depending on the ordering
+mode of depsets that are merged.
+
+* {obj}`PyInfo.site_packages_symlinks` uses topological ordering.
+
+See {obj}`PyInfo` for more information about the ordering of its depsets and
+how its fields are merged.
+:::
 """,
         ),
         "precompile": lambda: attrb.String(

python/private/builders.bzl

@@ -15,12 +15,19 @@
 
 load("@bazel_skylib//lib:types.bzl", "types")
 
-def _DepsetBuilder():
-    """Create a builder for a depset."""
+def _DepsetBuilder(order = None):
+    """Create a builder for a depset.
+
+    Args:
+        order: {type}`str | None` The order to initialize the depset to, if any.
+
+    Returns:
+        {type}`DepsetBuilder`
+    """
 
     # buildifier: disable=uninitialized
     self = struct(
-        _order = [None],
+        _order = [order],
         add = lambda *a, **k: _DepsetBuilder_add(self, *a, **k),
         build = lambda *a, **k: _DepsetBuilder_build(self, *a, **k),
         direct = [],

python/private/common.bzl

@@ -30,6 +30,16 @@ PackageSpecificationInfo = getattr(py_internal, "PackageSpecificationInfo", None
 # Extensions without the dot
 _PYTHON_SOURCE_EXTENSIONS = ["py"]
 
+# Extensions that mean a file is relevant to Python
+PYTHON_FILE_EXTENSIONS = [
+    "dll",  # Python C modules, Windows specific
+    "dylib",  # Python C modules, Mac specific
+    "py",
+    "pyc",
+    "pyi",
+    "so",  # Python C modules, usually Linux
+]
+
 def create_binary_semantics_struct(
         *,
         create_executable,
@@ -367,7 +377,8 @@ def create_py_info(
         required_pyc_files,
         implicit_pyc_files,
         implicit_pyc_source_files,
-        imports):
+        imports,
+        site_packages_symlinks = []):
     """Create PyInfo provider.
 
     Args:
@@ -385,13 +396,17 @@ def create_py_info(
         implicit_pyc_files: {type}`depset[File]` Implicitly generated pyc files
             that a binary can choose to include.
         imports: depset of strings; the import path values to propagate.
+        site_packages_symlinks: {type}`list[tuple[str, str]]` tuples of
+            `(runfiles_path, site_packages_path)` for symlinks to create
+            in the consuming binary's venv site packages.
 
     Returns:
         A tuple of the PyInfo instance and a depset of the
         transitive sources collected from dependencies (the latter is only
         necessary for deprecated extra actions support).
     """
     py_info = PyInfoBuilder()
+    py_info.site_packages_symlinks.add(site_packages_symlinks)
     py_info.direct_original_sources.add(original_sources)
     py_info.direct_pyc_files.add(required_pyc_files)
     py_info.direct_pyi_files.add(ctx.files.pyi_srcs)

python/private/enum.bzl

@@ -43,3 +43,23 @@ def enum(methods = {}, **kwargs):
 
     self = struct(__members__ = members, **kwargs)
     return self
+
+def _FlagEnum_flag_values(self):
+    return sorted(self.__members__.values())
+
+def FlagEnum(**kwargs):
+    """Define an enum specialized for flags.
+
+    Args:
+        **kwargs: members of the enum.
+
+    Returns:
+        {type}`FlagEnum` struct. This is an enum with the following extras:
+        * `flag_values`: A function that returns a sorted list of the
+          flag values (enum `__members__`). Useful for passing to the
+          `values` attribute for string flags.
+    """
+    return enum(
+        methods = dict(flag_values = _FlagEnum_flag_values),
+        **kwargs
+    )

python/private/flags.bzl

@@ -19,27 +19,7 @@ unnecessary files when all that are needed are flag definitions.
 """
 
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
-load(":enum.bzl", "enum")
-
-def _FlagEnum_flag_values(self):
-    return sorted(self.__members__.values())
-
-def FlagEnum(**kwargs):
-    """Define an enum specialized for flags.
-
-    Args:
-        **kwargs: members of the enum.
-
-    Returns:
-        {type}`FlagEnum` struct. This is an enum with the following extras:
-        * `flag_values`: A function that returns a sorted list of the
-          flag values (enum `__members__`). Useful for passing to the
-          `values` attribute for string flags.
-    """
-    return enum(
-        methods = dict(flag_values = _FlagEnum_flag_values),
-        **kwargs
-    )
+load(":enum.bzl", "FlagEnum", "enum")
 
 def _AddSrcsToRunfilesFlag_is_enabled(ctx):
     value = ctx.attr._add_srcs_to_runfiles_flag[BuildSettingInfo].value
@@ -138,6 +118,22 @@ VenvsUseDeclareSymlinkFlag = FlagEnum(
     get_value = _venvs_use_declare_symlink_flag_get_value,
 )
 
+def _venvs_site_packages_is_enabled(ctx):
+    if not ctx.attr.experimental_venvs_site_packages:
+        return False
+    flag_value = ctx.attr.experimental_venvs_site_packages[BuildSettingInfo].value
+    return flag_value == VenvsSitePackages.YES
+
+# Decides if libraries try to use a site-packages layout using site_packages_symlinks
+# buildifier: disable=name-conventions
+VenvsSitePackages = FlagEnum(
+    # Use site_packages_symlinks
+    YES = "yes",
+    # Don't use site_packages_symlinks
+    NO = "no",
+    is_enabled = _venvs_site_packages_is_enabled,
+)
+
 # Used for matching freethreaded toolchains and would have to be used in wheels
 # as well.
 # buildifier: disable=name-conventions