wip: libs in venv site packages

Author: rickeylev

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma
 
 test --test_output=errors
 

CHANGELOG.md

@@ -60,7 +60,14 @@ Unreleased changes template.
 
 {#v0-0-0-added}
 ### Added
-* Nothing added.
+* (providers) {obj}`PyInfo.site_packages_symlinks` field added to allow
+  specifying links to create within the venv site packages
+  (only applicable with {obj}`--bootstrap_impl=script`)
+  ([#2156](https://github.com/bazelbuild/rules_python/issues/2156)).
+* (rules) {obj}`py_library.site_packages_root` attribute added to allow
+  specifying a library's sources follow a site-packages file layout.
+  (only applicable with {obj}`--bootstrap_impl=script`)
+  ([#2156](https://github.com/bazelbuild/rules_python/issues/2156)).
 
 {#v0-0-0-removed}
 ### Removed

MODULE.bazel

@@ -85,6 +85,7 @@ bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True)
 bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True)
 bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
 bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True)
+bazel_dep(name = "other", version = "0", dev_dependency = True)
 
 # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests.
 # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides.
@@ -106,6 +107,11 @@ local_path_override(
     path = "gazelle",
 )
 
+local_path_override(
+    module_name = "other",
+    path = "tests/modules/other",
+)
+
 dev_python = use_extension(
     "//python/extensions:python.bzl",
     "python",

python/features.bzl

@@ -23,4 +23,5 @@ features = struct(
     version = _VERSION_PRIVATE if "$Format" not in _VERSION_PRIVATE else "",
     precompile = True,
     uses_builtin_rules = not config.enable_pystar,
+    site_packages_root_attr = True,
 )

python/private/attributes.bzl

@@ -282,6 +282,12 @@ that depend on this rule. The strings are repo-runfiles-root relative,
 
 Absolute paths (paths that start with `/`) and paths that references a path
 above the execution root are not allowed and will result in an error.
+
+:::{attention}
+Setting both this and the {attr}`site_packages_root` attribute may result in
+undefined behavior. Both will result in the code being importable, but from
+different sys.path (and thus `__file__`) entries.
+:::
 """,
     ),
 }

python/private/builders.bzl

@@ -15,12 +15,19 @@
 
 load("@bazel_skylib//lib:types.bzl", "types")
 
-def _DepsetBuilder():
-    """Create a builder for a depset."""
+def _DepsetBuilder(order = None):
+    """Create a builder for a depset.
+
+    Args:
+        order: {type}`str | None` The order to initialize the depset to, if any.
+
+    Returns:
+        {type}`DepsetBuilder`
+    """
 
     # buildifier: disable=uninitialized
     self = struct(
-        _order = [None],
+        _order = [order],
         add = lambda *a, **k: _DepsetBuilder_add(self, *a, **k),
         build = lambda *a, **k: _DepsetBuilder_build(self, *a, **k),
         direct = [],

python/private/common.bzl

@@ -30,6 +30,15 @@ PackageSpecificationInfo = getattr(py_internal, "PackageSpecificationInfo", None
 # Extensions without the dot
 _PYTHON_SOURCE_EXTENSIONS = ["py"]
 
+# Extensions that make a file considered importable
+PYTHON_FILE_EXTENSIONS = [
+    "py",
+    "so",  # Python C modules, usually Linux
+    "dylib",  # Python C modules, Mac specific
+    "pyc",
+    "dll",  # Python C modules, Windows specific
+]
+
 def create_binary_semantics_struct(
         *,
         create_executable,
@@ -413,7 +422,8 @@ def create_py_info(
         required_pyc_files,
         implicit_pyc_files,
         implicit_pyc_source_files,
-        imports):
+        imports,
+        site_packages_symlinks = []):
     """Create PyInfo provider.
 
     Args:
@@ -431,13 +441,17 @@ def create_py_info(
         implicit_pyc_files: {type}`depset[File]` Implicitly generated pyc files
             that a binary can choose to include.
         imports: depset of strings; the import path values to propagate.
+        site_packages_symlinks: {type}`list[tuple[str, str]]` tuples of
+            `(runfiles_path, site_packages_path)` for symlinks to create
+            in the consuming binary's venv site packages.
 
     Returns:
         A tuple of the PyInfo instance and a depset of the
         transitive sources collected from dependencies (the latter is only
         necessary for deprecated extra actions support).
     """
     py_info = PyInfoBuilder()
+    py_info.site_packages_symlinks.add(site_packages_symlinks)
     py_info.direct_original_sources.add(original_sources)
     py_info.direct_pyc_files.add(required_pyc_files)
     py_info.direct_pyi_files.add(ctx.files.pyi_srcs)

python/private/py_executable.bzl

@@ -591,15 +591,78 @@ def _create_venv(ctx, output_prefix, imports, runtime_details):
         },
         computed_substitutions = computed_subs,
     )
+    site_packages_symlinks = _create_site_packages_symlinks(ctx, site_packages)
 
     return struct(
         interpreter = interpreter,
         recreate_venv_at_runtime = not venvs_use_declare_symlink_enabled,
         # Runfiles root relative path or absolute path
         interpreter_actual_path = interpreter_actual_path,
-        files_without_interpreter = [pyvenv_cfg, pth, site_init],
+        files_without_interpreter = [pyvenv_cfg, pth, site_init] + site_packages_symlinks,
     )
 
+def _create_site_packages_symlinks(ctx, site_packages):
+    """Creates symlinks within site-packages.
+
+    Args:
+        ctx: current rule ctx
+        site_packages: runfiles-root-relative path to the site-packages directory
+
+    Returns:
+        {type}`list[File]` list of the File symlink objects created.
+    """
+
+    # maps site-package symlink to the runfiles path it should point to
+    entries = depset(
+        # NOTE: Topological ordering is used so that dependencies closer to the
+        # binary have precedence in creating their symlinks. This allows the
+        # binary a modicum of control over the result.
+        order = "topological",
+        transitive = [
+            dep[PyInfo].site_packages_symlinks
+            for dep in ctx.attr.deps
+            if PyInfo in dep
+        ],
+    ).to_list()
+    link_map = {}
+    for link_to_runfiles_path, site_packages_path in entries:
+        if site_packages_path in link_map:
+            # We ignore duplicates by design. The dependency closer to the
+            # binary gets precedence due to the topological ordering.
+            continue
+        else:
+            link_map[site_packages_path] = link_to_runfiles_path
+
+    # An empty link_to value means to not create the site package symlink.
+    # Because of the topological ordering, this allows binaries to remove
+    # entries by having an earlier dependency produce empty link_to values
+    for sp_dir_path, link_to in link_map.items():
+        if not link_to:
+            link_map.pop(sp_dir_path)
+
+    # This is N^2; we can certainly do better by sorting and exploiting the
+    # order.
+    # A trailing slash is appended / to prevent /X matching /XY
+    sp_dirs = [x + "/" for x in link_map.keys()]
+    for search_for in sp_dirs:
+        for prefix in sp_dirs:
+            if search_for != prefix and search_for.startswith(prefix):
+                fail("sub-link: {} under {}", search_for, prefix)
+
+    sp_files = []
+    for sp_dir_path, link_to in link_map.items():
+        sp_link = ctx.actions.declare_symlink(paths.join(site_packages, sp_dir_path))
+        sp_link_rf_path = runfiles_root_path(ctx, sp_link.short_path)
+        rel_path = relative_path(
+            # dirname is necessary because a relative symlink is relative to
+            # the directory the symlink resides within.
+            from_ = paths.dirname(sp_link_rf_path),
+            to = link_to,
+        )
+        ctx.actions.symlink(output = sp_link, target_path = rel_path)
+        sp_files.append(sp_link)
+    return sp_files
+
 def _map_each_identity(v):
     return v
 

python/private/py_info.bzl

@@ -42,7 +42,8 @@ def _PyInfo_init(
         direct_original_sources = depset(),
         transitive_original_sources = depset(),
         direct_pyi_files = depset(),
-        transitive_pyi_files = depset()):
+        transitive_pyi_files = depset(),
+        site_packages_symlinks = depset()):
     _check_arg_type("transitive_sources", "depset", transitive_sources)
 
     # Verify it's postorder compatible, but retain is original ordering.
@@ -70,6 +71,7 @@ def _PyInfo_init(
         "has_py2_only_sources": has_py2_only_sources,
         "has_py3_only_sources": has_py2_only_sources,
         "imports": imports,
+        "site_packages_symlinks": site_packages_symlinks,
         "transitive_implicit_pyc_files": transitive_implicit_pyc_files,
         "transitive_implicit_pyc_source_files": transitive_implicit_pyc_source_files,
         "transitive_original_sources": transitive_original_sources,
@@ -140,6 +142,31 @@ A depset of import path strings to be added to the `PYTHONPATH` of executable
 Python targets. These are accumulated from the transitive `deps`.
 The order of the depset is not guaranteed and may be changed in the future. It
 is recommended to use `default` order (the default).
+""",
+        "site_packages_symlinks": """
+:type: depset[tuple[str | None, str]]
+
+A depset with `topological` ordering.
+
+Tuples of `(runfiles_path, site_packages_path)`. Where
+* `runfiles_path` is a runfiles-root relative path. It is the path that
+  has the code to make importable. If `None` or empty string, then it means
+  to not create a site packages directory with the `site_packages_path`
+  name.
+* `site_packages_path` is a path relative to the site-packages directory of
+  the venv for whatever creates the venv (typically py_binary). It makes
+  the code in `runfiles_path` available for import. Note that this
+  is created as a "raw" symlink (via `declare_symlink`).
+
+:::{tip}
+The topological ordering means dependencies earlier and closer to the consumer
+have precedence. This allows e.g. a binary to add dependencies that override
+values from further way dependencies, such as forcing symlinks to point to
+specific paths or preventing symlinks from being created.
+:::
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
 """,
         "transitive_implicit_pyc_files": """
 :type: depset[File]
@@ -266,6 +293,7 @@ def PyInfoBuilder():
         transitive_pyc_files = builders.DepsetBuilder(),
         transitive_pyi_files = builders.DepsetBuilder(),
         transitive_sources = builders.DepsetBuilder(),
+        site_packages_symlinks = builders.DepsetBuilder(order = "topological"),
     )
     return self
 
@@ -351,6 +379,7 @@ def _PyInfoBuilder_merge_all(self, transitive, *, direct = []):
             self.transitive_original_sources.add(info.transitive_original_sources)
             self.transitive_pyc_files.add(info.transitive_pyc_files)
             self.transitive_pyi_files.add(info.transitive_pyi_files)
+            self.site_packages_symlinks.add(info.site_packages_symlinks)
 
     return self
 
@@ -400,6 +429,7 @@ def _PyInfoBuilder_build(self):
             transitive_original_sources = self.transitive_original_sources.build(),
             transitive_pyc_files = self.transitive_pyc_files.build(),
             transitive_pyi_files = self.transitive_pyi_files.build(),
+            site_packages_symlinks = self.site_packages_symlinks.build(),
         )
     else:
         kwargs = {}