fix(venv): symlink shared libraries directly

docs/pyproject.toml

@@ -13,5 +13,7 @@ dependencies = [
     "absl-py",
     "typing-extensions",
     "sphinx-reredirects",
-    "pefile"
+    "pefile",
+    "pyelftools",
+    "macholib",
 ]

docs/requirements.txt

@@ -14,6 +14,10 @@ alabaster==1.0.0 ; python_full_version >= '3.10' \
     --hash=sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e \
     --hash=sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b
     # via sphinx
+altgraph==0.17.4 \
+    --hash=sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406 \
+    --hash=sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff
+    # via macholib
 astroid==3.3.11 \
     --hash=sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce \
     --hash=sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec
@@ -111,9 +115,9 @@ colorama==0.4.6 ; sys_platform == 'win32' \
     --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
     --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
     # via sphinx
-docutils==0.22.2 \
-    --hash=sha256:9fdb771707c8784c8f2728b67cb2c691305933d68137ef95a75db5f4dfbc213d \
-    --hash=sha256:b0e98d679283fc3bb0ead8a5da7f501baa632654e7056e9c5846842213d674d8
+docutils==0.21.2 \
+    --hash=sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f \
+    --hash=sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2
     # via
     #   myst-parser
     #   sphinx
@@ -137,9 +141,13 @@ jinja2==3.1.6 \
     #   myst-parser
     #   readthedocs-sphinx-ext
     #   sphinx
-markdown-it-py==4.0.0 \
-    --hash=sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147 \
-    --hash=sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3
+macholib==1.16.3 \
+    --hash=sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30 \
+    --hash=sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c
+    # via rules-python-docs (docs/pyproject.toml)
+markdown-it-py==3.0.0 \
+    --hash=sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1 \
+    --hash=sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb
     # via
     #   mdit-py-plugins
     #   myst-parser
@@ -264,6 +272,10 @@ pefile==2024.8.26 \
     --hash=sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632 \
     --hash=sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f
     # via rules-python-docs (docs/pyproject.toml)
+pyelftools==0.32 \
+    --hash=sha256:013df952a006db5e138b1edf6d8a68ecc50630adbd0d83a2d41e7f846163d738 \
+    --hash=sha256:6de90ee7b8263e740c8715a925382d4099b354f29ac48ea40d840cf7aa14ace5
+    # via rules-python-docs (docs/pyproject.toml)
 pygments==2.19.2 \
     --hash=sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887 \
     --hash=sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b
@@ -432,39 +444,49 @@ sphinxcontrib-serializinghtml==2.0.0 \
     --hash=sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331 \
     --hash=sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d
     # via sphinx
-tomli==2.2.1 ; python_full_version < '3.11' \
-    --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \
-    --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \
-    --hash=sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c \
-    --hash=sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b \
-    --hash=sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8 \
-    --hash=sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6 \
-    --hash=sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77 \
-    --hash=sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff \
-    --hash=sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea \
-    --hash=sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192 \
-    --hash=sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249 \
-    --hash=sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee \
-    --hash=sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4 \
-    --hash=sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98 \
-    --hash=sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8 \
-    --hash=sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4 \
-    --hash=sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281 \
-    --hash=sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744 \
-    --hash=sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69 \
-    --hash=sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13 \
-    --hash=sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140 \
-    --hash=sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e \
-    --hash=sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e \
-    --hash=sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc \
-    --hash=sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff \
-    --hash=sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec \
-    --hash=sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2 \
-    --hash=sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222 \
-    --hash=sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106 \
-    --hash=sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272 \
-    --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \
-    --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7
+tomli==2.3.0 ; python_full_version < '3.11' \
+    --hash=sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456 \
+    --hash=sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845 \
+    --hash=sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999 \
+    --hash=sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0 \
+    --hash=sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878 \
+    --hash=sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf \
+    --hash=sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3 \
+    --hash=sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be \
+    --hash=sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52 \
+    --hash=sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b \
+    --hash=sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67 \
+    --hash=sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549 \
+    --hash=sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba \
+    --hash=sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22 \
+    --hash=sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c \
+    --hash=sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f \
+    --hash=sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6 \
+    --hash=sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba \
+    --hash=sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45 \
+    --hash=sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f \
+    --hash=sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77 \
+    --hash=sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606 \
+    --hash=sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441 \
+    --hash=sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0 \
+    --hash=sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f \
+    --hash=sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530 \
+    --hash=sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05 \
+    --hash=sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8 \
+    --hash=sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005 \
+    --hash=sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879 \
+    --hash=sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae \
+    --hash=sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc \
+    --hash=sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b \
+    --hash=sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b \
+    --hash=sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e \
+    --hash=sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf \
+    --hash=sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac \
+    --hash=sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8 \
+    --hash=sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b \
+    --hash=sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf \
+    --hash=sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463 \
+    --hash=sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876
     # via
     #   sphinx
     #   sphinx-autodoc2

python/private/py_info.bzl

@@ -47,12 +47,17 @@ VenvSymlinkKind = struct(
     INCLUDE = "INCLUDE",
 )
 
+def _VenvSymlinkEntry_init(**kwargs):
+    kwargs.setdefault("link_to_file", None)
+    return kwargs
+
 # A provider is used for memory efficiency.
 # buildifier: disable=name-conventions
-VenvSymlinkEntry = provider(
+VenvSymlinkEntry, _ = provider(
     doc = """
 An entry in `PyInfo.venv_symlinks`
 """,
+    init = _VenvSymlinkEntry_init,
     fields = {
         "files": """
 :type: depset[File]
@@ -67,12 +72,18 @@ if one adds files to `venv_path=a/` and another adds files to `venv_path=a/b/`.
 
 One of the {obj}`VenvSymlinkKind` values. It represents which directory within
 the venv to create the path under.
+""",
+        "link_to_file": """
+:type: File | None
+
+A file that `venv_path` should point to. The file to link to should also be in
+`files`.
 """,
         "link_to_path": """
 :type: str | None
 
-A runfiles-root relative path that `venv_path` will symlink to. If `None`,
-it means to not create a symlink.
+A runfiles-root relative path that `venv_path` will symlink to (if
+`link_to_file` is None). If `None`, it means to not create it in the venv.
 """,
         "package": """
 :type: str | None

python/private/venv_runfiles.bzl

@@ -81,7 +81,7 @@ def build_link_map(ctx, entries):
     Returns:
         {type}`dict[str, dict[str, str|File]]` Mappings of venv paths to their
         backing files. The first key is a `VenvSymlinkKind` value.
-        The inner dict keys are venv paths relative to the kind's diretory. The
+        The inner dict keys are venv paths relative to the kind's directory. The
         inner dict values are strings or Files to link to.
     """
 
@@ -116,7 +116,10 @@ def build_link_map(ctx, entries):
             # If there's just one group, we can symlink to the directory
             if len(group) == 1:
                 entry = group[0]
-                keep_kind_link_map[entry.venv_path] = entry.link_to_path
+                if entry.link_to_file:
+                    keep_kind_link_map[entry.venv_path] = entry.link_to_file
+                else:
+                    keep_kind_link_map[entry.venv_path] = entry.link_to_path
             else:
                 # Merge a group of overlapping prefixes
                 _merge_venv_path_group(ctx, group, keep_kind_link_map)
@@ -172,7 +175,9 @@ def _merge_venv_path_group(ctx, group, keep_map):
     # TODO: Compute the minimum number of entries to create. This can't avoid
     # flattening the files depset, but can lower the number of materialized
     # files significantly. Usually overlaps are limited to a small number
-    # of directories.
+    # of directories. Note that, when doing so, shared libraries need to
+    # be symlinked directly, not the directory containing them, due to
+    # dynamic linker symlink resolution semantics on Linux.
     for entry in group:
         prefix = entry.venv_path
         for file in entry.files.to_list():
@@ -249,13 +254,27 @@ def get_venv_symlinks(ctx, files, package, version_str, site_packages_root):
             continue
         path = path.removeprefix(site_packages_root)
         dir_name, _, filename = path.rpartition("/")
+        runfiles_dir_name, _, _ = runfiles_root_path(ctx, src.short_path).partition("/")
+
+        if _is_linker_loaded_library(filename):
+            entry = VenvSymlinkEntry(
+                kind = VenvSymlinkKind.LIB,
+                # todo: omit setting link_to_path
+                link_to_path = paths.join(runfiles_dir_name, site_packages_root, filename),
+                link_to_file = src,
+                package = package,
+                version = version_str,
+                venv_path = path,
+                files = depset([src]),
+            )
+            venv_symlinks.append(entry)
+            continue
 
         if dir_name in dir_symlinks:
             # we already have this dir, this allows us to short-circuit since most of the
             # ctx.files.data might share the same directories as ctx.files.srcs
             continue
 
-        runfiles_dir_name, _, _ = runfiles_root_path(ctx, src.short_path).partition("/")
         if dir_name:
             # This can be either:
             # * a directory with libs (e.g. numpy.libs, created by auditwheel)
@@ -278,6 +297,8 @@ def get_venv_symlinks(ctx, files, package, version_str, site_packages_root):
                 files = depset([src]),
             )
             venv_symlinks.append(entry)
+        else:
+            fail("hit", src)
 
     # Sort so that we encounter `foo` before `foo/bar`. This ensures we
     # see the top-most explicit package first.
@@ -312,6 +333,22 @@ def get_venv_symlinks(ctx, files, package, version_str, site_packages_root):
 
     return venv_symlinks
 
+def _is_linker_loaded_library(filename):
+    """Tells if a filename is one that `dlopen()` or the runtime linker handles.
+
+    This should return true for `libfoo.so` (regular C library), but not
+    `foo.so` (Python C extension library).
+    """
+    if not filename.startswith("lib"):
+        return False
+    if filename.endswith((".so", ".dylib", ".lib")):
+        return True
+
+    # Versioned library, e.g. libfoo.so.1.2.3
+    if ".so." in filename:
+        return True
+    return False
+
 def _repo_relative_short_path(short_path):
     # Convert `../+pypi+foo/some/file.py` to `some/file.py`
     if short_path.startswith("../"):

tests/support/copy_file.bzl

@@ -0,0 +1,25 @@
+"""Copies a file to a directory."""
+
+def _copy_file_to_dir_impl(ctx):
+    out_file = ctx.actions.declare_file(
+        "{}/{}".format(ctx.attr.out_dir, ctx.file.src.basename),
+    )
+    ctx.actions.run_shell(
+        inputs = [ctx.file.src],
+        outputs = [out_file],
+        arguments = [ctx.file.src.path, out_file.path],
+        command = 'cp -f "$1" "$2"',
+        progress_message = "Copying %{input} to %{output}",
+    )
+    return [DefaultInfo(files = depset([out_file]))]
+
+copy_file_to_dir = rule(
+    implementation = _copy_file_to_dir_impl,
+    attrs = {
+        "out_dir": attr.string(mandatory = True),
+        "src": attr.label(
+            allow_single_file = True,
+            mandatory = True,
+        ),
+    },
+)

tests/venv_site_packages_libs/BUILD.bazel

@@ -1,6 +1,10 @@
 load("//python:py_library.bzl", "py_library")
 load("//tests/support:py_reconfig.bzl", "py_reconfig_test")
-load("//tests/support:support.bzl", "SUPPORTS_BOOTSTRAP_SCRIPT")
+load(
+    "//tests/support:support.bzl",
+    "NOT_WINDOWS",
+    "SUPPORTS_BOOTSTRAP_SCRIPT",
+)
 
 py_library(
     name = "user_lib",
@@ -34,3 +38,17 @@ py_reconfig_test(
         "@other//with_external_data",
     ],
 )
+
+py_reconfig_test(
+    name = "shared_lib_loading_test",
+    srcs = ["shared_lib_loading_test.py"],
+    bootstrap_impl = "script",
+    main = "shared_lib_loading_test.py",
+    target_compatible_with = NOT_WINDOWS,
+    venvs_site_packages = "yes",
+    deps = [
+        "//tests/venv_site_packages_libs/ext_with_libs",
+        "@dev_pip//macholib",
+        "@dev_pip//pyelftools",
+    ],
+)