fix: Add libdir to library search path (#2476)

We discovered when dealing with libraries such as `psycopg2` that the
wheel would attempt to link against `libpython.a`. This fix points the
linker at the correct python version being used.

---------

Co-authored-by: Ignas Anikevicius <[email protected]>

Author: shs96c

CHANGELOG.md

@@ -77,6 +77,8 @@ Unreleased changes template.
 * {obj}`//python/bin:python`: convenience target for directly running an
   interpreter. {obj}`--//python/bin:python_src` can be used to specify a
   binary whose interpreter to use.
+* (pypi) An extra argument to add the interpreter lib dir to `LDFLAGS` when
+  building wheels from `sdist`.
 
 {#v0-0-0-removed}
 ### Removed

python/private/pypi/attrs.bzl

@@ -15,6 +15,15 @@
 "common attributes for whl_library and pip_repository"
 
 ATTRS = {
+    "add_libdir_to_library_search_path": attr.bool(
+        default = False,
+        doc = """
+If true, add the lib dir of the bundled interpreter to the library search path via `LDFLAGS`.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+""",
+    ),
     "download_only": attr.bool(
         doc = """
 Whether to use "pip download" instead of "pip wheel". Disables building wheels from source, but allows use of

python/private/pypi/extension.bzl

@@ -203,6 +203,7 @@ def _create_whl_repos(
         )
         maybe_args = dict(
             # The following values are safe to omit if they have false like values
+            add_libdir_to_library_search_path = pip_attr.add_libdir_to_library_search_path,
             annotation = whl_modifications.get(whl_name),
             download_only = pip_attr.download_only,
             enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs,

python/private/pypi/whl_library.bzl

@@ -140,11 +140,28 @@ def _parse_optional_attrs(rctx, args, extra_pip_args = None):
     if rctx.attr.enable_implicit_namespace_pkgs:
         args.append("--enable_implicit_namespace_pkgs")
 
+    env = {}
     if rctx.attr.environment != None:
-        args += [
-            "--environment",
-            json.encode(struct(arg = rctx.attr.environment)),
-        ]
+        for key, value in rctx.attr.environment.items():
+            env[key] = value
+
+    # This is super hacky, but working out something nice is tricky.
+    # This is in particular needed for psycopg2 which attempts to link libpython.a,
+    # in order to point the linker at the correct python intepreter.
+    if rctx.attr.add_libdir_to_library_search_path:
+        if "LDFLAGS" in env:
+            fail("Can't set both environment LDFLAGS and add_libdir_to_library_search_path")
+        command = [pypi_repo_utils.resolve_python_interpreter(rctx), "-c", "import sys ; sys.stdout.write('{}/lib'.format(sys.exec_prefix))"]
+        result = rctx.execute(command)
+        if result.return_code != 0:
+            fail("Failed to get LDFLAGS path: command: {}, exit code: {}, stdout: {}, stderr: {}".format(command, result.return_code, result.stdout, result.stderr))
+        libdir = result.stdout
+        env["LDFLAGS"] = "-L{}".format(libdir)
+
+    args += [
+        "--environment",
+        json.encode(struct(arg = env)),
+    ]
 
     return args
 

tests/pypi/extension/extension_tests.bzl

@@ -77,6 +77,7 @@ def _parse(
         hub_name,
         python_version,
         _evaluate_markers_srcs = [],
+        add_libdir_to_library_search_path = False,
         auth_patterns = {},
         download_only = False,
         enable_implicit_namespace_pkgs = False,
@@ -105,6 +106,7 @@ def _parse(
     return struct(
         _evaluate_markers_srcs = _evaluate_markers_srcs,
         auth_patterns = auth_patterns,
+        add_libdir_to_library_search_path = add_libdir_to_library_search_path,
         download_only = download_only,
         enable_implicit_namespace_pkgs = enable_implicit_namespace_pkgs,
         environment = environment,