fix(local_runtime): Improve local_runtime usability in macos / windows

python/private/get_local_runtime_info.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Returns information about the local Python runtime as JSON."""
+
 import json
 import os
 import sys
@@ -46,8 +48,8 @@ def _search_directories(get_config):
     # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842
     multiarch = get_config("MULTIARCH")
     if multiarch:
-        for config_var_name in ["LIBPL", "LIBDIR"]:
-            config_value = get_config(config_var_name)
+        for x in ("LIBPL", "LIBDIR"):
+            config_value = get_config(x)
             if config_value and not config_value.endswith(multiarch):
                 lib_dirs.append(os.path.join(config_value, multiarch))
 
@@ -80,29 +82,28 @@ def _search_library_names(get_config):
     # 'Python.framework/Versions/3.9/Python' on MacOS.
     #
     # A typical LDLIBRARY is 'libpythonX.Y.so' on Linux, or 'pythonXY.dll' on
-    # Windows.
+    # Windows, or 'Python.framework/Versions/3.9/Python' on MacOS.
     #
     # A typical LIBRARY is 'libpythonX.Y.a' on Linux.
     lib_names = [
         get_config(x)
         for x in (
-            "INSTSONAME",
             "LDLIBRARY",
+            "INSTSONAME",
             "PY3LIBRARY",
             "LIBRARY",
             "DLLLIBRARY",
         )
     ]
 
     # Set the prefix and suffix to construct the library name used for linking.
+    # The suffix and version are set here to the default values for the OS,
+    # since they are used below to construct "default" library names.
     if _IS_DARWIN:
-        # SHLIB_SUFFIX may be ".so"; always override on darwin to be ".dynlib"
         suffix = ".dylib"
         prefix = "lib"
     elif _IS_WINDOWS:
-        # SHLIB_SUFFIX on windows is ".dll"; however the compiler needs to
-        # link with the ".lib".
-        suffix = ".lib"
+        suffix = ".dll"
         prefix = ""
     else:
         suffix = get_config("SHLIB_SUFFIX")
@@ -111,9 +112,8 @@ def _search_library_names(get_config):
             suffix = ".so"
 
     version = get_config("VERSION")
-    # On Windows, extensions should link with the pythonXY.lib files.
-    # See: https://docs.python.org/3/extending/windows.html
-    # So ensure that the pythonXY.lib files are included in the search.
+
+    # Ensure that the pythonXY.dll files are included in the search.
     lib_names.append(f"{prefix}python{version}{suffix}")
 
     # If there are ABIFLAGS, also add them to the python version lib search.
@@ -130,7 +130,8 @@ def _get_python_library_info():
     """Returns a dictionary with the static and dynamic python libraries."""
     config_vars = sysconfig.get_config_vars()
 
-    # VERSION is X.Y in Linux/macOS and XY in Windows.
+    # VERSION is X.Y in Linux/macOS and XY in Windows.  This is used to
+    # construct library paths such as python3.12, so ensure it exists.
     if not config_vars.get("VERSION"):
         if sys.platform == "win32":
             config_vars["VERSION"] = f"{sys.version_info.major}{sys.version_info.minor}"
@@ -142,25 +143,47 @@ def _get_python_library_info():
     search_directories = _search_directories(config_vars.get)
     search_libnames = _search_library_names(config_vars.get)
 
+    def _add_if_exists(target, path):
+        if os.path.exists(path) or os.path.isdir(path):
+            target[path] = None
+
+    interface_libraries = {}
     dynamic_libraries = {}
     static_libraries = {}
     for root_dir in search_directories:
         for libname in search_libnames:
             composed_path = os.path.join(root_dir, libname)
-            if not os.path.exists(composed_path) or os.path.isdir(composed_path):
+            if libname.endswith(".a"):
+                _add_if_exists(static_libraries, composed_path)
                 continue
-            if composed_path.endswith(".lib") or composed_path.endswith(".a"):
-                static_libraries[composed_path] = None
-            else:
-                dynamic_libraries[composed_path] = None
+
+            _add_if_exists(dynamic_libraries, composed_path)
+            if libname.endswith(".dll"):
+                # On windows a .lib file may be an "import library" or a static library.
+                # The file could be inspected to determine which it is; typically python
+                # is used as a shared library.
+                #
+                # On Windows, extensions should link with the pythonXY.lib interface
+                # libraries.
+                #
+                # See: https://docs.python.org/3/extending/windows.html
+                # https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-creation
+                _add_if_exists(
+                    interface_libraries, os.path.join(root_dir, libname[:-3] + "lib")
+                )
+            elif libname.endswith(".so"):
+                # It's possible, though unlikely, that interface stubs (.ifso) exist.
+                _add_if_exists(
+                    interface_libraries, os.path.join(root_dir, libname[:-2] + "ifso")
+                )
 
     # When no libraries are found it's likely that the python interpreter is not
     # configured to use shared or static libraries (minilinux).  If this seems
-    # suspicious try running `uv tool run find_libpython --list-all -v` or
-    # `python-config --libs`.
+    # suspicious try running `uv tool run find_libpython --list-all -v`
     return {
         "dynamic_libraries": list(dynamic_libraries.keys()),
         "static_libraries": list(static_libraries.keys()),
+        "interface_libraries": list(interface_libraries.keys()),
     }
 
 

python/private/local_runtime_repo.bzl

@@ -35,7 +35,8 @@ define_local_runtime_toolchain_impl(
     minor = "{minor}",
     micro = "{micro}",
     interpreter_path = "{interpreter_path}",
-    interface_library = "{interface_library}",
+    interface_library = {interface_library},
+    shared_library = {shared_library},
     implementation_name = "{implementation_name}",
     os = "{os}",
 )
@@ -48,36 +49,30 @@ def _norm_path(path):
         path = path[:-1]
     return path
 
-def _symlink_libs(rctx, logger, library_targets):
+def _symlink_first_library(rctx, logger, libraries):
     """Symlinks the shared libraries into the lib/ directory.
 
     Args:
         rctx: A repository_ctx object
         logger: A repo_utils.logger object
-        library_targets: A list of library targets to potentially symlink.
+        libraries: A list of static library paths to potentially symlink.
     Returns:
-        A library target suitable for a cc_import rule.
-
-    The specific files are symlinked instead of the whole directory because
-    shared_lib_dirs contains multiple search paths for the shared libraries,
-    and the python files may be missing from any of those directories, and
-    any of those directories may include non-python runtime libraries,
-    as would be the case if LIBDIR were, for example, /usr/lib.
+        A single library path linked by the action.
     """
-    found = ""
-    for target in library_targets:
+    linked = None
+    for target in libraries:
         origin = rctx.path(target)
         if not origin.exists:
             # The reported names don't always exist; it depends on the particulars
             # of the runtime installation.
             continue
-        found = "lib/{}".format(origin.basename)
-        logger.debug("Symlinking {} to {}".format(origin, found))
+        linked = "lib/{}".format(origin.basename)
+        logger.debug("Symlinking {} to {}".format(origin, linked))
         repo_utils.watch(rctx, origin)
-        rctx.symlink(origin, found)
+        rctx.symlink(origin, linked)
         break
 
-    return found
+    return linked
 
 def _local_runtime_repo_impl(rctx):
     logger = repo_utils.logger(rctx)
@@ -153,20 +148,20 @@ def _local_runtime_repo_impl(rctx):
     # appear as part of this repo.
     rctx.symlink(info["include"], "include")
 
-    if repo_utils.get_platforms_os_name == "windows":
-        library_targets = info["static_libraries"]
-    else:
-        library_targets = info["dynamic_libraries"]
-
     rctx.report_progress("Symlinking external Python shared libraries")
-    interface_library = _symlink_libs(rctx, logger, library_targets)
+    interface_library = _symlink_first_library(rctx, logger, info["interface_libraries"])
+    shared_library = _symlink_first_library(rctx, logger, info["dynamic_libraries"])
+
+    if not interface_library and not shared_library:
+        logger.warn("No external python libraries found.")
 
     build_bazel = _TOOLCHAIN_IMPL_TEMPLATE.format(
         major = info["major"],
         minor = info["minor"],
         micro = info["micro"],
         interpreter_path = _norm_path(interpreter_path),
-        interface_library = interface_library,
+        interface_library = repr(interface_library),
+        shared_library = repr(shared_library),
         implementation_name = info["implementation_name"],
         os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)),
     )
@@ -235,7 +230,8 @@ def _expand_incompatible_template():
     return _TOOLCHAIN_IMPL_TEMPLATE.format(
         interpreter_path = "/incompatible",
         implementation_name = "incompatible",
-        interface_library = "",
+        interface_library = "None",
+        shared_library = "None",
         major = "0",
         minor = "0",
         micro = "0",

python/private/local_runtime_repo_setup.bzl

@@ -31,6 +31,7 @@ def define_local_runtime_toolchain_impl(
         micro,
         interpreter_path,
         interface_library,
+        shared_library,
         implementation_name,
         os):
     """Defines a toolchain implementation for a local Python runtime.
@@ -50,8 +51,10 @@ def define_local_runtime_toolchain_impl(
         minor: `str` The minor Python version, e.g. `9` of `3.9.1`.
         micro: `str` The micro Python version, e.g. "1" of `3.9.1`.
         interpreter_path: `str` Absolute path to the interpreter.
-        interface_library: `str` A path to a .lib or .so file to link against.
+        interface_library: `str` Path to the interface library.
             e.g. "lib/python312.lib"
+        shared_library: `str` Path to the dynamic library.
+            e.g. "lib/python312.dll" or "lib/python312.so"
         implementation_name: `str` The implementation name, as returned by
             `sys.implementation.name`.
         os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for
@@ -64,14 +67,14 @@ def define_local_runtime_toolchain_impl(
     # See https://docs.python.org/3/extending/windows.html
     # However not all python installations (such as manylinux) include shared or static libraries,
     # so only create the import library when interface_library is set.
-    _libpython_deps = []
+    import_deps = []
     if interface_library:
         cc_import(
-            name = "_python_import_lib",
+            name = "_python_interface_library",
             interface_library = interface_library,
             system_provided = 1,
         )
-        _libpython_deps.append(":_python_import_lib")
+        import_deps = [":_python_interface_library"]
 
     cc_library(
         name = "_python_headers",
@@ -81,13 +84,15 @@ def define_local_runtime_toolchain_impl(
             # A Python install may not have C headers
             allow_empty = True,
         ),
+        deps = import_deps,
         includes = ["include"],
     )
 
     cc_library(
         name = "_libpython",
         hdrs = [":_python_headers"],
-        deps = _libpython_deps,
+        srcs = [shared_library] if shared_library else [],
+        deps = import_deps,
     )
 
     py_runtime(