fix(local_runtime): Improve local_runtime usability in macos / windows (#3148)

local_runtime fails to handle many variations of python install on
Windows and
MacOS, such as:

* LDLIBRARY on MacOS may refer to a file under PYTHONFRAMEWORKPREFIX,
not LIBDIR
* LDLIBRARY on Windows refers to pythonXY.dll, not the linkable
pythonXY.lib
* LIBDIR may not be correctly set on Windows.
* On windows, interpreter_path needs to be normalized. Other paths also
require this.
* SHLIB_SUFFIX does not indicate the correct suffix.

For examples, see: https://docs.python.org/3/extending/windows.html

In order to resolve this the shared library resolution has been moved
into get_local_runtime_info.py, which now does the following:

* Constructs a list of paths to search based on LIBDIR, LIBPL,
PYTHONFRAMEWORKPREFIX,
  and the executable directory.
* Constructs a list of libraries to search based on INSTSONAME,
LDLIBRARY,
  pythonXY.lib, etc.
* Checks to see which files exist, partitioning the result into a list
of
  "dynamic_libraries" and "static_libraries" 

On Windows and macOS, since SHLIB_SUFFIX does not always indicate the
filenames
needed searching, this has been removed from local_runtime_repo_setup
and
replaced with an explicit file.

On Windows the interpreter_path and other search paths are now
normalized
(`\` converted to `/`).

Additional logging added to local_runtime_repo.

Fixes #3055
Work towards #824

---------

Co-authored-by: Richard Levasseur <[email protected]>

Author: laramiel

.bazelci/presubmit.yml

@@ -57,6 +57,7 @@ buildifier:
     - "--enable_workspace"
     - "--build_tag_filters=-integration-test"
   bazel: 7.x
+# NOTE: The Mac and Windows bazelinbazel jobs override parts of this config.
 .common_bazelinbazel_config: &common_bazelinbazel_config
     build_flags:
       - "--build_tag_filters=integration-test"
@@ -503,6 +504,24 @@ tasks:
     <<: *common_bazelinbazel_config
     name: "tests/integration bazel-in-bazel: Debian"
     platform: debian11
+  # The bazelinbazel tests were disabled on Mac to save CI jobs slots, and
+  # have bitrotted a bit. For now, just run a subset of what we're most
+  # interested in.
+  integration_test_bazelinbazel_macos:
+    <<: *common_bazelinbazel_config
+    name: "tests/integration bazel-in-bazel: macOS (subset)"
+    platform: macos
+    build_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
+    test_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
+  # The bazelinbazel tests were disabled on Windows to save CI jobs slots, and
+  # have bitrotted a bit. For now, just run a subset of what we're most
+  # interested in.
+  integration_test_bazelinbazel_windows:
+    <<: *common_bazelinbazel_config
+    name: "tests/integration bazel-in-bazel: Windows (subset)"
+    platform: windows
+    build_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
+    test_targets: ["//tests/integration:local_toolchains_test_bazel_self"]
 
   integration_test_compile_pip_requirements_ubuntu:
     <<: *reusable_build_test_all

.gitignore

@@ -37,6 +37,8 @@
 /bazel-genfiles
 /bazel-out
 /bazel-testlogs
+**/bazel-*
+
 user.bazelrc
 
 # vim swap files

CHANGELOG.md

@@ -106,6 +106,12 @@ END_UNRELEASED_TEMPLATE
 * (toolchains) use "command -v" to find interpreter in `$PATH`
   ([#3150](https://github.com/bazel-contrib/rules_python/pull/3150)).
 * (pypi) `bazel vendor` now works in `bzlmod` ({gh-issue}`3079`).
+* (toolchains) `local_runtime_repo` now works on Windows
+  ([#3055](https://github.com/bazel-contrib/rules_python/issues/3055)).
+* (toolchains) `local_runtime_repo` supports more types of Python
+  installations (Mac frameworks, missing dynamic libraries, and other
+  esoteric cases, see
+  [#3148](https://github.com/bazel-contrib/rules_python/pull/3148) for details).
 
 {#v0-0-0-added}
 ### Added

python/private/get_local_runtime_info.py

@@ -12,47 +12,188 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Returns information about the local Python runtime as JSON."""
+
 import json
+import os
 import sys
 import sysconfig
 
-data = {
-    "major": sys.version_info.major,
-    "minor": sys.version_info.minor,
-    "micro": sys.version_info.micro,
-    "include": sysconfig.get_path("include"),
-    "implementation_name": sys.implementation.name,
-    "base_executable": sys._base_executable,
-}
+_IS_WINDOWS = sys.platform == "win32"
+_IS_DARWIN = sys.platform == "darwin"
+
 
-config_vars = [
-    # The libpythonX.Y.so file. Usually?
-    # It might be a static archive (.a) file instead.
-    "LDLIBRARY",
-    # The directory with library files. Supposedly.
-    # It's not entirely clear how to get the directory with libraries.
+def _search_directories(get_config):
+    """Returns a list of library directories to search for shared libraries."""
     # There's several types of libraries with different names and a plethora
-    # of settings.
+    # of settings, and many different config variables to check:
+    #
+    # LIBPL is used in python-config when shared library is not enabled:
+    # https://github.com/python/cpython/blob/v3.12.0/Misc/python-config.in#L63
+    #
+    # LIBDIR may also be the python directory with library files.
     # https://stackoverflow.com/questions/47423246/get-pythons-lib-path
-    # For now, it seems LIBDIR has what is needed, so just use that.
     # See also: MULTIARCH
-    "LIBDIR",
+    #
+    # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks,
+    # such as "Python.framework/Versions/3.12/Python", not a file under the
+    # LIBDIR/LIBPL directory, so include PYTHONFRAMEWORKPREFIX.
+    lib_dirs = [get_config(x) for x in ("PYTHONFRAMEWORKPREFIX", "LIBPL", "LIBDIR")]
+
     # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't
     # tell the location of the libs, just the base directory. The `MULTIARCH`
     # sysconfig variable tells the subdirectory within it with the libs.
     # See:
     # https://wiki.debian.org/Python/MultiArch
     # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842
-    "MULTIARCH",
-    # The versioned libpythonX.Y.so.N file. Usually?
-    # It might be a static archive (.a) file instead.
-    "INSTSONAME",
-    # The libpythonX.so file. Usually?
-    # It might be a static archive (a.) file instead.
-    "PY3LIBRARY",
-    # The platform-specific filename suffix for library files.
-    # Includes the dot, e.g. `.so`
-    "SHLIB_SUFFIX",
-]
-data.update(zip(config_vars, sysconfig.get_config_vars(*config_vars)))
+    multiarch = get_config("MULTIARCH")
+    if multiarch:
+        for x in ("LIBPL", "LIBDIR"):
+            config_value = get_config(x)
+            if config_value and not config_value.endswith(multiarch):
+                lib_dirs.append(os.path.join(config_value, multiarch))
+
+    if _IS_WINDOWS:
+        # On Windows DLLs go in the same directory as the executable, while .lib
+        # files live in the lib/ or libs/ subdirectory.
+        lib_dirs.append(get_config("BINDIR"))
+        lib_dirs.append(os.path.join(os.path.dirname(sys.executable)))
+        lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "lib"))
+        lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "libs"))
+    elif not _IS_DARWIN:
+        # On most systems the executable is in a bin/ directory and the libraries
+        # are in a sibling lib/ directory.
+        lib_dirs.append(
+            os.path.join(os.path.dirname(os.path.dirname(sys.executable)), "lib")
+        )
+
+    # Dedup and remove empty values, keeping the order.
+    lib_dirs = [v for v in lib_dirs if v]
+    return {k: None for k in lib_dirs}.keys()
+
+
+def _search_library_names(get_config):
+    """Returns a list of library files to search for shared libraries."""
+    # Quoting configure.ac in the cpython code base:
+    # "INSTSONAME is the name of the shared library that will be use to install
+    # on the system - some systems like version suffix, others don't.""
+    #
+    # A typical INSTSONAME is 'libpython3.8.so.1.0' on Linux, or
+    # 'Python.framework/Versions/3.9/Python' on MacOS.
+    #
+    # A typical LDLIBRARY is 'libpythonX.Y.so' on Linux, or 'pythonXY.dll' on
+    # Windows, or 'Python.framework/Versions/3.9/Python' on MacOS.
+    #
+    # A typical LIBRARY is 'libpythonX.Y.a' on Linux.
+    lib_names = [
+        get_config(x)
+        for x in (
+            "LDLIBRARY",
+            "INSTSONAME",
+            "PY3LIBRARY",
+            "LIBRARY",
+            "DLLLIBRARY",
+        )
+    ]
+
+    # Set the prefix and suffix to construct the library name used for linking.
+    # The suffix and version are set here to the default values for the OS,
+    # since they are used below to construct "default" library names.
+    if _IS_DARWIN:
+        suffix = ".dylib"
+        prefix = "lib"
+    elif _IS_WINDOWS:
+        suffix = ".dll"
+        prefix = ""
+    else:
+        suffix = get_config("SHLIB_SUFFIX")
+        prefix = "lib"
+        if not suffix:
+            suffix = ".so"
+
+    version = get_config("VERSION")
+
+    # Ensure that the pythonXY.dll files are included in the search.
+    lib_names.append(f"{prefix}python{version}{suffix}")
+
+    # If there are ABIFLAGS, also add them to the python version lib search.
+    abiflags = get_config("ABIFLAGS") or get_config("abiflags") or ""
+    if abiflags:
+        lib_names.append(f"{prefix}python{version}{abiflags}{suffix}")
+
+    # Dedup and remove empty values, keeping the order.
+    lib_names = [v for v in lib_names if v]
+    return {k: None for k in lib_names}.keys()
+
+
+def _get_python_library_info():
+    """Returns a dictionary with the static and dynamic python libraries."""
+    config_vars = sysconfig.get_config_vars()
+
+    # VERSION is X.Y in Linux/macOS and XY in Windows.  This is used to
+    # construct library paths such as python3.12, so ensure it exists.
+    if not config_vars.get("VERSION"):
+        if sys.platform == "win32":
+            config_vars["VERSION"] = f"{sys.version_info.major}{sys.version_info.minor}"
+        else:
+            config_vars["VERSION"] = (
+                f"{sys.version_info.major}.{sys.version_info.minor}"
+            )
+
+    search_directories = _search_directories(config_vars.get)
+    search_libnames = _search_library_names(config_vars.get)
+
+    def _add_if_exists(target, path):
+        if os.path.exists(path) or os.path.isdir(path):
+            target[path] = None
+
+    interface_libraries = {}
+    dynamic_libraries = {}
+    static_libraries = {}
+    for root_dir in search_directories:
+        for libname in search_libnames:
+            composed_path = os.path.join(root_dir, libname)
+            if libname.endswith(".a"):
+                _add_if_exists(static_libraries, composed_path)
+                continue
+
+            _add_if_exists(dynamic_libraries, composed_path)
+            if libname.endswith(".dll"):
+                # On windows a .lib file may be an "import library" or a static library.
+                # The file could be inspected to determine which it is; typically python
+                # is used as a shared library.
+                #
+                # On Windows, extensions should link with the pythonXY.lib interface
+                # libraries.
+                #
+                # See: https://docs.python.org/3/extending/windows.html
+                # https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-creation
+                _add_if_exists(
+                    interface_libraries, os.path.join(root_dir, libname[:-3] + "lib")
+                )
+            elif libname.endswith(".so"):
+                # It's possible, though unlikely, that interface stubs (.ifso) exist.
+                _add_if_exists(
+                    interface_libraries, os.path.join(root_dir, libname[:-2] + "ifso")
+                )
+
+    # When no libraries are found it's likely that the python interpreter is not
+    # configured to use shared or static libraries (minilinux).  If this seems
+    # suspicious try running `uv tool run find_libpython --list-all -v`
+    return {
+        "dynamic_libraries": list(dynamic_libraries.keys()),
+        "static_libraries": list(static_libraries.keys()),
+        "interface_libraries": list(interface_libraries.keys()),
+    }
+
+
+data = {
+    "major": sys.version_info.major,
+    "minor": sys.version_info.minor,
+    "micro": sys.version_info.micro,
+    "include": sysconfig.get_path("include"),
+    "implementation_name": sys.implementation.name,
+    "base_executable": sys._base_executable,
+}
+data.update(_get_python_library_info())
 print(json.dumps(data))