diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 6457363ccd..5889823d3d 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -57,6 +57,7 @@ buildifier: - "--enable_workspace" - "--build_tag_filters=-integration-test" bazel: 7.x +# NOTE: The Mac and Windows bazelinbazel jobs override parts of this config. .common_bazelinbazel_config: &common_bazelinbazel_config build_flags: - "--build_tag_filters=integration-test" @@ -503,6 +504,24 @@ tasks: <<: *common_bazelinbazel_config name: "tests/integration bazel-in-bazel: Debian" platform: debian11 + # The bazelinbazel tests were disabled on Mac to save CI jobs slots, and + # have bitrotted a bit. For now, just run a subset of what we're most + # interested in. + integration_test_bazelinbazel_macos: + <<: *common_bazelinbazel_config + name: "tests/integration bazel-in-bazel: macOS (subset)" + platform: macos + build_targets: ["//tests/integration:local_toolchains_test_bazel_self"] + test_targets: ["//tests/integration:local_toolchains_test_bazel_self"] + # The bazelinbazel tests were disabled on Windows to save CI jobs slots, and + # have bitrotted a bit. For now, just run a subset of what we're most + # interested in. + integration_test_bazelinbazel_windows: + <<: *common_bazelinbazel_config + name: "tests/integration bazel-in-bazel: Windows (subset)" + platform: windows + build_targets: ["//tests/integration:local_toolchains_test_bazel_self"] + test_targets: ["//tests/integration:local_toolchains_test_bazel_self"] integration_test_compile_pip_requirements_ubuntu: <<: *reusable_build_test_all diff --git a/.gitignore b/.gitignore index 863b0e9c3f..fb1b17e466 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,8 @@ /bazel-genfiles /bazel-out /bazel-testlogs +**/bazel-* + user.bazelrc # vim swap files diff --git a/CHANGELOG.md b/CHANGELOG.md index 55c4659b39..3707d33c04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,12 @@ END_UNRELEASED_TEMPLATE * (toolchains) use "command -v" to find interpreter in `$PATH` ([#3150](https://github.com/bazel-contrib/rules_python/pull/3150)). * (pypi) `bazel vendor` now works in `bzlmod` ({gh-issue}`3079`). +* (toolchains) `local_runtime_repo` now works on Windows + ([#3055](https://github.com/bazel-contrib/rules_python/issues/3055)). +* (toolchains) `local_runtime_repo` supports more types of Python + installations (Mac frameworks, missing dynamic libraries, and other + esoteric cases, see + [#3148](https://github.com/bazel-contrib/rules_python/pull/3148) for details). {#v0-0-0-added} ### Added diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index c8371357c2..ff3b0aeb01 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -12,47 +12,188 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Returns information about the local Python runtime as JSON.""" + import json +import os import sys import sysconfig -data = { - "major": sys.version_info.major, - "minor": sys.version_info.minor, - "micro": sys.version_info.micro, - "include": sysconfig.get_path("include"), - "implementation_name": sys.implementation.name, - "base_executable": sys._base_executable, -} +_IS_WINDOWS = sys.platform == "win32" +_IS_DARWIN = sys.platform == "darwin" + -config_vars = [ - # The libpythonX.Y.so file. Usually? - # It might be a static archive (.a) file instead. - "LDLIBRARY", - # The directory with library files. Supposedly. - # It's not entirely clear how to get the directory with libraries. +def _search_directories(get_config): + """Returns a list of library directories to search for shared libraries.""" # There's several types of libraries with different names and a plethora - # of settings. + # of settings, and many different config variables to check: + # + # LIBPL is used in python-config when shared library is not enabled: + # https://github.com/python/cpython/blob/v3.12.0/Misc/python-config.in#L63 + # + # LIBDIR may also be the python directory with library files. # https://stackoverflow.com/questions/47423246/get-pythons-lib-path - # For now, it seems LIBDIR has what is needed, so just use that. # See also: MULTIARCH - "LIBDIR", + # + # On MacOS, the LDLIBRARY may be a relative path under /Library/Frameworks, + # such as "Python.framework/Versions/3.12/Python", not a file under the + # LIBDIR/LIBPL directory, so include PYTHONFRAMEWORKPREFIX. + lib_dirs = [get_config(x) for x in ("PYTHONFRAMEWORKPREFIX", "LIBPL", "LIBDIR")] + # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't # tell the location of the libs, just the base directory. The `MULTIARCH` # sysconfig variable tells the subdirectory within it with the libs. # See: # https://wiki.debian.org/Python/MultiArch # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842 - "MULTIARCH", - # The versioned libpythonX.Y.so.N file. Usually? - # It might be a static archive (.a) file instead. - "INSTSONAME", - # The libpythonX.so file. Usually? - # It might be a static archive (a.) file instead. - "PY3LIBRARY", - # The platform-specific filename suffix for library files. - # Includes the dot, e.g. `.so` - "SHLIB_SUFFIX", -] -data.update(zip(config_vars, sysconfig.get_config_vars(*config_vars))) + multiarch = get_config("MULTIARCH") + if multiarch: + for x in ("LIBPL", "LIBDIR"): + config_value = get_config(x) + if config_value and not config_value.endswith(multiarch): + lib_dirs.append(os.path.join(config_value, multiarch)) + + if _IS_WINDOWS: + # On Windows DLLs go in the same directory as the executable, while .lib + # files live in the lib/ or libs/ subdirectory. + lib_dirs.append(get_config("BINDIR")) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "lib")) + lib_dirs.append(os.path.join(os.path.dirname(sys.executable), "libs")) + elif not _IS_DARWIN: + # On most systems the executable is in a bin/ directory and the libraries + # are in a sibling lib/ directory. + lib_dirs.append( + os.path.join(os.path.dirname(os.path.dirname(sys.executable)), "lib") + ) + + # Dedup and remove empty values, keeping the order. + lib_dirs = [v for v in lib_dirs if v] + return {k: None for k in lib_dirs}.keys() + + +def _search_library_names(get_config): + """Returns a list of library files to search for shared libraries.""" + # Quoting configure.ac in the cpython code base: + # "INSTSONAME is the name of the shared library that will be use to install + # on the system - some systems like version suffix, others don't."" + # + # A typical INSTSONAME is 'libpython3.8.so.1.0' on Linux, or + # 'Python.framework/Versions/3.9/Python' on MacOS. + # + # A typical LDLIBRARY is 'libpythonX.Y.so' on Linux, or 'pythonXY.dll' on + # Windows, or 'Python.framework/Versions/3.9/Python' on MacOS. + # + # A typical LIBRARY is 'libpythonX.Y.a' on Linux. + lib_names = [ + get_config(x) + for x in ( + "LDLIBRARY", + "INSTSONAME", + "PY3LIBRARY", + "LIBRARY", + "DLLLIBRARY", + ) + ] + + # Set the prefix and suffix to construct the library name used for linking. + # The suffix and version are set here to the default values for the OS, + # since they are used below to construct "default" library names. + if _IS_DARWIN: + suffix = ".dylib" + prefix = "lib" + elif _IS_WINDOWS: + suffix = ".dll" + prefix = "" + else: + suffix = get_config("SHLIB_SUFFIX") + prefix = "lib" + if not suffix: + suffix = ".so" + + version = get_config("VERSION") + + # Ensure that the pythonXY.dll files are included in the search. + lib_names.append(f"{prefix}python{version}{suffix}") + + # If there are ABIFLAGS, also add them to the python version lib search. + abiflags = get_config("ABIFLAGS") or get_config("abiflags") or "" + if abiflags: + lib_names.append(f"{prefix}python{version}{abiflags}{suffix}") + + # Dedup and remove empty values, keeping the order. + lib_names = [v for v in lib_names if v] + return {k: None for k in lib_names}.keys() + + +def _get_python_library_info(): + """Returns a dictionary with the static and dynamic python libraries.""" + config_vars = sysconfig.get_config_vars() + + # VERSION is X.Y in Linux/macOS and XY in Windows. This is used to + # construct library paths such as python3.12, so ensure it exists. + if not config_vars.get("VERSION"): + if sys.platform == "win32": + config_vars["VERSION"] = f"{sys.version_info.major}{sys.version_info.minor}" + else: + config_vars["VERSION"] = ( + f"{sys.version_info.major}.{sys.version_info.minor}" + ) + + search_directories = _search_directories(config_vars.get) + search_libnames = _search_library_names(config_vars.get) + + def _add_if_exists(target, path): + if os.path.exists(path) or os.path.isdir(path): + target[path] = None + + interface_libraries = {} + dynamic_libraries = {} + static_libraries = {} + for root_dir in search_directories: + for libname in search_libnames: + composed_path = os.path.join(root_dir, libname) + if libname.endswith(".a"): + _add_if_exists(static_libraries, composed_path) + continue + + _add_if_exists(dynamic_libraries, composed_path) + if libname.endswith(".dll"): + # On windows a .lib file may be an "import library" or a static library. + # The file could be inspected to determine which it is; typically python + # is used as a shared library. + # + # On Windows, extensions should link with the pythonXY.lib interface + # libraries. + # + # See: https://docs.python.org/3/extending/windows.html + # https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-creation + _add_if_exists( + interface_libraries, os.path.join(root_dir, libname[:-3] + "lib") + ) + elif libname.endswith(".so"): + # It's possible, though unlikely, that interface stubs (.ifso) exist. + _add_if_exists( + interface_libraries, os.path.join(root_dir, libname[:-2] + "ifso") + ) + + # When no libraries are found it's likely that the python interpreter is not + # configured to use shared or static libraries (minilinux). If this seems + # suspicious try running `uv tool run find_libpython --list-all -v` + return { + "dynamic_libraries": list(dynamic_libraries.keys()), + "static_libraries": list(static_libraries.keys()), + "interface_libraries": list(interface_libraries.keys()), + } + + +data = { + "major": sys.version_info.major, + "minor": sys.version_info.minor, + "micro": sys.version_info.micro, + "include": sysconfig.get_path("include"), + "implementation_name": sys.implementation.name, + "base_executable": sys._base_executable, +} +data.update(_get_python_library_info()) print(json.dumps(data)) diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index b8b7164b54..6427ed022d 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -31,27 +31,67 @@ load("@rules_python//python/private:local_runtime_repo_setup.bzl", "define_local define_local_runtime_toolchain_impl( name = "local_runtime", - lib_ext = "{lib_ext}", major = "{major}", minor = "{minor}", micro = "{micro}", interpreter_path = "{interpreter_path}", + interface_library = {interface_library}, + libraries = {libraries}, implementation_name = "{implementation_name}", os = "{os}", ) """ +def _norm_path(path): + """Returns a path using '/' separators and no trailing slash.""" + path = path.replace("\\", "/") + if path[-1] == "/": + path = path[:-1] + return path + +def _symlink_first_library(rctx, logger, libraries): + """Symlinks the shared libraries into the lib/ directory. + + Args: + rctx: A repository_ctx object + logger: A repo_utils.logger object + libraries: A list of static library paths to potentially symlink. + Returns: + A single library path linked by the action. + """ + linked = None + for target in libraries: + origin = rctx.path(target) + if not origin.exists: + # The reported names don't always exist; it depends on the particulars + # of the runtime installation. + continue + if target.endswith("/Python"): + linked = "lib/{}.dylib".format(origin.basename) + else: + linked = "lib/{}".format(origin.basename) + logger.debug("Symlinking {} to {}".format(origin, linked)) + repo_utils.watch(rctx, origin) + rctx.symlink(origin, linked) + break + + return linked + def _local_runtime_repo_impl(rctx): logger = repo_utils.logger(rctx) on_failure = rctx.attr.on_failure - result = _resolve_interpreter_path(rctx) - if not result.resolved_path: + def _emit_log(msg): if on_failure == "fail": - fail("interpreter not found: {}".format(result.describe_failure())) + logger.fail(msg) + elif on_failure == "warn": + logger.warn(msg) + else: + logger.debug(msg) - if on_failure == "warn": - logger.warn(lambda: "interpreter not found: {}".format(result.describe_failure())) + result = _resolve_interpreter_path(rctx) + if not result.resolved_path: + _emit_log(lambda: "interpreter not found: {}".format(result.describe_failure())) # else, on_failure must be skip rctx.file("BUILD.bazel", _expand_incompatible_template()) @@ -72,10 +112,7 @@ def _local_runtime_repo_impl(rctx): logger = logger, ) if exec_result.return_code != 0: - if on_failure == "fail": - fail("GetPythonInfo failed: {}".format(exec_result.describe_failure())) - if on_failure == "warn": - logger.warn(lambda: "GetPythonInfo failed: {}".format(exec_result.describe_failure())) + _emit_log(lambda: "GetPythonInfo failed: {}".format(exec_result.describe_failure())) # else, on_failure must be skip rctx.file("BUILD.bazel", _expand_incompatible_template()) @@ -114,51 +151,35 @@ def _local_runtime_repo_impl(rctx): # appear as part of this repo. rctx.symlink(info["include"], "include") - shared_lib_names = [ - info["PY3LIBRARY"], - info["LDLIBRARY"], - info["INSTSONAME"], - ] - - # In some cases, the value may be empty. Not clear why. - shared_lib_names = [v for v in shared_lib_names if v] - - # In some cases, the same value is returned for multiple keys. Not clear why. - shared_lib_names = {v: None for v in shared_lib_names}.keys() - shared_lib_dir = info["LIBDIR"] - multiarch = info["MULTIARCH"] - - # The specific files are symlinked instead of the whole directory - # because it can point to a directory that has more than just - # the Python runtime shared libraries, e.g. /usr/lib, or a Python - # specific directory with pip-installed shared libraries. rctx.report_progress("Symlinking external Python shared libraries") - for name in shared_lib_names: - origin = rctx.path("{}/{}".format(shared_lib_dir, name)) - - # If the origin doesn't exist, try the multiarch location, in case - # it's an older Python / Debian release. - if not origin.exists and multiarch: - origin = rctx.path("{}/{}/{}".format(shared_lib_dir, multiarch, name)) - - # The reported names don't always exist; it depends on the particulars - # of the runtime installation. - if origin.exists: - repo_utils.watch(rctx, origin) - rctx.symlink(origin, "lib/" + name) + interface_library = _symlink_first_library(rctx, logger, info["interface_libraries"]) + shared_library = _symlink_first_library(rctx, logger, info["dynamic_libraries"]) + static_library = _symlink_first_library(rctx, logger, info["static_libraries"]) + + libraries = [] + if shared_library: + libraries.append(shared_library) + elif static_library: + libraries.append(static_library) + else: + logger.warn("No external python libraries found.") - rctx.file("WORKSPACE", "") - rctx.file("MODULE.bazel", "") - rctx.file("REPO.bazel", "") - rctx.file("BUILD.bazel", _TOOLCHAIN_IMPL_TEMPLATE.format( + build_bazel = _TOOLCHAIN_IMPL_TEMPLATE.format( major = info["major"], minor = info["minor"], micro = info["micro"], - interpreter_path = interpreter_path, - lib_ext = info["SHLIB_SUFFIX"], + interpreter_path = _norm_path(interpreter_path), + interface_library = repr(interface_library), + libraries = repr(libraries), implementation_name = info["implementation_name"], os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)), - )) + ) + logger.debug("BUILD.bazel\n{}".format(build_bazel)) + + rctx.file("WORKSPACE", "") + rctx.file("MODULE.bazel", "") + rctx.file("REPO.bazel", "") + rctx.file("BUILD.bazel", build_bazel) local_runtime_repo = repository_rule( implementation = _local_runtime_repo_impl, @@ -218,7 +239,8 @@ def _expand_incompatible_template(): return _TOOLCHAIN_IMPL_TEMPLATE.format( interpreter_path = "/incompatible", implementation_name = "incompatible", - lib_ext = "incompatible", + interface_library = "None", + libraries = "[]", major = "0", minor = "0", micro = "0", diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index 37eab59575..1890ef0a0f 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -15,6 +15,7 @@ """Setup code called by the code generated by `local_runtime_repo`.""" load("@bazel_skylib//lib:selects.bzl", "selects") +load("@rules_cc//cc:cc_import.bzl", "cc_import") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python:py_runtime.bzl", "py_runtime") load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair") @@ -25,11 +26,12 @@ _PYTHON_VERSION_FLAG = Label("@rules_python//python/config_settings:python_versi def define_local_runtime_toolchain_impl( name, - lib_ext, major, minor, micro, interpreter_path, + interface_library, + libraries, implementation_name, os): """Defines a toolchain implementation for a local Python runtime. @@ -45,11 +47,14 @@ def define_local_runtime_toolchain_impl( Args: name: `str` Only present to satisfy tooling - lib_ext: `str` The file extension for the `libpython` shared libraries major: `str` The major Python version, e.g. `3` of `3.9.1`. minor: `str` The minor Python version, e.g. `9` of `3.9.1`. micro: `str` The micro Python version, e.g. "1" of `3.9.1`. interpreter_path: `str` Absolute path to the interpreter. + interface_library: `str` Path to the interface library. + e.g. "lib/python312.lib" + libraries: `list[str]` Path[s] to the python libraries. + e.g. ["lib/python312.dll"] or ["lib/python312.so"] implementation_name: `str` The implementation name, as returned by `sys.implementation.name`. os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for @@ -58,6 +63,19 @@ def define_local_runtime_toolchain_impl( major_minor = "{}.{}".format(major, minor) major_minor_micro = "{}.{}".format(major_minor, micro) + # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib + # See https://docs.python.org/3/extending/windows.html + # However not all python installations (such as manylinux) include shared or static libraries, + # so only create the import library when interface_library is set. + import_deps = [] + if interface_library: + cc_import( + name = "_python_interface_library", + interface_library = interface_library, + system_provided = 1, + ) + import_deps = [":_python_interface_library"] + cc_library( name = "_python_headers", # NOTE: Keep in sync with watch_tree() called in local_runtime_repo @@ -66,22 +84,15 @@ def define_local_runtime_toolchain_impl( # A Python install may not have C headers allow_empty = True, ), + deps = import_deps, includes = ["include"], ) cc_library( name = "_libpython", - # Don't use a recursive glob because the lib/ directory usually contains - # a subdirectory of the stdlib -- lots of unrelated files - srcs = native.glob( - [ - "lib/*{}".format(lib_ext), # Match libpython*.so - "lib/*{}*".format(lib_ext), # Also match libpython*.so.1.0 - ], - # A Python install may not have shared libraries. - allow_empty = True, - ), hdrs = [":_python_headers"], + srcs = libraries, + deps = [], ) py_runtime( diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index d178e0f01c..df7fe15444 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -95,6 +95,17 @@ rules_python_integration_test( ], ) +rules_python_integration_test( + name = "local_toolchains_workspace_test", + bazel_versions = [ + version + for version in bazel_binaries.versions.all + if not version.startswith("6.") + ], + bzlmod = False, + workspace_path = "local_toolchains", +) + rules_python_integration_test( name = "pip_parse_test", ) diff --git a/tests/integration/local_toolchains/BUILD.bazel b/tests/integration/local_toolchains/BUILD.bazel index 6b731181a6..5bd55d8cbe 100644 --- a/tests/integration/local_toolchains/BUILD.bazel +++ b/tests/integration/local_toolchains/BUILD.bazel @@ -13,7 +13,9 @@ # limitations under the License. load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python:py_test.bzl", "py_test") +load("@rules_python//tests/support:py_extension.bzl", "py_extension") py_test( name = "test", @@ -35,3 +37,30 @@ string_flag( name = "py", build_setting_default = "", ) + +# Build rules to generate a python extension. +cc_library( + name = "echo_ext_cc", + testonly = True, + srcs = ["echo_ext.cc"], + deps = [ + "@rules_python//python/cc:current_py_cc_headers", + ], + alwayslink = True, +) + +py_extension( + name = "echo_ext", + testonly = True, + copts = select({ + "@rules_cc//cc/compiler:msvc-cl": [], + "//conditions:default": ["-fvisibility=hidden"], + }), + deps = [":echo_ext_cc"], +) + +py_test( + name = "echo_test", + srcs = ["echo_test.py"], + deps = [":echo_ext"], +) diff --git a/tests/integration/local_toolchains/MODULE.bazel b/tests/integration/local_toolchains/MODULE.bazel index 6c06909cd7..45afaafbc9 100644 --- a/tests/integration/local_toolchains/MODULE.bazel +++ b/tests/integration/local_toolchains/MODULE.bazel @@ -16,6 +16,7 @@ module(name = "module_under_test") bazel_dep(name = "rules_python", version = "0.0.0") bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "platforms", version = "0.0.11") +bazel_dep(name = "rules_cc", version = "0.0.16") local_path_override( module_name = "rules_python", diff --git a/tests/integration/local_toolchains/WORKSPACE b/tests/integration/local_toolchains/WORKSPACE index e69de29bb2..480cd2794a 100644 --- a/tests/integration/local_toolchains/WORKSPACE +++ b/tests/integration/local_toolchains/WORKSPACE @@ -0,0 +1,31 @@ +workspace( + name = "module_under_test", +) + +local_repository( + name = "rules_python", + path = "../../..", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + +load("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_repo", "local_runtime_toolchains_repo") + +# Step 1: Define the python runtime. +local_runtime_repo( + name = "local_python3", + interpreter_path = "python3", + on_failure = "fail", + # or interpreter_path = "C:\\path\\to\\python.exe" +) + +# Step 2: Create toolchains for the runtimes +local_runtime_toolchains_repo( + name = "local_toolchains", + runtimes = ["local_python3"], +) + +# Step 3: Register the toolchains +register_toolchains("@local_toolchains//:all") diff --git a/tests/integration/local_toolchains/echo_ext.cc b/tests/integration/local_toolchains/echo_ext.cc new file mode 100644 index 0000000000..367d1a13b3 --- /dev/null +++ b/tests/integration/local_toolchains/echo_ext.cc @@ -0,0 +1,21 @@ +#include + +static PyObject *echoArgs(PyObject *self, PyObject *args) { return args; } + +static PyMethodDef echo_methods[] = { + { "echo", echoArgs, METH_VARARGS, "Returns a tuple of the input args" }, + { NULL, NULL, 0, NULL }, +}; + +extern "C" { + +PyMODINIT_FUNC PyInit_echo_ext(void) { + static struct PyModuleDef echo_module_def = { + // Module definition + PyModuleDef_HEAD_INIT, "echo_ext", "'echo_ext' module", -1, echo_methods + }; + + return PyModule_Create(&echo_module_def); +} + +} // extern "C" diff --git a/tests/integration/local_toolchains/echo_test.py b/tests/integration/local_toolchains/echo_test.py new file mode 100644 index 0000000000..4cc31ff759 --- /dev/null +++ b/tests/integration/local_toolchains/echo_test.py @@ -0,0 +1,9 @@ +import unittest + +import echo_ext + + +class ExtensionTest(unittest.TestCase): + + def test_echo_extension(self): + self.assertEqual(echo_ext.echo(42, "str"), tuple(42, "str")) diff --git a/tests/support/py_extension.bzl b/tests/support/py_extension.bzl new file mode 100644 index 0000000000..5d37fd7824 --- /dev/null +++ b/tests/support/py_extension.bzl @@ -0,0 +1,154 @@ +# Copyright 2025 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Macro to build a python C/C++ extension. + +There are variants of py_extension in many other projects, such as: +* https://github.com/protocolbuffers/protobuf/tree/main/python/py_extension.bzl +* https://github.com/google/riegeli/blob/master/python/riegeli/py_extension.bzl +* https://github.com/pybind/pybind11_bazel/blob/master/build_defs.bzl + +The issue for a generic verion is: +* https://github.com/bazel-contrib/rules_python/issues/824 +""" + +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("@rules_python//python:defs.bzl", "py_library") + +def py_extension( + *, + name, + deps = None, + linkopts = None, + imports = None, + visibility = None, + **kwargs): + """Creates a Python module implemented in C++. + + A Python extension has 2 essential parts: + 1. An internal shared object / pyd package for the extension, `name.pyd`/`name.so` + 2. The py_library target for the extension.` + + Python modules can depend on a py_extension. + + Args: + name: `str`. Name for this target. This is typically the module name. + deps: `list`. Required. C++ libraries to link into the module. + linkopts: `list`. Linking options for the shared library. + imports: `list`. Additional imports for the py_library rule. + visibility: `str`. Visibility for target. + **kwargs: Additional options for the cc_library rule. + """ + if not name: + fail("py_extension requires a name") + if not deps: + fail("py_extension requires a non-empty deps attribute") + if "linkshared" in kwargs: + fail("py_extension attribute linkshared not allowed") + + if not linkopts: + linkopts = [] + + testonly = kwargs.get("testonly") + tags = kwargs.pop("tags", []) + + cc_binary_so_name = name + ".so" + cc_binary_dll_name = name + ".dll" + cc_binary_pyd_name = name + ".pyd" + linker_script_name = name + ".lds" + linker_script_name_rule = name + "_lds" + shared_objects_name = name + "__shared_objects" + + # On Unix, restrict symbol visibility. + exported_symbol = "PyInit_" + name + + # Generate linker script used on non-macOS unix platforms. + native.genrule( + name = linker_script_name_rule, + outs = [linker_script_name], + cmd = "\n".join([ + "cat <<'EOF' >$@", + "{", + " global: " + exported_symbol + ";", + " local: *;", + "};", + "EOF", + ]), + ) + + for cc_binary_name in [cc_binary_dll_name, cc_binary_so_name]: + cur_linkopts = linkopts + cur_deps = deps + if cc_binary_name == cc_binary_so_name: + cur_linkopts = linkopts + select({ + "@platforms//os:macos": [ + # Avoid undefined symbol errors for CPython symbols that + # will be resolved at runtime. + "-undefined", + "dynamic_lookup", + # On macOS, the linker does not support version scripts. Use + # the `-exported_symbol` option instead to restrict symbol + # visibility. + "-Wl,-exported_symbol", + # On macOS, the symbol starts with an underscore. + "-Wl,_" + exported_symbol, + ], + # On non-macOS unix, use a version script to restrict symbol + # visibility. + "//conditions:default": [ + "-Wl,--version-script", + "-Wl,$(location :" + linker_script_name + ")", + ], + }) + cur_deps = cur_deps + select({ + "@platforms//os:macos": [], + "//conditions:default": [linker_script_name], + }) + + cc_binary( + name = cc_binary_name, + linkshared = True, + visibility = ["//visibility:private"], + deps = cur_deps, + tags = tags + ["manual"], + linkopts = cur_linkopts, + **kwargs + ) + + copy_file( + name = cc_binary_pyd_name + "__pyd_copy", + src = ":" + cc_binary_dll_name, + out = cc_binary_pyd_name, + visibility = visibility, + tags = ["manual"], + testonly = testonly, + ) + + native.filegroup( + name = shared_objects_name, + data = select({ + "@platforms//os:windows": [":" + cc_binary_pyd_name], + "//conditions:default": [":" + cc_binary_so_name], + }), + testonly = testonly, + ) + py_library( + name = name, + data = [":" + shared_objects_name], + imports = imports, + tags = tags, + testonly = testonly, + visibility = visibility, + )