diff --git a/.bazelrc b/.bazelrc index d7e1771336..24d85c9771 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/docs,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/docs,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg test --test_output=errors diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel index a378775968..c8b6b8b11c 100644 --- a/examples/build_file_generation/BUILD.bazel +++ b/examples/build_file_generation/BUILD.bazel @@ -28,6 +28,7 @@ modules_mapping( "^_|(\\._)+", # This is the default. "(\\.tests)+", # Add a custom one to get rid of the psutil tests. ], + # skip_private_shared_objects = True, wheels = all_whl_requirements, ) diff --git a/gazelle/docs/installation_and_usage.md b/gazelle/docs/installation_and_usage.md index b151ade25e..e3b1494a0a 100644 --- a/gazelle/docs/installation_and_usage.md +++ b/gazelle/docs/installation_and_usage.md @@ -96,6 +96,15 @@ modules_mapping( # for tools like type checkers and IDEs, improving the development experience and # reducing manual overhead in managing separate stub packages. include_stub_packages = True, + + # skip_private_shared_objects: bool (default: False) + # If set to True, this flag skips private shared objects under .libs directories + # when generating the modules mapping. These are non-importable dependency libraries + # (like libopenblas.so) that vary between Linux distributions and break build + # hermiticity. Ensures identical manifests across platforms by excluding libraries + # that cannot be imported in Python code. macOS uses .dylib files which are + # naturally excluded by this Linux-specific .so filtering. + skip_private_shared_objects = True, ) # Gazelle python extension needs a manifest file mapping from diff --git a/gazelle/modules_mapping/def.bzl b/gazelle/modules_mapping/def.bzl index 48a5477b93..85ba536a06 100644 --- a/gazelle/modules_mapping/def.bzl +++ b/gazelle/modules_mapping/def.bzl @@ -38,6 +38,8 @@ def _modules_mapping_impl(ctx): args.set_param_file_format(format = "multiline") if ctx.attr.include_stub_packages: args.add("--include_stub_packages") + if ctx.attr.skip_private_shared_objects: + args.add("--skip_private_shared_objects") args.add("--output_file", modules_mapping) args.add_all("--exclude_patterns", ctx.attr.exclude_patterns) args.add_all("--wheels", all_wheels) @@ -69,6 +71,14 @@ modules_mapping = rule( doc = "The name for the output JSON file.", mandatory = False, ), + "skip_private_shared_objects": attr.bool( + default = False, + doc = "Whether to skip private shared objects under .libs directories when generating mappings. " + + "When True, excludes non-importable dependency libraries (like libopenblas.so) that vary " + + "between Linux platforms and break build hermiticity. These .libs files are not actual " + + "Python modules and cannot be imported. macOS uses .dylib files which are naturally excluded.", + mandatory = False, + ), "wheels": attr.label_list( allow_files = True, doc = "The list of wheels, usually the 'all_whl_requirements' from @//:requirements.bzl", diff --git a/gazelle/modules_mapping/generator.py b/gazelle/modules_mapping/generator.py index ea11f3e236..ebbf09e08d 100644 --- a/gazelle/modules_mapping/generator.py +++ b/gazelle/modules_mapping/generator.py @@ -26,11 +26,19 @@ class Generator: output_file = None excluded_patterns = None - def __init__(self, stderr, output_file, excluded_patterns, include_stub_packages): + def __init__( + self, + stderr, + output_file, + excluded_patterns, + include_stub_packages, + skip_private_shared_objects=False, + ): self.stderr = stderr self.output_file = output_file self.excluded_patterns = [re.compile(pattern) for pattern in excluded_patterns] self.include_stub_packages = include_stub_packages + self.skip_private_shared_objects = skip_private_shared_objects self.mapping = {} # dig_wheel analyses the wheel .whl file determining the modules it provides @@ -74,6 +82,14 @@ def simplify(self): def module_for_path(self, path, whl): ext = pathlib.Path(path).suffix if ext == ".py" or ext == ".so": + # Skip private shared objects under .libs directories on Linux. + # These are non-importable dependency libraries (like libopenblas.so) that vary + # between platforms and make builds non-hermetic. macOS uses .dylib files + # which are naturally excluded by the .so check. + if ext == ".so" and self.skip_private_shared_objects: + if ".libs/" in path or path.split("/")[0].endswith(".libs"): + return + if "purelib" in path or "platlib" in path: root = "/".join(path.split("/")[2:]) else: @@ -158,10 +174,15 @@ def data_has_purelib_or_platlib(path): ) parser.add_argument("--output_file", type=str) parser.add_argument("--include_stub_packages", action="store_true") + parser.add_argument("--skip_private_shared_objects", action="store_true") parser.add_argument("--exclude_patterns", nargs="+", default=[]) parser.add_argument("--wheels", nargs="+", default=[]) args = parser.parse_args() generator = Generator( - sys.stderr, args.output_file, args.exclude_patterns, args.include_stub_packages + sys.stderr, + args.output_file, + args.exclude_patterns, + args.include_stub_packages, + args.skip_private_shared_objects, ) sys.exit(generator.run(args.wheels)) diff --git a/gazelle/modules_mapping/test_generator.py b/gazelle/modules_mapping/test_generator.py index d6d2f19039..e25ae7c9e1 100644 --- a/gazelle/modules_mapping/test_generator.py +++ b/gazelle/modules_mapping/test_generator.py @@ -7,7 +7,7 @@ class GeneratorTest(unittest.TestCase): def test_generator(self): whl = pathlib.Path(__file__).parent / "pytest-8.3.3-py3-none-any.whl" - gen = Generator(None, None, {}, False) + gen = Generator(None, None, {}, False, False) gen.dig_wheel(whl) self.assertLessEqual( { @@ -21,7 +21,7 @@ def test_generator(self): def test_stub_generator(self): whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl" - gen = Generator(None, None, {}, True) + gen = Generator(None, None, {}, True, False) gen.dig_wheel(whl) self.assertLessEqual( { @@ -32,13 +32,66 @@ def test_stub_generator(self): def test_stub_excluded(self): whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl" - gen = Generator(None, None, {}, False) + gen = Generator(None, None, {}, False, False) gen.dig_wheel(whl) self.assertEqual( {}.items(), gen.mapping.items(), ) + def test_skip_private_shared_objects(self): + # Test the skip_private_shared_objects functionality with the module_for_path method + gen_with_private_libs = Generator(None, None, {}, False, False) + gen_without_private_libs = Generator(None, None, {}, False, True) + + # Simulate Python files - should be included in both cases + gen_with_private_libs.module_for_path( + "cv2/__init__.py", + "opencv_python_headless-4.12.0.88-cp37-abi3-manylinux2014_x86_64.whl", + ) + gen_without_private_libs.module_for_path( + "cv2/__init__.py", + "opencv_python_headless-4.12.0.88-cp37-abi3-manylinux2014_x86_64.whl", + ) + gen_with_private_libs.module_for_path( + "numpy/__init__.py", "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.whl" + ) + gen_without_private_libs.module_for_path( + "numpy/__init__.py", "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.whl" + ) + + # Real-world examples from wheels + private_shared_objects = [ + "opencv_python_headless.libs/libopenblas-r0-f650aae0.so", + "numpy.libs/libscipy_openblas64_-56d6093b.so", + ] + + # Add all private shared objects to both generators + for lib_path in private_shared_objects: + wheel_name = ( + "opencv_python_headless-4.12.0.88" + if "opencv" in lib_path + else "numpy-2.2.6" + ) + gen_with_private_libs.module_for_path(lib_path, f"{wheel_name}.whl") + gen_without_private_libs.module_for_path(lib_path, f"{wheel_name}.whl") + + # Both should have the Python module mappings + self.assertIn("cv2", gen_with_private_libs.mapping) + self.assertIn("cv2", gen_without_private_libs.mapping) + self.assertIn("numpy", gen_with_private_libs.mapping) + self.assertIn("numpy", gen_without_private_libs.mapping) + + # Only gen_with_private_libs should have the private shared object mappings + expected_private_mappings = [ + "opencv_python_headless.libs.libopenblas-r0-f650aae0", + "numpy.libs.libscipy_openblas64_-56d6093b", + ] + + for mapping in expected_private_mappings: + self.assertIn(mapping, gen_with_private_libs.mapping) + self.assertNotIn(mapping, gen_without_private_libs.mapping) + if __name__ == "__main__": unittest.main()