feat(modules_mapping): add skip_private_shared_objects parameter for hermetic gazelle file generation

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/docs,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/docs,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
 
 test --test_output=errors
 

examples/build_file_generation/BUILD.bazel

@@ -28,6 +28,7 @@ modules_mapping(
         "^_|(\\._)+",  # This is the default.
         "(\\.tests)+",  # Add a custom one to get rid of the psutil tests.
     ],
+    # skip_private_shared_objects = True,
     wheels = all_whl_requirements,
 )
 

gazelle/docs/installation_and_usage.md

@@ -96,6 +96,15 @@ modules_mapping(
     # for tools like type checkers and IDEs, improving the development experience and
     # reducing manual overhead in managing separate stub packages.
     include_stub_packages = True,
+
+    # skip_private_shared_objects: bool (default: False)
+    # If set to True, this flag skips private shared objects under .libs directories
+    # when generating the modules mapping. These are non-importable dependency libraries
+    # (like libopenblas.so) that vary between Linux distributions and break build
+    # hermiticity. Ensures identical manifests across platforms by excluding libraries
+    # that cannot be imported in Python code. macOS uses .dylib files which are
+    # naturally excluded by this Linux-specific .so filtering.
+    skip_private_shared_objects = True,
 )
 
 # Gazelle python extension needs a manifest file mapping from

gazelle/modules_mapping/def.bzl

@@ -38,6 +38,8 @@ def _modules_mapping_impl(ctx):
     args.set_param_file_format(format = "multiline")
     if ctx.attr.include_stub_packages:
         args.add("--include_stub_packages")
+    if ctx.attr.skip_private_shared_objects:
+        args.add("--skip_private_shared_objects")
     args.add("--output_file", modules_mapping)
     args.add_all("--exclude_patterns", ctx.attr.exclude_patterns)
     args.add_all("--wheels", all_wheels)
@@ -69,6 +71,14 @@ modules_mapping = rule(
             doc = "The name for the output JSON file.",
             mandatory = False,
         ),
+        "skip_private_shared_objects": attr.bool(
+            default = False,
+            doc = "Whether to skip private shared objects under .libs directories when generating mappings. " +
+                  "When True, excludes non-importable dependency libraries (like libopenblas.so) that vary " +
+                  "between Linux platforms and break build hermiticity. These .libs files are not actual " +
+                  "Python modules and cannot be imported. macOS uses .dylib files which are naturally excluded.",
+            mandatory = False,
+        ),
         "wheels": attr.label_list(
             allow_files = True,
             doc = "The list of wheels, usually the 'all_whl_requirements' from @<pip_repository>//:requirements.bzl",

gazelle/modules_mapping/generator.py

@@ -26,11 +26,19 @@ class Generator:
     output_file = None
     excluded_patterns = None
 
-    def __init__(self, stderr, output_file, excluded_patterns, include_stub_packages):
+    def __init__(
+        self,
+        stderr,
+        output_file,
+        excluded_patterns,
+        include_stub_packages,
+        skip_private_shared_objects=False,
+    ):
         self.stderr = stderr
         self.output_file = output_file
         self.excluded_patterns = [re.compile(pattern) for pattern in excluded_patterns]
         self.include_stub_packages = include_stub_packages
+        self.skip_private_shared_objects = skip_private_shared_objects
         self.mapping = {}
 
     # dig_wheel analyses the wheel .whl file determining the modules it provides
@@ -74,6 +82,14 @@ def simplify(self):
     def module_for_path(self, path, whl):
         ext = pathlib.Path(path).suffix
         if ext == ".py" or ext == ".so":
+            # Skip private shared objects under .libs directories on Linux.
+            # These are non-importable dependency libraries (like libopenblas.so) that vary
+            # between platforms and make builds non-hermetic. macOS uses .dylib files
+            # which are naturally excluded by the .so check.
+            if ext == ".so" and self.skip_private_shared_objects:
+                if ".libs/" in path or path.split("/")[0].endswith(".libs"):
+                    return
+
             if "purelib" in path or "platlib" in path:
                 root = "/".join(path.split("/")[2:])
             else:
@@ -158,10 +174,15 @@ def data_has_purelib_or_platlib(path):
     )
     parser.add_argument("--output_file", type=str)
     parser.add_argument("--include_stub_packages", action="store_true")
+    parser.add_argument("--skip_private_shared_objects", action="store_true")
     parser.add_argument("--exclude_patterns", nargs="+", default=[])
     parser.add_argument("--wheels", nargs="+", default=[])
     args = parser.parse_args()
     generator = Generator(
-        sys.stderr, args.output_file, args.exclude_patterns, args.include_stub_packages
+        sys.stderr,
+        args.output_file,
+        args.exclude_patterns,
+        args.include_stub_packages,
+        args.skip_private_shared_objects,
     )
     sys.exit(generator.run(args.wheels))

gazelle/modules_mapping/test_generator.py

@@ -7,7 +7,7 @@
 class GeneratorTest(unittest.TestCase):
     def test_generator(self):
         whl = pathlib.Path(__file__).parent / "pytest-8.3.3-py3-none-any.whl"
-        gen = Generator(None, None, {}, False)
+        gen = Generator(None, None, {}, False, False)
         gen.dig_wheel(whl)
         self.assertLessEqual(
             {
@@ -21,7 +21,7 @@ def test_generator(self):
 
     def test_stub_generator(self):
         whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl"
-        gen = Generator(None, None, {}, True)
+        gen = Generator(None, None, {}, True, False)
         gen.dig_wheel(whl)
         self.assertLessEqual(
             {
@@ -32,13 +32,66 @@ def test_stub_generator(self):
 
     def test_stub_excluded(self):
         whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl"
-        gen = Generator(None, None, {}, False)
+        gen = Generator(None, None, {}, False, False)
         gen.dig_wheel(whl)
         self.assertEqual(
             {}.items(),
             gen.mapping.items(),
         )
 
+    def test_skip_private_shared_objects(self):
+        # Test the skip_private_shared_objects functionality with the module_for_path method
+        gen_with_private_libs = Generator(None, None, {}, False, False)
+        gen_without_private_libs = Generator(None, None, {}, False, True)
+
+        # Simulate Python files - should be included in both cases
+        gen_with_private_libs.module_for_path(
+            "cv2/__init__.py",
+            "opencv_python_headless-4.12.0.88-cp37-abi3-manylinux2014_x86_64.whl",
+        )
+        gen_without_private_libs.module_for_path(
+            "cv2/__init__.py",
+            "opencv_python_headless-4.12.0.88-cp37-abi3-manylinux2014_x86_64.whl",
+        )
+        gen_with_private_libs.module_for_path(
+            "numpy/__init__.py", "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.whl"
+        )
+        gen_without_private_libs.module_for_path(
+            "numpy/__init__.py", "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.whl"
+        )
+
+        # Real-world examples from wheels
+        private_shared_objects = [
+            "opencv_python_headless.libs/libopenblas-r0-f650aae0.so",
+            "numpy.libs/libscipy_openblas64_-56d6093b.so",
+        ]
+
+        # Add all private shared objects to both generators
+        for lib_path in private_shared_objects:
+            wheel_name = (
+                "opencv_python_headless-4.12.0.88"
+                if "opencv" in lib_path
+                else "numpy-2.2.6"
+            )
+            gen_with_private_libs.module_for_path(lib_path, f"{wheel_name}.whl")
+            gen_without_private_libs.module_for_path(lib_path, f"{wheel_name}.whl")
+
+        # Both should have the Python module mappings
+        self.assertIn("cv2", gen_with_private_libs.mapping)
+        self.assertIn("cv2", gen_without_private_libs.mapping)
+        self.assertIn("numpy", gen_with_private_libs.mapping)
+        self.assertIn("numpy", gen_without_private_libs.mapping)
+
+        # Only gen_with_private_libs should have the private shared object mappings
+        expected_private_mappings = [
+            "opencv_python_headless.libs.libopenblas-r0-f650aae0",
+            "numpy.libs.libscipy_openblas64_-56d6093b",
+        ]
+
+        for mapping in expected_private_mappings:
+            self.assertIn(mapping, gen_with_private_libs.mapping)
+            self.assertNotIn(mapping, gen_without_private_libs.mapping)
+
 
 if __name__ == "__main__":
     unittest.main()