refact

Author: AFMiziara

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/docs,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/docs,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
 
 test --test_output=errors
 

examples/build_file_generation/BUILD.bazel

@@ -29,8 +29,8 @@ modules_mapping(
         "(\\.tests)+",  # Add a custom one to get rid of the psutil tests.
     ],
     # Uncomment the next line to enable hermetic builds across platforms
-    # by ignoring platform-specific native libraries (useful for opencv-python-headless, etc.)
-    # ignore_native_libs = True,
+    # by skipping private shared objects under .libs directories (useful for opencv-python-headless, etc.)
+    # skip_private_shared_objects = True,
     wheels = all_whl_requirements,
 )
 

gazelle/docs/installation_and_usage.md

@@ -97,13 +97,14 @@ modules_mapping(
     # reducing manual overhead in managing separate stub packages.
     include_stub_packages = True,
 
-    # ignore_native_libs: bool (default: False)
-    # If set to True, this flag ignores platform-specific native libraries (.so files)
-    # when generating the modules mapping. This ensures hermetic builds across different
-    # platforms (Linux, macOS, etc.) by producing identical manifests regardless of
-    # platform-specific wheel contents. Useful for packages like opencv-python-headless
-    # that include different native libraries on different platforms.
-    ignore_native_libs = True,
+    # skip_private_shared_objects: bool (default: False)
+    # If set to True, this flag skips private shared objects under .libs directories
+    # when generating the modules mapping. These are non-importable dependency libraries
+    # (like libopenblas.so) that vary between Linux distributions and break build
+    # hermiticity. Ensures identical manifests across platforms by excluding libraries
+    # that cannot be imported in Python code. macOS uses .dylib files which are
+    # naturally excluded by this Linux-specific .so filtering.
+    skip_private_shared_objects = True,
 )
 
 # Gazelle python extension needs a manifest file mapping from

gazelle/modules_mapping/def.bzl

@@ -38,8 +38,8 @@ def _modules_mapping_impl(ctx):
     args.set_param_file_format(format = "multiline")
     if ctx.attr.include_stub_packages:
         args.add("--include_stub_packages")
-    if ctx.attr.ignore_native_libs:
-        args.add("--ignore_native_libs")
+    if ctx.attr.skip_private_shared_objects:
+        args.add("--skip_private_shared_objects")
     args.add("--output_file", modules_mapping)
     args.add_all("--exclude_patterns", ctx.attr.exclude_patterns)
     args.add_all("--wheels", all_wheels)
@@ -66,19 +66,19 @@ modules_mapping = rule(
             doc = "Whether to include stub packages in the mapping.",
             mandatory = False,
         ),
-        "ignore_native_libs": attr.bool(
-            default = False,
-            doc = "Whether to ignore platform-specific native libraries (*.so files) when generating mappings. " +
-                  "When True, ensures hermetic builds across different platforms by excluding native library " +
-                  "mappings that vary between Linux, macOS, etc. Useful for packages like opencv-python-headless " +
-                  "that bundle different native libraries on different platforms.",
-            mandatory = False,
-        ),
         "modules_mapping_name": attr.string(
             default = "modules_mapping.json",
             doc = "The name for the output JSON file.",
             mandatory = False,
         ),
+        "skip_private_shared_objects": attr.bool(
+            default = False,
+            doc = "Whether to skip private shared objects under .libs directories when generating mappings. " +
+                  "When True, excludes non-importable dependency libraries (like libopenblas.so) that vary " +
+                  "between Linux platforms and break build hermiticity. These .libs files are not actual " +
+                  "Python modules and cannot be imported. macOS uses .dylib files which are naturally excluded.",
+            mandatory = False,
+        ),
         "wheels": attr.label_list(
             allow_files = True,
             doc = "The list of wheels, usually the 'all_whl_requirements' from @<pip_repository>//:requirements.bzl",

gazelle/modules_mapping/generator.py

@@ -26,12 +26,19 @@ class Generator:
     output_file = None
     excluded_patterns = None
 
-    def __init__(self, stderr, output_file, excluded_patterns, include_stub_packages, ignore_native_libs=False):
+    def __init__(
+        self,
+        stderr,
+        output_file,
+        excluded_patterns,
+        include_stub_packages,
+        skip_private_shared_objects=False,
+    ):
         self.stderr = stderr
         self.output_file = output_file
         self.excluded_patterns = [re.compile(pattern) for pattern in excluded_patterns]
         self.include_stub_packages = include_stub_packages
-        self.ignore_native_libs = ignore_native_libs
+        self.skip_private_shared_objects = skip_private_shared_objects
         self.mapping = {}
 
     # dig_wheel analyses the wheel .whl file determining the modules it provides
@@ -75,9 +82,13 @@ def simplify(self):
     def module_for_path(self, path, whl):
         ext = pathlib.Path(path).suffix
         if ext == ".py" or ext == ".so":
-            # Skip native libraries if ignore_native_libs is enabled
-            if ext == ".so" and self.ignore_native_libs:
-                return
+            # Skip private shared objects under .libs directories on Linux.
+            # These are non-importable dependency libraries (like libopenblas.so) that vary
+            # between platforms and make builds non-hermetic. macOS uses .dylib files
+            # which are naturally excluded by the .so check.
+            if ext == ".so" and self.skip_private_shared_objects:
+                if ".libs/" in path or path.split("/")[0].endswith(".libs"):
+                    return
 
             if "purelib" in path or "platlib" in path:
                 root = "/".join(path.split("/")[2:])
@@ -163,11 +174,15 @@ def data_has_purelib_or_platlib(path):
     )
     parser.add_argument("--output_file", type=str)
     parser.add_argument("--include_stub_packages", action="store_true")
-    parser.add_argument("--ignore_native_libs", action="store_true")
+    parser.add_argument("--skip_private_shared_objects", action="store_true")
     parser.add_argument("--exclude_patterns", nargs="+", default=[])
     parser.add_argument("--wheels", nargs="+", default=[])
     args = parser.parse_args()
     generator = Generator(
-        sys.stderr, args.output_file, args.exclude_patterns, args.include_stub_packages, args.ignore_native_libs
+        sys.stderr,
+        args.output_file,
+        args.exclude_patterns,
+        args.include_stub_packages,
+        args.skip_private_shared_objects,
     )
     sys.exit(generator.run(args.wheels))

gazelle/modules_mapping/test_generator.py

@@ -39,26 +39,58 @@ def test_stub_excluded(self):
             gen.mapping.items(),
         )
 
-    def test_ignore_native_libs(self):
-        # Test the ignore_native_libs functionality with the module_for_path method
-        gen_with_native_libs = Generator(None, None, {}, False, False)
-        gen_without_native_libs = Generator(None, None, {}, False, True)
+    def test_skip_private_shared_objects(self):
+        # Test the skip_private_shared_objects functionality with the module_for_path method
+        gen_with_private_libs = Generator(None, None, {}, False, False)
+        gen_without_private_libs = Generator(None, None, {}, False, True)
 
-        # Simulate a Python file - should be included in both cases
-        gen_with_native_libs.module_for_path("cv2/__init__.py", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
-        gen_without_native_libs.module_for_path("cv2/__init__.py", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
+        # Simulate Python files - should be included in both cases
+        gen_with_private_libs.module_for_path(
+            "cv2/__init__.py",
+            "opencv_python_headless-4.12.0.88-cp37-abi3-manylinux2014_x86_64.whl",
+        )
+        gen_without_private_libs.module_for_path(
+            "cv2/__init__.py",
+            "opencv_python_headless-4.12.0.88-cp37-abi3-manylinux2014_x86_64.whl",
+        )
+        gen_with_private_libs.module_for_path(
+            "numpy/__init__.py", "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.whl"
+        )
+        gen_without_private_libs.module_for_path(
+            "numpy/__init__.py", "numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.whl"
+        )
+
+        # Real-world examples from wheels
+        private_shared_objects = [
+            "opencv_python_headless.libs/libopenblas-r0-f650aae0.so",
+            "numpy.libs/libscipy_openblas64_-56d6093b.so",
+        ]
+
+        # Add all private shared objects to both generators
+        for lib_path in private_shared_objects:
+            wheel_name = (
+                "opencv_python_headless-4.12.0.88"
+                if "opencv" in lib_path
+                else "numpy-2.2.6"
+            )
+            gen_with_private_libs.module_for_path(lib_path, f"{wheel_name}.whl")
+            gen_without_private_libs.module_for_path(lib_path, f"{wheel_name}.whl")
 
-        # Simulate a native library - should be included only when ignore_native_libs=False
-        gen_with_native_libs.module_for_path("opencv_python_headless.libs/libopenblas-r0-f650aae0.so", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
-        gen_without_native_libs.module_for_path("opencv_python_headless.libs/libopenblas-r0-f650aae0.so", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
+        # Both should have the Python module mappings
+        self.assertIn("cv2", gen_with_private_libs.mapping)
+        self.assertIn("cv2", gen_without_private_libs.mapping)
+        self.assertIn("numpy", gen_with_private_libs.mapping)
+        self.assertIn("numpy", gen_without_private_libs.mapping)
 
-        # Both should have the Python module mapping
-        self.assertIn("cv2", gen_with_native_libs.mapping)
-        self.assertIn("cv2", gen_without_native_libs.mapping)
+        # Only gen_with_private_libs should have the private shared object mappings
+        expected_private_mappings = [
+            "opencv_python_headless.libs.libopenblas-r0-f650aae0",
+            "numpy.libs.libscipy_openblas64_-56d6093b",
+        ]
 
-        # Only gen_with_native_libs should have the native library mapping
-        self.assertIn("opencv_python_headless.libs.libopenblas-r0-f650aae0", gen_with_native_libs.mapping)
-        self.assertNotIn("opencv_python_headless.libs.libopenblas-r0-f650aae0", gen_without_native_libs.mapping)
+        for mapping in expected_private_mappings:
+            self.assertIn(mapping, gen_with_private_libs.mapping)
+            self.assertNotIn(mapping, gen_without_private_libs.mapping)
 
 
 if __name__ == "__main__":