feat(modules_mapping): add skip_private_shared_objects parameter for hermetic gazelle file generation

examples/build_file_generation/BUILD.bazel

@@ -28,6 +28,9 @@ modules_mapping(
         "^_|(\\._)+",  # This is the default.
         "(\\.tests)+",  # Add a custom one to get rid of the psutil tests.
     ],
+    # Uncomment the next line to enable hermetic builds across platforms
+    # by ignoring platform-specific native libraries (useful for opencv-python-headless, etc.)
+    # ignore_native_libs = True,
     wheels = all_whl_requirements,
 )
 

gazelle/docs/installation_and_usage.md

@@ -96,6 +96,14 @@ modules_mapping(
     # for tools like type checkers and IDEs, improving the development experience and
     # reducing manual overhead in managing separate stub packages.
     include_stub_packages = True,
+
+    # ignore_native_libs: bool (default: False)
+    # If set to True, this flag ignores platform-specific native libraries (.so files)
+    # when generating the modules mapping. This ensures hermetic builds across different
+    # platforms (Linux, macOS, etc.) by producing identical manifests regardless of
+    # platform-specific wheel contents. Useful for packages like opencv-python-headless
+    # that include different native libraries on different platforms.
+    # ignore_native_libs = True,
 )
 
 # Gazelle python extension needs a manifest file mapping from

gazelle/modules_mapping/def.bzl

@@ -38,6 +38,8 @@ def _modules_mapping_impl(ctx):
     args.set_param_file_format(format = "multiline")
     if ctx.attr.include_stub_packages:
         args.add("--include_stub_packages")
+    if ctx.attr.ignore_native_libs:
+        args.add("--ignore_native_libs")
     args.add("--output_file", modules_mapping)
     args.add_all("--exclude_patterns", ctx.attr.exclude_patterns)
     args.add_all("--wheels", all_wheels)
@@ -64,6 +66,14 @@ modules_mapping = rule(
             doc = "Whether to include stub packages in the mapping.",
             mandatory = False,
         ),
+        "ignore_native_libs": attr.bool(
+            default = False,
+            doc = "Whether to ignore platform-specific native libraries (*.so files) when generating mappings. " +
+                  "When True, ensures hermetic builds across different platforms by excluding native library " +
+                  "mappings that vary between Linux, macOS, etc. Useful for packages like opencv-python-headless " +
+                  "that bundle different native libraries on different platforms.",
+            mandatory = False,
+        ),
         "modules_mapping_name": attr.string(
             default = "modules_mapping.json",
             doc = "The name for the output JSON file.",

gazelle/modules_mapping/generator.py

@@ -26,11 +26,12 @@ class Generator:
     output_file = None
     excluded_patterns = None
 
-    def __init__(self, stderr, output_file, excluded_patterns, include_stub_packages):
+    def __init__(self, stderr, output_file, excluded_patterns, include_stub_packages, ignore_native_libs=False):
         self.stderr = stderr
         self.output_file = output_file
         self.excluded_patterns = [re.compile(pattern) for pattern in excluded_patterns]
         self.include_stub_packages = include_stub_packages
+        self.ignore_native_libs = ignore_native_libs
         self.mapping = {}
 
     # dig_wheel analyses the wheel .whl file determining the modules it provides
@@ -74,6 +75,10 @@ def simplify(self):
     def module_for_path(self, path, whl):
         ext = pathlib.Path(path).suffix
         if ext == ".py" or ext == ".so":
+            # Skip native libraries if ignore_native_libs is enabled
+            if ext == ".so" and self.ignore_native_libs:
+                return
+
             if "purelib" in path or "platlib" in path:
                 root = "/".join(path.split("/")[2:])
             else:
@@ -158,10 +163,11 @@ def data_has_purelib_or_platlib(path):
     )
     parser.add_argument("--output_file", type=str)
     parser.add_argument("--include_stub_packages", action="store_true")
+    parser.add_argument("--ignore_native_libs", action="store_true")
     parser.add_argument("--exclude_patterns", nargs="+", default=[])
     parser.add_argument("--wheels", nargs="+", default=[])
     args = parser.parse_args()
     generator = Generator(
-        sys.stderr, args.output_file, args.exclude_patterns, args.include_stub_packages
+        sys.stderr, args.output_file, args.exclude_patterns, args.include_stub_packages, args.ignore_native_libs
     )
     sys.exit(generator.run(args.wheels))

gazelle/modules_mapping/test_generator.py

@@ -7,7 +7,7 @@
 class GeneratorTest(unittest.TestCase):
     def test_generator(self):
         whl = pathlib.Path(__file__).parent / "pytest-8.3.3-py3-none-any.whl"
-        gen = Generator(None, None, {}, False)
+        gen = Generator(None, None, {}, False, False)
         gen.dig_wheel(whl)
         self.assertLessEqual(
             {
@@ -21,7 +21,7 @@ def test_generator(self):
 
     def test_stub_generator(self):
         whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl"
-        gen = Generator(None, None, {}, True)
+        gen = Generator(None, None, {}, True, False)
         gen.dig_wheel(whl)
         self.assertLessEqual(
             {
@@ -32,13 +32,34 @@ def test_stub_generator(self):
 
     def test_stub_excluded(self):
         whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl"
-        gen = Generator(None, None, {}, False)
+        gen = Generator(None, None, {}, False, False)
         gen.dig_wheel(whl)
         self.assertEqual(
             {}.items(),
             gen.mapping.items(),
         )
 
+    def test_ignore_native_libs(self):
+        # Test the ignore_native_libs functionality with the module_for_path method
+        gen_with_native_libs = Generator(None, None, {}, False, False)
+        gen_without_native_libs = Generator(None, None, {}, False, True)
+
+        # Simulate a Python file - should be included in both cases
+        gen_with_native_libs.module_for_path("cv2/__init__.py", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
+        gen_without_native_libs.module_for_path("cv2/__init__.py", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
+
+        # Simulate a native library - should be included only when ignore_native_libs=False
+        gen_with_native_libs.module_for_path("opencv_python_headless.libs/libopenblas-r0-f650aae0.so", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
+        gen_without_native_libs.module_for_path("opencv_python_headless.libs/libopenblas-r0-f650aae0.so", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
+
+        # Both should have the Python module mapping
+        self.assertIn("cv2", gen_with_native_libs.mapping)
+        self.assertIn("cv2", gen_without_native_libs.mapping)
+
+        # Only gen_with_native_libs should have the native library mapping
+        self.assertIn("opencv_python_headless.libs.libopenblas-r0-f650aae0", gen_with_native_libs.mapping)
+        self.assertNotIn("opencv_python_headless.libs.libopenblas-r0-f650aae0", gen_without_native_libs.mapping)
+
 
 if __name__ == "__main__":
     unittest.main()