Add ignore_native_libs option for hermetic manifest generation

Fixes platform-specific inconsistency in gazelle_python_manifest where
different native libraries (.so files) in platform-specific wheels
caused different manifests on Linux vs macOS.

Changes:
- Add ignore_native_libs parameter to modules_mapping rule (default: False)
- Skip .so file processing in generator.py when flag is enabled
- Add comprehensive test coverage for the new functionality

Usage:
```starlark
modules_mapping(
    name = "modules_map",
    wheels = all_whl_requirements,
    ignore_native_libs = True,  # Enables hermetic cross-platform builds
)
```

This solves the opencv-python-headless case and similar packages with
platform-specific native libraries, enabling hermetic builds across
different development platforms.

Author: AFMiziara

gazelle/modules_mapping/def.bzl

@@ -38,6 +38,8 @@ def _modules_mapping_impl(ctx):
     args.set_param_file_format(format = "multiline")
     if ctx.attr.include_stub_packages:
         args.add("--include_stub_packages")
+    if ctx.attr.ignore_native_libs:
+        args.add("--ignore_native_libs")
     args.add("--output_file", modules_mapping)
     args.add_all("--exclude_patterns", ctx.attr.exclude_patterns)
     args.add_all("--wheels", all_wheels)
@@ -64,6 +66,11 @@ modules_mapping = rule(
             doc = "Whether to include stub packages in the mapping.",
             mandatory = False,
         ),
+        "ignore_native_libs": attr.bool(
+            default = False,
+            doc = "Whether to ignore native libraries (*.so files) for platform-independent mappings.",
+            mandatory = False,
+        ),
         "modules_mapping_name": attr.string(
             default = "modules_mapping.json",
             doc = "The name for the output JSON file.",

gazelle/modules_mapping/generator.py

@@ -26,11 +26,12 @@ class Generator:
     output_file = None
     excluded_patterns = None
 
-    def __init__(self, stderr, output_file, excluded_patterns, include_stub_packages):
+    def __init__(self, stderr, output_file, excluded_patterns, include_stub_packages, ignore_native_libs=False):
         self.stderr = stderr
         self.output_file = output_file
         self.excluded_patterns = [re.compile(pattern) for pattern in excluded_patterns]
         self.include_stub_packages = include_stub_packages
+        self.ignore_native_libs = ignore_native_libs
         self.mapping = {}
 
     # dig_wheel analyses the wheel .whl file determining the modules it provides
@@ -74,6 +75,10 @@ def simplify(self):
     def module_for_path(self, path, whl):
         ext = pathlib.Path(path).suffix
         if ext == ".py" or ext == ".so":
+            # Skip native libraries if ignore_native_libs is enabled
+            if ext == ".so" and self.ignore_native_libs:
+                return
+
             if "purelib" in path or "platlib" in path:
                 root = "/".join(path.split("/")[2:])
             else:
@@ -158,10 +163,11 @@ def data_has_purelib_or_platlib(path):
     )
     parser.add_argument("--output_file", type=str)
     parser.add_argument("--include_stub_packages", action="store_true")
+    parser.add_argument("--ignore_native_libs", action="store_true")
     parser.add_argument("--exclude_patterns", nargs="+", default=[])
     parser.add_argument("--wheels", nargs="+", default=[])
     args = parser.parse_args()
     generator = Generator(
-        sys.stderr, args.output_file, args.exclude_patterns, args.include_stub_packages
+        sys.stderr, args.output_file, args.exclude_patterns, args.include_stub_packages, args.ignore_native_libs
     )
     sys.exit(generator.run(args.wheels))

gazelle/modules_mapping/test_generator.py

@@ -7,7 +7,7 @@
 class GeneratorTest(unittest.TestCase):
     def test_generator(self):
         whl = pathlib.Path(__file__).parent / "pytest-8.3.3-py3-none-any.whl"
-        gen = Generator(None, None, {}, False)
+        gen = Generator(None, None, {}, False, False)
         gen.dig_wheel(whl)
         self.assertLessEqual(
             {
@@ -21,7 +21,7 @@ def test_generator(self):
 
     def test_stub_generator(self):
         whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl"
-        gen = Generator(None, None, {}, True)
+        gen = Generator(None, None, {}, True, False)
         gen.dig_wheel(whl)
         self.assertLessEqual(
             {
@@ -32,13 +32,34 @@ def test_stub_generator(self):
 
     def test_stub_excluded(self):
         whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl"
-        gen = Generator(None, None, {}, False)
+        gen = Generator(None, None, {}, False, False)
         gen.dig_wheel(whl)
         self.assertEqual(
             {}.items(),
             gen.mapping.items(),
         )
 
+    def test_ignore_native_libs(self):
+        # Test the ignore_native_libs functionality with the module_for_path method
+        gen_with_native_libs = Generator(None, None, {}, False, False)
+        gen_without_native_libs = Generator(None, None, {}, False, True)
+
+        # Simulate a Python file - should be included in both cases
+        gen_with_native_libs.module_for_path("cv2/__init__.py", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
+        gen_without_native_libs.module_for_path("cv2/__init__.py", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
+
+        # Simulate a native library - should be included only when ignore_native_libs=False
+        gen_with_native_libs.module_for_path("opencv_python_headless.libs/libopenblas-r0-f650aae0.so", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
+        gen_without_native_libs.module_for_path("opencv_python_headless.libs/libopenblas-r0-f650aae0.so", "opencv_python_headless-4.8.1-cp310-cp310-linux_x86_64.whl")
+
+        # Both should have the Python module mapping
+        self.assertIn("cv2", gen_with_native_libs.mapping)
+        self.assertIn("cv2", gen_without_native_libs.mapping)
+
+        # Only gen_with_native_libs should have the native library mapping
+        self.assertIn("opencv_python_headless.libs.libopenblas-r0-f650aae0", gen_with_native_libs.mapping)
+        self.assertNotIn("opencv_python_headless.libs.libopenblas-r0-f650aae0", gen_without_native_libs.mapping)
+
 
 if __name__ == "__main__":
     unittest.main()