Remove binary wheels and add internal_dev_deps

Author: ewianda

gazelle/MODULE.bazel

@@ -33,3 +33,14 @@ use_repo(
     python_stdlib_list,
     "python_stdlib_list",
 )
+
+internal_dev_deps = use_extension(
+    "//:internal_dev_deps.bzl",
+    "internal_dev_deps_extension",
+    dev_dependency = True,
+)
+use_repo(
+    internal_dev_deps,
+    "django-types",
+    "pytest",
+)

gazelle/internal_dev_deps.bzl

@@ -0,0 +1,47 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module extension for internal dev_dependency=True setup."""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
+
+def internal_dev_deps():
+    """This extension creates internal rules_python_gazelle dev dependencies."""
+    http_file(
+        name = "pytest",
+        downloaded_file_path = "pytest-8.3.3-py3-none-any.whl",
+        sha256 = "a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2",
+        urls = [
+            "https://files.pythonhosted.org/packages/6b/77/7440a06a8ead44c7757a64362dd22df5760f9b12dc5f11b6188cd2fc27a0/pytest-8.3.3-py3-none-any.whl",
+        ],
+    )
+    http_file(
+        name = "django-types",
+        downloaded_file_path = "django_types-0.19.1-py3-none-any.whl",
+        sha256 = "b3f529de17f6374d41ca67232aa01330c531bbbaa3ac4097896f31ac33c96c30",
+        urls = [
+            "https://files.pythonhosted.org/packages/25/cb/d088c67245a9d5759a08dbafb47e040ee436e06ee433a3cdc7f3233b3313/django_types-0.19.1-py3-none-any.whl",
+        ],
+    )
+
+def _internal_dev_deps_impl(mctx):
+    _ = mctx  # @unused
+
+    # This wheel is purely here to validate the wheel extraction code. It's not
+    # intended for anything else.
+    internal_dev_deps()
+
+internal_dev_deps_extension = module_extension(
+    implementation = _internal_dev_deps_impl,
+    doc = "This extension creates internal rules_python_gazelle dev dependencies.",
+)

gazelle/modules_mapping/BUILD.bazel

@@ -1,3 +1,4 @@
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@rules_python//python:defs.bzl", "py_binary", "py_test")
 
 # gazelle:exclude *.py
@@ -8,10 +9,25 @@ py_binary(
     visibility = ["//visibility:public"],
 )
 
+copy_file(
+    name = "pytest_wheel",
+    src = "@pytest//file",
+    out = "pytest-8.3.3-py3-none-any.whl",
+)
+
+copy_file(
+    name = "django_types_wheel",
+    src = "@django-types//file",
+    out = "django_types-0.19.1-py3-none-any.whl",
+)
+
 py_test(
     name = "test_generator",
     srcs = ["test_generator.py"],
-    data = glob(["testdata/**"]),
+    data = [
+        "django_types_wheel",
+        "pytest_wheel",
+    ],
     imports = ["."],
     main = "test_generator.py",
     deps = [":generator"],

gazelle/modules_mapping/def.bzl

@@ -51,16 +51,16 @@ modules_mapping = rule(
             doc = "A set of regex patterns to match against each calculated module path. By default, exclude the modules starting with underscores.",
             mandatory = False,
         ),
-        "modules_mapping_name": attr.string(
-            default = "modules_mapping.json",
-            doc = "The name for the output JSON file.",
-            mandatory = False,
-        ),
         "include_stub_packages": attr.bool(
             default = False,
             doc = "Whether to include stub packages in the mapping.",
             mandatory = False,
         ),
+        "modules_mapping_name": attr.string(
+            default = "modules_mapping.json",
+            doc = "The name for the output JSON file.",
+            mandatory = False,
+        ),
         "wheels": attr.label_list(
             allow_files = True,
             doc = "The list of wheels, usually the 'all_whl_requirements' from @<pip_repository>//:requirements.bzl",

gazelle/modules_mapping/generator.py

@@ -38,7 +38,10 @@ def __init__(self, stderr, output_file, excluded_patterns, include_stub_packages
     def dig_wheel(self, whl):
         # Skip stubs and types wheels.
         wheel_name = get_wheel_name(whl)
-        if wheel_name.endswith(("_stubs", "_types")) and self.include_stub_packages:
+        if self.include_stub_packages and (
+            wheel_name.endswith(("_stubs", "_types"))
+            or wheel_name.startswith(("types_", "stubs_"))
+        ):
             self.mapping[wheel_name.lower()] = wheel_name.lower()
             return
         with zipfile.ZipFile(whl, "r") as zip_file:

gazelle/modules_mapping/test_generator.py

@@ -6,9 +6,7 @@
 
 class GeneratorTest(unittest.TestCase):
     def test_generator(self):
-        whl = pathlib.Path(
-            pathlib.Path(__file__).parent, "testdata", "pytest-7.1.1-py3-none-any.whl"
-        )
+        whl = pathlib.Path(__file__).parent / "pytest-8.3.3-py3-none-any.whl"
         gen = Generator(None, None, {}, False)
         gen.dig_wheel(whl)
         self.assertLessEqual(
@@ -22,11 +20,7 @@ def test_generator(self):
         )
 
     def test_stub_generator(self):
-        whl = pathlib.Path(
-            pathlib.Path(__file__).parent,
-            "testdata",
-            "django_types-0.15.0-py3-none-any.whl",
-        )
+        whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl"
         gen = Generator(None, None, {}, True)
         gen.dig_wheel(whl)
         self.assertLessEqual(
@@ -37,11 +31,7 @@ def test_stub_generator(self):
         )
 
     def test_stub_excluded(self):
-        whl = pathlib.Path(
-            pathlib.Path(__file__).parent
-            / "testdata"
-            / "django_types-0.15.0-py3-none-any.whl"
-        )
+        whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl"
         gen = Generator(None, None, {}, False)
         gen.dig_wheel(whl)
         self.assertEqual(

gazelle/modules_mapping/testdata/django_types-0.15.0-py3-none-any.whl

@@ -192,13 +192,16 @@ func (py *Resolver) Resolve(
 					if dep, distributionName, ok := cfg.FindThirdPartyDependency(moduleName); ok {
 						deps.Add(dep)
 						// Add the type and stub dependencies if they exist.
-						typeModule := fmt.Sprintf("%s_types", strings.ToLower(distributionName))
-						if dep, _, ok := cfg.FindThirdPartyDependency(typeModule); ok {
-							deps.Add(dep)
+						modules := []string{
+							fmt.Sprintf("%s_stubs", strings.ToLower(distributionName)),
+							fmt.Sprintf("%s_types", strings.ToLower(distributionName)),
+							fmt.Sprintf("types_%s", strings.ToLower(distributionName)),
+							fmt.Sprintf("stubs_%s", strings.ToLower(distributionName)),
 						}
-						stubModule := fmt.Sprintf("%s_stubs", strings.ToLower(distributionName))
-						if dep, _, ok := cfg.FindThirdPartyDependency(stubModule); ok {
-							deps.Add(dep)
+						for _, module := range modules {
+							if dep, _, ok := cfg.FindThirdPartyDependency(module); ok {
+								deps.Add(dep)
+							}
 						}
 						if explainDependency == dep {
 							log.Printf("Explaining dependency (%s): "+