diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bdc5e4483..0cee2c8c07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,7 +60,10 @@ Unreleased changes template. {#v0-0-0-added} ### Added -* Nothing added. +* (gazelle) Added `include_stub_packages` flag to `modules_mapping`. When set to `True`, this + automatically includes corresponding stub packages for third-party libraries + that are present and used (e.g., `boto3` → `boto3-stubs`), improving + type-checking support. {#v0-0-0-removed} ### Removed diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index a0047668cb..95bb5f88f4 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -32,11 +32,26 @@ modules_mapping( "^_|(\\._)+", # This is the default. "(\\.tests)+", # Add a custom one to get rid of the psutil tests. "^colorama", # Get rid of colorama on Windows. + "^tzdata", # Get rid of tzdata on Windows. "^lazy_object_proxy\\.cext$", # Get rid of this on Linux because it isn't included on Windows. ], wheels = all_whl_requirements, ) +modules_mapping( + name = "modules_map_with_types", + exclude_patterns = [ + "^_|(\\._)+", # This is the default. + "(\\.tests)+", # Add a custom one to get rid of the psutil tests. + "^colorama", # Get rid of colorama on Windows. + "^tzdata", # Get rid of tzdata on Windows. + "^lazy_object_proxy\\.cext$", # Get rid of this on Linux because it isn't included on Windows. + ], + include_stub_packages = True, + modules_mapping_name = "modules_mapping_with_types.json", + wheels = all_whl_requirements, +) + # Gazelle python extension needs a manifest file mapping from # an import to the installed package that provides it. # This macro produces two targets: @@ -54,6 +69,14 @@ gazelle_python_manifest( tags = ["exclusive"], ) +gazelle_python_manifest( + name = "gazelle_python_manifest_with_types", + manifest = "gazelle_python_with_types.yaml", + modules_mapping = ":modules_map_with_types", + pip_repository_name = "pip", + tags = ["exclusive"], +) + # Our gazelle target points to the python gazelle binary. # This is the simple case where we only need one language supported. # If you also had proto, go, or other gazelle-supported languages, diff --git a/examples/bzlmod_build_file_generation/gazelle_python.yaml b/examples/bzlmod_build_file_generation/gazelle_python.yaml index d0d322446e..c94f93a070 100644 --- a/examples/bzlmod_build_file_generation/gazelle_python.yaml +++ b/examples/bzlmod_build_file_generation/gazelle_python.yaml @@ -6,16 +6,20 @@ manifest: modules_mapping: S3: s3cmd + asgiref: asgiref astroid: astroid certifi: certifi chardet: chardet dateutil: python_dateutil dill: dill + django: Django + django_stubs_ext: django_stubs_ext idna: idna isort: isort lazy_object_proxy: lazy_object_proxy magic: python_magic mccabe: mccabe + mypy_django_plugin: django_stubs pathspec: pathspec pkg_resources: setuptools platformdirs: platformdirs @@ -23,6 +27,7 @@ manifest: requests: requests setuptools: setuptools six: six + sqlparse: sqlparse tabulate: tabulate tomli: tomli tomlkit: tomlkit diff --git a/examples/bzlmod_build_file_generation/gazelle_python_with_types.yaml b/examples/bzlmod_build_file_generation/gazelle_python_with_types.yaml new file mode 100644 index 0000000000..b6b0687ea4 --- /dev/null +++ b/examples/bzlmod_build_file_generation/gazelle_python_with_types.yaml @@ -0,0 +1,42 @@ +# GENERATED FILE - DO NOT EDIT! +# +# To update this file, run: +# bazel run //:gazelle_python_manifest_with_types.update + +manifest: + modules_mapping: + S3: s3cmd + asgiref: asgiref + astroid: astroid + certifi: certifi + chardet: chardet + dateutil: python_dateutil + dill: dill + django: Django + django_stubs: django_stubs + django_stubs_ext: django_stubs_ext + idna: idna + isort: isort + lazy_object_proxy: lazy_object_proxy + magic: python_magic + mccabe: mccabe + pathspec: pathspec + pkg_resources: setuptools + platformdirs: platformdirs + pylint: pylint + requests: requests + setuptools: setuptools + six: six + sqlparse: sqlparse + tabulate: tabulate + tomli: tomli + tomlkit: tomlkit + types_pyyaml: types_pyyaml + types_tabulate: types_tabulate + typing_extensions: typing_extensions + urllib3: urllib3 + wrapt: wrapt + yaml: PyYAML + yamllint: yamllint + pip_repository: + name: pip diff --git a/examples/bzlmod_build_file_generation/requirements.in b/examples/bzlmod_build_file_generation/requirements.in index a709195442..fb3b45176c 100644 --- a/examples/bzlmod_build_file_generation/requirements.in +++ b/examples/bzlmod_build_file_generation/requirements.in @@ -2,5 +2,8 @@ requests~=2.25.1 s3cmd~=2.1.0 yamllint>=1.28.0 tabulate~=0.9.0 +types-tabulate pylint~=2.15.5 python-dateutil>=2.8.2 +django +django-stubs diff --git a/examples/bzlmod_build_file_generation/requirements_lock.txt b/examples/bzlmod_build_file_generation/requirements_lock.txt index 9d9ad9453e..cdcebb72f7 100644 --- a/examples/bzlmod_build_file_generation/requirements_lock.txt +++ b/examples/bzlmod_build_file_generation/requirements_lock.txt @@ -4,6 +4,12 @@ # # bazel run //:requirements.update # +asgiref==3.8.1 \ + --hash=sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47 \ + --hash=sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590 + # via + # django + # django-stubs astroid==2.12.13 \ --hash=sha256:10e0ad5f7b79c435179d0d0f0df69998c4eef4597534aae44910db060baeb907 \ --hash=sha256:1493fe8bd3dfd73dc35bd53c9d5b6e49ead98497c47b2307662556a5692d29d7 @@ -20,6 +26,21 @@ dill==0.3.6 \ --hash=sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0 \ --hash=sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373 # via pylint +django==4.2.16 \ + --hash=sha256:1ddc333a16fc139fd253035a1606bb24261951bbc3a6ca256717fa06cc41a898 \ + --hash=sha256:6f1616c2786c408ce86ab7e10f792b8f15742f7b7b7460243929cb371e7f1dad + # via + # -r requirements.in + # django-stubs + # django-stubs-ext +django-stubs==5.0.0 \ + --hash=sha256:084484cbe16a6d388e80ec687e46f529d67a232f3befaf55c936b3b476be289d \ + --hash=sha256:b8a792bee526d6cab31e197cb414ee7fa218abd931a50948c66a80b3a2548621 + # via -r requirements.in +django-stubs-ext==5.1.1 \ + --hash=sha256:3907f99e178c93323e2ce908aef8352adb8c047605161f8d9e5e7b4efb5a6a9c \ + --hash=sha256:db7364e4f50ae7e5360993dbd58a3a57ea4b2e7e5bab0fbd525ccdb3e7975d1c + # via django-stubs idna==2.10 \ --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 @@ -129,6 +150,10 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via python-dateutil +sqlparse==0.5.2 \ + --hash=sha256:9e37b35e16d1cc652a2545f0997c1deb23ea28fa1f3eefe609eee3063c3b105f \ + --hash=sha256:e99bc85c78160918c3e1d9230834ab8d80fc06c59d03f8db2618f65f65dda55e + # via django tabulate==0.9.0 \ --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \ --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f @@ -136,16 +161,29 @@ tabulate==0.9.0 \ tomli==2.0.1 \ --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f - # via pylint + # via + # django-stubs + # pylint tomlkit==0.11.6 \ --hash=sha256:07de26b0d8cfc18f871aec595fda24d95b08fef89d147caa861939f37230bf4b \ --hash=sha256:71b952e5721688937fb02cf9d354dbcf0785066149d2855e44531ebdd2b65d73 # via pylint +types-pyyaml==6.0.12.20240917 \ + --hash=sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570 \ + --hash=sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587 + # via django-stubs +types-tabulate==0.9.0.20240106 \ + --hash=sha256:0378b7b6fe0ccb4986299496d027a6d4c218298ecad67199bbd0e2d7e9d335a1 \ + --hash=sha256:c9b6db10dd7fcf55bd1712dd3537f86ddce72a08fd62bb1af4338c7096ce947e + # via -r requirements.in typing-extensions==4.4.0 \ --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via + # asgiref # astroid + # django-stubs + # django-stubs-ext # pylint urllib3==1.26.13 \ --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc \ diff --git a/examples/bzlmod_build_file_generation/requirements_windows.txt b/examples/bzlmod_build_file_generation/requirements_windows.txt index 5b31ff5541..e591c8dc80 100644 --- a/examples/bzlmod_build_file_generation/requirements_windows.txt +++ b/examples/bzlmod_build_file_generation/requirements_windows.txt @@ -4,6 +4,12 @@ # # bazel run //:requirements.update # +asgiref==3.8.1 \ + --hash=sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47 \ + --hash=sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590 + # via + # django + # django-stubs astroid==2.12.13 \ --hash=sha256:10e0ad5f7b79c435179d0d0f0df69998c4eef4597534aae44910db060baeb907 \ --hash=sha256:1493fe8bd3dfd73dc35bd53c9d5b6e49ead98497c47b2307662556a5692d29d7 @@ -24,6 +30,21 @@ dill==0.3.6 \ --hash=sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0 \ --hash=sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373 # via pylint +django==4.2.16 \ + --hash=sha256:1ddc333a16fc139fd253035a1606bb24261951bbc3a6ca256717fa06cc41a898 \ + --hash=sha256:6f1616c2786c408ce86ab7e10f792b8f15742f7b7b7460243929cb371e7f1dad + # via + # -r requirements.in + # django-stubs + # django-stubs-ext +django-stubs==5.1.1 \ + --hash=sha256:126d354bbdff4906c4e93e6361197f6fbfb6231c3df6def85a291dae6f9f577b \ + --hash=sha256:c4dc64260bd72e6d32b9e536e8dd0d9247922f0271f82d1d5132a18f24b388ac + # via -r requirements.in +django-stubs-ext==5.1.1 \ + --hash=sha256:3907f99e178c93323e2ce908aef8352adb8c047605161f8d9e5e7b4efb5a6a9c \ + --hash=sha256:db7364e4f50ae7e5360993dbd58a3a57ea4b2e7e5bab0fbd525ccdb3e7975d1c + # via django-stubs idna==2.10 \ --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 @@ -133,6 +154,10 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via python-dateutil +sqlparse==0.5.2 \ + --hash=sha256:9e37b35e16d1cc652a2545f0997c1deb23ea28fa1f3eefe609eee3063c3b105f \ + --hash=sha256:e99bc85c78160918c3e1d9230834ab8d80fc06c59d03f8db2618f65f65dda55e + # via django tabulate==0.9.0 \ --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \ --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f @@ -140,17 +165,34 @@ tabulate==0.9.0 \ tomli==2.0.1 \ --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f - # via pylint + # via + # django-stubs + # pylint tomlkit==0.11.6 \ --hash=sha256:07de26b0d8cfc18f871aec595fda24d95b08fef89d147caa861939f37230bf4b \ --hash=sha256:71b952e5721688937fb02cf9d354dbcf0785066149d2855e44531ebdd2b65d73 # via pylint -typing-extensions==4.4.0 \ - --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ - --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e +types-pyyaml==6.0.12.20240917 \ + --hash=sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570 \ + --hash=sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587 + # via django-stubs +types-tabulate==0.9.0.20240106 \ + --hash=sha256:0378b7b6fe0ccb4986299496d027a6d4c218298ecad67199bbd0e2d7e9d335a1 \ + --hash=sha256:c9b6db10dd7fcf55bd1712dd3537f86ddce72a08fd62bb1af4338c7096ce947e + # via -r requirements.in +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ + --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via + # asgiref # astroid + # django-stubs + # django-stubs-ext # pylint +tzdata==2024.2 \ + --hash=sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc \ + --hash=sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd + # via django urllib3==1.26.13 \ --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc \ --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8 @@ -162,23 +204,30 @@ wrapt==1.14.1 \ --hash=sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2 \ --hash=sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656 \ --hash=sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3 \ + --hash=sha256:2020f391008ef874c6d9e208b24f28e31bcb85ccff4f335f15a3251d222b92d9 \ --hash=sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff \ + --hash=sha256:240b1686f38ae665d1b15475966fe0472f78e71b1b4903c143a842659c8e4cb9 \ --hash=sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310 \ + --hash=sha256:26046cd03936ae745a502abf44dac702a5e6880b2b01c29aea8ddf3353b68224 \ --hash=sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a \ --hash=sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57 \ --hash=sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069 \ + --hash=sha256:2feecf86e1f7a86517cab34ae6c2f081fd2d0dac860cb0c0ded96d799d20b335 \ --hash=sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383 \ --hash=sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe \ + --hash=sha256:358fe87cc899c6bb0ddc185bf3dbfa4ba646f05b1b0b9b5a27c2cb92c2cea204 \ --hash=sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87 \ --hash=sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d \ --hash=sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b \ --hash=sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907 \ + --hash=sha256:49ef582b7a1152ae2766557f0550a9fcbf7bbd76f43fbdc94dd3bf07cc7168be \ --hash=sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f \ --hash=sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0 \ --hash=sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28 \ --hash=sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1 \ --hash=sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853 \ --hash=sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc \ + --hash=sha256:6447e9f3ba72f8e2b985a1da758767698efa72723d5b59accefd716e9e8272bf \ --hash=sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3 \ --hash=sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3 \ --hash=sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164 \ @@ -201,8 +250,10 @@ wrapt==1.14.1 \ --hash=sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4 \ --hash=sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d \ --hash=sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d \ + --hash=sha256:a9008dad07d71f68487c91e96579c8567c98ca4c3881b9b113bc7b33e9fd78b8 \ --hash=sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8 \ --hash=sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5 \ + --hash=sha256:acae32e13a4153809db37405f5eba5bac5fbe2e2ba61ab227926a22901051c0a \ --hash=sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471 \ --hash=sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00 \ --hash=sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68 \ @@ -217,6 +268,7 @@ wrapt==1.14.1 \ --hash=sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb \ --hash=sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b \ --hash=sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f \ + --hash=sha256:ecee4132c6cd2ce5308e21672015ddfed1ff975ad0ac8d27168ea82e71413f55 \ --hash=sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462 \ --hash=sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015 \ --hash=sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel index d216ad5dc1..0a553831c3 100644 --- a/gazelle/MODULE.bazel +++ b/gazelle/MODULE.bazel @@ -34,3 +34,14 @@ use_repo( python_stdlib_list, "python_stdlib_list", ) + +internal_dev_deps = use_extension( + "//:internal_dev_deps.bzl", + "internal_dev_deps_extension", + dev_dependency = True, +) +use_repo( + internal_dev_deps, + "django-types", + "pytest", +) diff --git a/gazelle/README.md b/gazelle/README.md index c0494d141b..55c9cc9bff 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -119,6 +119,16 @@ gazelle_python_manifest( # the integrity field is not added to the manifest which can help avoid # merge conflicts in large repos. requirements = "//:requirements_lock.txt", + # include_stub_packages: bool (default: False) + # If set to True, this flag automatically includes any corresponding type stub packages + # for the third-party libraries that are present and used. For example, if you have + # `boto3` as a dependency, and this flag is enabled, the corresponding `boto3-stubs` + # package will be automatically included in the BUILD file. + # + # Enabling this feature helps ensure that type hints and stubs are readily available + # for tools like type checkers and IDEs, improving the development experience and + # reducing manual overhead in managing separate stub packages. + include_stub_packages = True ) ``` diff --git a/gazelle/WORKSPACE b/gazelle/WORKSPACE index d9f0645071..14a124d5f2 100644 --- a/gazelle/WORKSPACE +++ b/gazelle/WORKSPACE @@ -38,6 +38,10 @@ load("@rules_python//python:repositories.bzl", "py_repositories") py_repositories() +load("//:internal_dev_deps.bzl", "internal_dev_deps") + +internal_dev_deps() + load("//:deps.bzl", _py_gazelle_deps = "gazelle_deps") # gazelle:repository_macro deps.bzl%go_deps diff --git a/gazelle/internal_dev_deps.bzl b/gazelle/internal_dev_deps.bzl new file mode 100644 index 0000000000..f05f5fbb88 --- /dev/null +++ b/gazelle/internal_dev_deps.bzl @@ -0,0 +1,47 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module extension for internal dev_dependency=True setup.""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + +def internal_dev_deps(): + """This extension creates internal rules_python_gazelle dev dependencies.""" + http_file( + name = "pytest", + downloaded_file_path = "pytest-8.3.3-py3-none-any.whl", + sha256 = "a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2", + urls = [ + "https://files.pythonhosted.org/packages/6b/77/7440a06a8ead44c7757a64362dd22df5760f9b12dc5f11b6188cd2fc27a0/pytest-8.3.3-py3-none-any.whl", + ], + ) + http_file( + name = "django-types", + downloaded_file_path = "django_types-0.19.1-py3-none-any.whl", + sha256 = "b3f529de17f6374d41ca67232aa01330c531bbbaa3ac4097896f31ac33c96c30", + urls = [ + "https://files.pythonhosted.org/packages/25/cb/d088c67245a9d5759a08dbafb47e040ee436e06ee433a3cdc7f3233b3313/django_types-0.19.1-py3-none-any.whl", + ], + ) + +def _internal_dev_deps_impl(mctx): + _ = mctx # @unused + + # This wheel is purely here to validate the wheel extraction code. It's not + # intended for anything else. + internal_dev_deps() + +internal_dev_deps_extension = module_extension( + implementation = _internal_dev_deps_impl, + doc = "This extension creates internal rules_python_gazelle dev dependencies.", +) diff --git a/gazelle/modules_mapping/BUILD.bazel b/gazelle/modules_mapping/BUILD.bazel index d78b1fb51f..3a9a8a47f3 100644 --- a/gazelle/modules_mapping/BUILD.bazel +++ b/gazelle/modules_mapping/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_python//python:defs.bzl", "py_binary") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@rules_python//python:defs.bzl", "py_binary", "py_test") # gazelle:exclude *.py @@ -8,6 +9,30 @@ py_binary( visibility = ["//visibility:public"], ) +copy_file( + name = "pytest_wheel", + src = "@pytest//file", + out = "pytest-8.3.3-py3-none-any.whl", +) + +copy_file( + name = "django_types_wheel", + src = "@django-types//file", + out = "django_types-0.19.1-py3-none-any.whl", +) + +py_test( + name = "test_generator", + srcs = ["test_generator.py"], + data = [ + "django_types_wheel", + "pytest_wheel", + ], + imports = ["."], + main = "test_generator.py", + deps = [":generator"], +) + filegroup( name = "distribution", srcs = glob(["**"]), diff --git a/gazelle/modules_mapping/def.bzl b/gazelle/modules_mapping/def.bzl index 4da6267493..eb17f5c3d4 100644 --- a/gazelle/modules_mapping/def.bzl +++ b/gazelle/modules_mapping/def.bzl @@ -31,6 +31,8 @@ def _modules_mapping_impl(ctx): transitive = [dep[DefaultInfo].files for dep in ctx.attr.wheels] + [dep[DefaultInfo].data_runfiles.files for dep in ctx.attr.wheels], ) args.add("--output_file", modules_mapping.path) + if ctx.attr.include_stub_packages: + args.add("--include_stub_packages") args.add_all("--exclude_patterns", ctx.attr.exclude_patterns) args.add_all("--wheels", [whl.path for whl in all_wheels.to_list()]) ctx.actions.run( @@ -50,6 +52,11 @@ modules_mapping = rule( doc = "A set of regex patterns to match against each calculated module path. By default, exclude the modules starting with underscores.", mandatory = False, ), + "include_stub_packages": attr.bool( + default = False, + doc = "Whether to include stub packages in the mapping.", + mandatory = False, + ), "modules_mapping_name": attr.string( default = "modules_mapping.json", doc = "The name for the output JSON file.", diff --git a/gazelle/modules_mapping/generator.py b/gazelle/modules_mapping/generator.py index bbd579d416..99f565e8d6 100644 --- a/gazelle/modules_mapping/generator.py +++ b/gazelle/modules_mapping/generator.py @@ -25,16 +25,25 @@ class Generator: stderr = None output_file = None excluded_patterns = None - mapping = {} - def __init__(self, stderr, output_file, excluded_patterns): + def __init__(self, stderr, output_file, excluded_patterns, include_stub_packages): self.stderr = stderr self.output_file = output_file self.excluded_patterns = [re.compile(pattern) for pattern in excluded_patterns] + self.include_stub_packages = include_stub_packages + self.mapping = {} # dig_wheel analyses the wheel .whl file determining the modules it provides # by looking at the directory structure. def dig_wheel(self, whl): + # Skip stubs and types wheels. + wheel_name = get_wheel_name(whl) + if self.include_stub_packages and ( + wheel_name.endswith(("_stubs", "_types")) + or wheel_name.startswith(("types_", "stubs_")) + ): + self.mapping[wheel_name.lower()] = wheel_name.lower() + return with zipfile.ZipFile(whl, "r") as zip_file: for path in zip_file.namelist(): if is_metadata(path): @@ -145,8 +154,11 @@ def data_has_purelib_or_platlib(path): description="Generates the modules mapping used by the Gazelle manifest.", ) parser.add_argument("--output_file", type=str) + parser.add_argument("--include_stub_packages", action="store_true") parser.add_argument("--exclude_patterns", nargs="+", default=[]) parser.add_argument("--wheels", nargs="+", default=[]) args = parser.parse_args() - generator = Generator(sys.stderr, args.output_file, args.exclude_patterns) + generator = Generator( + sys.stderr, args.output_file, args.exclude_patterns, args.include_stub_packages + ) exit(generator.run(args.wheels)) diff --git a/gazelle/modules_mapping/test_generator.py b/gazelle/modules_mapping/test_generator.py new file mode 100644 index 0000000000..d6d2f19039 --- /dev/null +++ b/gazelle/modules_mapping/test_generator.py @@ -0,0 +1,44 @@ +import pathlib +import unittest + +from generator import Generator + + +class GeneratorTest(unittest.TestCase): + def test_generator(self): + whl = pathlib.Path(__file__).parent / "pytest-8.3.3-py3-none-any.whl" + gen = Generator(None, None, {}, False) + gen.dig_wheel(whl) + self.assertLessEqual( + { + "_pytest": "pytest", + "_pytest.__init__": "pytest", + "_pytest._argcomplete": "pytest", + "_pytest.config.argparsing": "pytest", + }.items(), + gen.mapping.items(), + ) + + def test_stub_generator(self): + whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl" + gen = Generator(None, None, {}, True) + gen.dig_wheel(whl) + self.assertLessEqual( + { + "django_types": "django_types", + }.items(), + gen.mapping.items(), + ) + + def test_stub_excluded(self): + whl = pathlib.Path(__file__).parent / "django_types-0.19.1-py3-none-any.whl" + gen = Generator(None, None, {}, False) + gen.dig_wheel(whl) + self.assertEqual( + {}.items(), + gen.mapping.items(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/gazelle/python/resolve.go b/gazelle/python/resolve.go index a7b716a829..88a688fa85 100644 --- a/gazelle/python/resolve.go +++ b/gazelle/python/resolve.go @@ -189,8 +189,20 @@ func (py *Resolver) Resolve( continue MODULES_LOOP } } else { - if dep, ok := cfg.FindThirdPartyDependency(moduleName); ok { + if dep, distributionName, ok := cfg.FindThirdPartyDependency(moduleName); ok { deps.Add(dep) + // Add the type and stub dependencies if they exist. + modules := []string{ + fmt.Sprintf("%s_stubs", strings.ToLower(distributionName)), + fmt.Sprintf("%s_types", strings.ToLower(distributionName)), + fmt.Sprintf("types_%s", strings.ToLower(distributionName)), + fmt.Sprintf("stubs_%s", strings.ToLower(distributionName)), + } + for _, module := range modules { + if dep, _, ok := cfg.FindThirdPartyDependency(module); ok { + deps.Add(dep) + } + } if explainDependency == dep { log.Printf("Explaining dependency (%s): "+ "in the target %q, the file %q imports %q at line %d, "+ diff --git a/gazelle/python/testdata/add_type_stub_packages/BUILD.in b/gazelle/python/testdata/add_type_stub_packages/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/add_type_stub_packages/BUILD.out b/gazelle/python/testdata/add_type_stub_packages/BUILD.out new file mode 100644 index 0000000000..d30540f61a --- /dev/null +++ b/gazelle/python/testdata/add_type_stub_packages/BUILD.out @@ -0,0 +1,14 @@ +load("@rules_python//python:defs.bzl", "py_binary") + +py_binary( + name = "add_type_stub_packages_bin", + srcs = ["__main__.py"], + main = "__main__.py", + visibility = ["//:__subpackages__"], + deps = [ + "@gazelle_python_test//boto3", + "@gazelle_python_test//boto3_stubs", + "@gazelle_python_test//django", + "@gazelle_python_test//django_types", + ], +) diff --git a/gazelle/python/testdata/add_type_stub_packages/README.md b/gazelle/python/testdata/add_type_stub_packages/README.md new file mode 100644 index 0000000000..c42e76f8be --- /dev/null +++ b/gazelle/python/testdata/add_type_stub_packages/README.md @@ -0,0 +1,4 @@ +# Add stubs to `deps` of `py_library` target + +This test case asserts that +* if a package has the corresponding stub available, it is added to the `deps` of the `py_library` target. diff --git a/gazelle/python/testdata/add_type_stub_packages/WORKSPACE b/gazelle/python/testdata/add_type_stub_packages/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/add_type_stub_packages/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/add_type_stub_packages/__main__.py b/gazelle/python/testdata/add_type_stub_packages/__main__.py new file mode 100644 index 0000000000..96384cfb13 --- /dev/null +++ b/gazelle/python/testdata/add_type_stub_packages/__main__.py @@ -0,0 +1,16 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import boto3 +import django diff --git a/gazelle/python/testdata/add_type_stub_packages/gazelle_python.yaml b/gazelle/python/testdata/add_type_stub_packages/gazelle_python.yaml new file mode 100644 index 0000000000..f498d07f2f --- /dev/null +++ b/gazelle/python/testdata/add_type_stub_packages/gazelle_python.yaml @@ -0,0 +1,22 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +manifest: + modules_mapping: + boto3: boto3 + boto3_stubs: boto3_stubs + django_types: django_types + django: Django + + pip_deps_repository_name: gazelle_python_test diff --git a/gazelle/python/testdata/add_type_stub_packages/test.yaml b/gazelle/python/testdata/add_type_stub_packages/test.yaml new file mode 100644 index 0000000000..fcea77710f --- /dev/null +++ b/gazelle/python/testdata/add_type_stub_packages/test.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index a24a90efeb..55121381dd 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -278,7 +278,7 @@ func (c *Config) SetGazelleManifest(gazelleManifest *manifest.Manifest) { // FindThirdPartyDependency scans the gazelle manifests for the current config // and the parent configs up to the root finding if it can resolve the module // name. -func (c *Config) FindThirdPartyDependency(modName string) (string, bool) { +func (c *Config) FindThirdPartyDependency(modName string) (string, string, bool) { for currentCfg := c; currentCfg != nil; currentCfg = currentCfg.parent { if currentCfg.gazelleManifest != nil { gazelleManifest := currentCfg.gazelleManifest @@ -291,11 +291,11 @@ func (c *Config) FindThirdPartyDependency(modName string) (string, bool) { } lbl := currentCfg.FormatThirdPartyDependency(distributionRepositoryName, distributionName) - return lbl.String(), true + return lbl.String(), distributionName, true } } } - return "", false + return "", "", false } // AddIgnoreFile adds a file to the list of ignored files for a given package.