Gazelle extension for Python (#514)

Gazelle plugin

* Add new example to --deleted_packages

* Update examples/build_file_generation/BUILD

Co-authored-by: Jonathon Belotti <[email protected]>

* fix: gazelle:exclude on coarse-grained

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* fix: comment on Kinds()

Co-authored-by: Jonathon Belotti <[email protected]>

* owner: f0rmiga

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* fix: build and setuptools pinned versions

With the recent change in pypa/setuptools#2769, some wheels started to
fail build immediately with an unpinned setuptools in isolation mode.

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* refactor: use local_repository in examples

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* bump: examples Bazel version

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* fix: add missing .gitignore to example

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* refactor: remove python_coarse_grained_generation

Also add the python_generation_mode directive.

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* fix: gazelle spam from org_golang_x_tools

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* revert: example .bazelversion

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* fix: simplify std_modules.py

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* feat: test py_library without __init__.py

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* feat: manifest generation tag manual

Signed-off-by: Thulio Ferraz Assis <[email protected]>

* fix: check std modules last

Performing the check last is more correct and yields better performance,
noticeable on large repositories.

Signed-off-by: Thulio Ferraz Assis <[email protected]>

Co-authored-by: Alex Eagle <[email protected]>
Co-authored-by: Jonathon Belotti <[email protected]>

Author: 3 people

.bazelrc

@@ -3,7 +3,15 @@
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
-build --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements
-query --deleted_packages=examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements
+build --deleted_packages=examples/build_file_generation,examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements
+query --deleted_packages=examples/build_file_generation,examples/legacy_pip_import/boto,examples/legacy_pip_import/extras,examples/legacy_pip_import/helloworld,examples/pip_install,examples/pip_parse,examples/py_import,examples/relative_requirements
 
 test --test_output=errors
+
+# Do NOT implicitly create empty __init__.py files in the runfiles tree.
+# By default, these are created in every directory containing Python source code
+# or shared libraries, and every parent directory of those directories,
+# excluding the repo root directory. With this flag set, we are responsible for
+# creating (possibly empty) __init__.py files and adding them to the srcs of
+# Python targets as required.
+build --incompatible_default_to_explicit_init_py

.github/CODEOWNERS

@@ -10,6 +10,9 @@
 /python/whl.bzl @thundergolfer @andyscott
 /python/requirements.txt @thundergolfer @andyscott
 
+# Directory containing the Gazelle extension and Go code.
+/gazelle/ @f0rmiga
+
 # The proposals dir corresponds to the Bazel proposals process, documented
 # here: https://bazel.build/designs/index.html
 /proposals/ @brandjon @lberki

.gitignore

@@ -41,3 +41,8 @@
 # vim swap files
 *.swp
 *.swo
+
+# Go/Gazelle files
+# These otherwise match patterns above
+!go.mod
+!BUILD.out

BUILD

@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+load("@bazel_gazelle//:def.bzl", "gazelle")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
@@ -51,3 +53,19 @@ filegroup(
     ],
     visibility = ["//visibility:public"],
 )
+
+# Gazelle configuration options.
+# See https://github.com/bazelbuild/bazel-gazelle#running-gazelle-with-bazel
+# gazelle:prefix github.com/bazelbuild/rules_python
+# gazelle:exclude bazel-out
+gazelle(name = "gazelle")
+
+gazelle(
+    name = "update_go_deps",
+    args = [
+        "-from_file=go.mod",
+        "-to_macro=gazelle/deps.bzl%gazelle_deps",
+        "-prune",
+    ],
+    command = "update-repos",
+)

WORKSPACE

@@ -24,3 +24,8 @@ rules_python_internal_deps()
 load("//:internal_setup.bzl", "rules_python_internal_setup")
 
 rules_python_internal_setup()
+
+load("//gazelle:deps.bzl", "gazelle_deps")
+
+# gazelle:repository_macro gazelle/deps.bzl%gazelle_deps
+gazelle_deps()

examples/build_file_generation/.bazelversion

@@ -0,0 +1 @@
+4.0.0

examples/build_file_generation/.gitignore

@@ -0,0 +1 @@
+bazel-*

examples/build_file_generation/BUILD

@@ -0,0 +1,37 @@
+load("@bazel_gazelle//:def.bzl", "gazelle")
+load("@rules_python//gazelle:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
+load("@rules_python//gazelle/manifest:defs.bzl", "gazelle_python_manifest")
+load("@rules_python//python:defs.bzl", "py_library")
+
+# Gazelle python extension needs a manifest file mapping from
+# an import to the installed package that provides it.
+# This macro produces two targets:
+# - //:gazelle_python_manifest.update can be used with `bazel run`
+#   to recalculate the manifest
+# - //:gazelle_python_manifest.test is a test target ensuring that
+#   the manifest doesn't need to be updated
+gazelle_python_manifest(
+    name = "gazelle_python_manifest",
+    modules_mapping = "@modules_map//:modules_mapping.json",
+    pip_deps_repository_name = "pip",
+    requirements = "//:requirements_lock.txt",
+)
+
+# Our gazelle target points to the python gazelle binary.
+# This is the simple case where we only need one language supported.
+# If you also had proto, go, or other gazelle-supported languages,
+# you would also need a gazelle_binary rule.
+# See https://github.com/bazelbuild/bazel-gazelle/blob/master/extend.rst#example
+gazelle(
+    name = "gazelle",
+    data = GAZELLE_PYTHON_RUNTIME_DEPS,
+    gazelle = "@rules_python//gazelle:gazelle_python_binary",
+)
+
+# This rule is auto-generated and managed by Gazelle,
+# because it found the __init__.py file in this folder.
+py_library(
+    name = "build_file_generation",
+    srcs = ["__init__.py"],
+    visibility = ["//:__subpackages__"],
+)

examples/build_file_generation/README.md

@@ -0,0 +1,20 @@
+# Build file generation with Gazelle
+
+This example shows a project that has Gazelle setup with the rules_python
+extension, so that targets like `py_library` and `py_binary` can be
+automatically created just by running
+
+```sh
+$ bazel run //:gazelle
+```
+
+As a demo, try creating a `__main__.py` file in this directory, then
+re-run that gazelle command. You'll see that a `py_binary` target
+is created in the `BUILD` file.
+
+Or, try importing the `requests` library in `__init__.py`.
+You'll see that `deps = ["@pip//pypi__requests"]` is automatically
+added to the `py_library` target in the `BUILD` file.
+
+For more information on the behavior of the rules_python gazelle extension,
+see the README.md file in the /gazelle folder.

examples/build_file_generation/WORKSPACE

@@ -0,0 +1,73 @@
+workspace(name = "build_file_generation_example")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+######################################################################
+# We need rules_go and bazel_gazelle, to build the gazelle plugin from source.
+# Setup instructions for this section are at
+# https://github.com/bazelbuild/bazel-gazelle#running-gazelle-with-bazel
+
+# Note, you could omit the rules_go dependency, if you have some way to statically
+# compile the gazelle binary for your workspace and distribute it to users on all
+# needed platforms.
+http_archive(
+    name = "io_bazel_rules_go",
+    sha256 = "69de5c704a05ff37862f7e0f5534d4f479418afc21806c887db544a316f3cb6b",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.27.0/rules_go-v0.27.0.tar.gz",
+    ],
+)
+
+# NB: bazel-gazelle version must be after 18 August 2021
+# to include https://github.com/bazelbuild/bazel-gazelle/commit/2834ea4
+http_archive(
+    name = "bazel_gazelle",
+    sha256 = "0bb8056ab9ed4cbcab5b74348d8530c0e0b939987b0cfe36c1ab53d35a99e4de",
+    strip_prefix = "bazel-gazelle-2834ea44b3ec6371c924baaf28704730ec9d4559",
+    urls = [
+        # No release since March, and we need subsequent fixes
+        "https://github.com/bazelbuild/bazel-gazelle/archive/2834ea44b3ec6371c924baaf28704730ec9d4559.zip",
+    ],
+)
+
+load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+
+go_rules_dependencies()
+
+go_register_toolchains(version = "1.16.5")
+
+gazelle_dependencies()
+
+######################################################################
+# Remaining setup is for rules_python
+
+local_repository(
+    name = "rules_python",
+    path = "../..",
+)
+
+load("@rules_python//python:pip.bzl", "pip_install")
+
+pip_install(
+    # Uses the default repository name "pip"
+    requirements = "//:requirements_lock.txt",
+)
+
+# The rules_python gazelle extension has some third-party go dependencies
+# which we need to fetch in order to compile it.
+load("@rules_python//gazelle:deps.bzl", _py_gazelle_deps = "gazelle_deps")
+
+_py_gazelle_deps()
+
+load("@rules_python//gazelle/modules_mapping:def.bzl", "modules_mapping")
+
+# This repository rule fetches the metadata for python packages we
+# depend on. That data is required for the gazelle_python_manifest
+# rule to update our manifest file.
+# To see what this rule does, try `bazel run @modules_map//:print`
+modules_mapping(
+    name = "modules_map",
+    requirements = "//:requirements_lock.txt",
+)