fix(gazelle): ensure that gazelle helper modules are on PYTHONPATH (#1590)

Before this change there was a bug in how the parsing helpers were being
used in case we were using Python 3.11 toolchain, which is using a more
strict version of the entrypoint template. This change adds `imports =
["."]`
to ensure that the gazelle helper components are on PYTHONPATH and
updates
the non-bzlmod tests to run under 3.11.

We also:
* Change `.bazelrc` to use explicit `__init__.py` definition to avoid
  non-reproducible errors in the future.
* Add a dedicated `gazelle_binary` that uses `DEFAULT_LANGUAGES` *and*
  `//python`.

Fixes #1589

Author: aignas

CHANGELOG.md

@@ -17,9 +17,14 @@ A brief description of the categories of changes:
 * Particular sub-systems are identified using parentheses, e.g. `(bzlmod)` or
   `(docs)`.
 
-## Unreleased
+## [0.27.1] - 2023-12-05
 
-[0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
+[0.27.1]: https://github.com/bazelbuild/rules_python/releases/tag/0.27.1
+
+### Fixed
+
+* (gazelle) The gazelle plugin helper was not working with Python toolchains 3.11
+  and above due to a bug in the helper components not being on `PYTHONPATH`.
 
 ## [0.27.0] - 2023-11-16
 

gazelle/.bazelrc

@@ -11,3 +11,11 @@ build --incompatible_default_to_explicit_init_py
 # Windows makes use of runfiles for some rules
 build --enable_runfiles
 startup --windows_enable_symlinks
+
+# Do NOT implicitly create empty __init__.py files in the runfiles tree.
+# By default, these are created in every directory containing Python source code
+# or shared libraries, and every parent directory of those directories,
+# excluding the repo root directory. With this flag set, we are responsible for
+# creating (possibly empty) __init__.py files and adding them to the srcs of
+# Python targets as required.
+build --incompatible_default_to_explicit_init_py

gazelle/BUILD.bazel

@@ -1,10 +1,18 @@
-load("@bazel_gazelle//:def.bzl", "gazelle")
+load("@bazel_gazelle//:def.bzl", "DEFAULT_LANGUAGES", "gazelle", "gazelle_binary")
 
 # Gazelle configuration options.
 # See https://github.com/bazelbuild/bazel-gazelle#running-gazelle-with-bazel
 # gazelle:prefix github.com/bazelbuild/rules_python/gazelle
 # gazelle:exclude bazel-out
-gazelle(name = "gazelle")
+gazelle(
+    name = "gazelle",
+    gazelle = ":gazelle_binary",
+)
+
+gazelle_binary(
+    name = "gazelle_binary",
+    languages = DEFAULT_LANGUAGES + ["//python"],
+)
 
 gazelle(
     name = "gazelle_update_repos",

gazelle/WORKSPACE

@@ -39,8 +39,8 @@ load("@rules_python//python:repositories.bzl", "py_repositories", "python_regist
 py_repositories()
 
 python_register_toolchains(
-    name = "python39",
-    python_version = "3.9",
+    name = "python_3_11",
+    python_version = "3.11",
 )
 
 load("//:deps.bzl", _py_gazelle_deps = "gazelle_deps")

gazelle/modules_mapping/BUILD.bazel

@@ -1,5 +1,7 @@
 load("@rules_python//python:defs.bzl", "py_binary")
 
+# gazelle:exclude *.py
+
 py_binary(
     name = "generator",
     srcs = ["generator.py"],

gazelle/python/BUILD.bazel

@@ -17,7 +17,15 @@ go_library(
         "std_modules.go",
         "target.go",
     ],
-    embedsrcs = [":helper.zip"],
+    # NOTE @aignas 2023-12-03: currently gazelle does not support embedding
+    # generated files, but helper.zip is generated by a build rule.
+    #
+    # You will get a benign error like when running gazelle locally:
+    # > 8 gazelle: .../rules_python/gazelle/python/lifecycle.go:26:3: pattern helper.zip: matched no files
+    #
+    # See following for more info:
+    # https://github.com/bazelbuild/bazel-gazelle/issues/1513
+    embedsrcs = [":helper.zip"],  # keep
     importpath = "github.com/bazelbuild/rules_python/gazelle/python",
     visibility = ["//visibility:public"],
     deps = [
@@ -44,6 +52,8 @@ py_binary(
         "parse.py",
         "std_modules.py",
     ],
+    # This is to make sure that the current directory is added to PYTHONPATH
+    imports = ["."],
     main = "__main__.py",
     visibility = ["//visibility:public"],
 )
@@ -54,6 +64,8 @@ filegroup(
     output_group = "python_zip_file",
 )
 
+# gazelle:exclude testdata/
+
 gazelle_test(
     name = "python_test",
     srcs = ["python_test.go"],

gazelle/python/__main__.py

@@ -16,9 +16,10 @@
 # STDIN receives parse requests, one per line. It outputs the parsed modules and
 # comments from all the files from each request.
 
+import sys
+
 import parse
 import std_modules
-import sys
 
 if __name__ == "__main__":
     if len(sys.argv) < 2:

gazelle/python/python_test.go

@@ -162,7 +162,7 @@ func testPath(t *testing.T, name string, files []bazel.RunfileEntry) {
 		cmd.Dir = workspaceRoot
 		helperScript, err := runfiles.Rlocation("rules_python_gazelle_plugin/python/helper")
 		if err != nil {
-			t.Fatalf("failed to initialize Python heler: %v", err)
+			t.Fatalf("failed to initialize Python helper: %v", err)
 		}
 		cmd.Env = append(os.Environ(), "GAZELLE_PYTHON_HELPER="+helperScript)
 		if err := cmd.Run(); err != nil {

gazelle/pythonconfig/BUILD.bazel

@@ -18,7 +18,7 @@ go_library(
 go_test(
     name = "pythonconfig_test",
     srcs = ["pythonconfig_test.go"],
-    deps = [":pythonconfig"],
+    embed = [":pythonconfig"],
 )
 
 filegroup(

gazelle/pythonconfig/pythonconfig_test.go

@@ -2,8 +2,6 @@ package pythonconfig
 
 import (
 	"testing"
-
-	"github.com/bazelbuild/rules_python/gazelle/pythonconfig"
 )
 
 func TestDistributionSanitizing(t *testing.T) {
@@ -19,7 +17,7 @@ func TestDistributionSanitizing(t *testing.T) {
 
 	for name, tc := range tests {
 		t.Run(name, func(t *testing.T) {
-			got := pythonconfig.SanitizeDistribution(tc.input)
+			got := SanitizeDistribution(tc.input)
 			if tc.want != got {
 				t.Fatalf("expected %q, got %q", tc.want, got)
 			}