feat(gazelle): Gazelle plugin generates py_proto_library

CHANGELOG.md

@@ -105,6 +105,8 @@ END_UNRELEASED_TEMPLATE
     * 3.12.11
     * 3.13.5
     * 3.14.0b3
+* (gazelle) New directive `gazelle:python_generate_proto`; when `true`,
+  Gazelle generates `py_proto_library` rules for `proto_library`. `false` by default.
 
 {#v0-0-0-removed}
 ### Removed

examples/bzlmod/py_proto_library/BUILD.bazel

@@ -6,15 +6,15 @@ py_test(
     srcs = ["test.py"],
     main = "test.py",
     deps = [
-        "//py_proto_library/example.com/proto:pricetag_proto_py_pb2",
+        "//py_proto_library/example.com/proto:pricetag_py_pb2",
     ],
 )
 
 py_test(
     name = "message_test",
     srcs = ["message_test.py"],
     deps = [
-        "//py_proto_library/example.com/another_proto:message_proto_py_pb2",
+        "//py_proto_library/example.com/another_proto:message_py_pb2",
     ],
 )
 

examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel

@@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
 load("@rules_python//python:proto.bzl", "py_proto_library")
 
 py_proto_library(
-    name = "message_proto_py_pb2",
+    name = "message_py_pb2",
     visibility = ["//visibility:public"],
     deps = [":message_proto"],
 )

examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel

@@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
 load("@rules_python//python:proto.bzl", "py_proto_library")
 
 py_proto_library(
-    name = "pricetag_proto_py_pb2",
+    name = "pricetag_py_pb2",
     visibility = ["//visibility:public"],
     deps = [":pricetag_proto"],
 )

examples/py_proto_library/BUILD.bazel

@@ -5,14 +5,14 @@ py_test(
     srcs = ["test.py"],
     main = "test.py",
     deps = [
-        "//example.com/proto:pricetag_proto_py_pb2",
+        "//example.com/proto:pricetag_py_pb2",
     ],
 )
 
 py_test(
     name = "message_test",
     srcs = ["message_test.py"],
     deps = [
-        "//example.com/another_proto:message_proto_py_pb2",
+        "//example.com/another_proto:message_py_pb2",
     ],
 )

examples/py_proto_library/example.com/another_proto/BUILD.bazel

@@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
 load("@rules_python//python:proto.bzl", "py_proto_library")
 
 py_proto_library(
-    name = "message_proto_py_pb2",
+    name = "message_py_pb2",
     visibility = ["//visibility:public"],
     deps = [":message_proto"],
 )

examples/py_proto_library/example.com/proto/BUILD.bazel

@@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
 load("@rules_python//python:proto.bzl", "py_proto_library")
 
 py_proto_library(
-    name = "pricetag_proto_py_pb2",
+    name = "pricetag_py_pb2",
     visibility = ["//visibility:public"],
     deps = [":pricetag_proto"],
 )

gazelle/README.md

@@ -224,6 +224,8 @@ Python-specific directives are as follows:
 | Controls whether Gazelle resolves dependencies for import statements that use paths relative to the current package. Can be "true" or "false".|
 | `# gazelle:python_generate_pyi_deps`                                                                                                                                                                                                                                                            | `false` |
 | Controls whether to generate a separate `pyi_deps` attribute for type-checking dependencies or merge them into the regular `deps` attribute. When `false` (default), type-checking dependencies are merged into `deps` for backward compatibility. When `true`, generates separate `pyi_deps`. Imports in blocks with the format `if typing.TYPE_CHECKING:`/`if TYPE_CHECKING:` and type-only stub packages (eg. boto3-stubs) are recognized as type-checking dependencies. |
+| [`# gazelle:python_generate_proto`](#directive-python_generate_proto)                                                                                                                                                                                                                                                                | `false` |
+| Controls whether to generate a `py_proto_library` for each `proto_library` in the package. By default we load this rule from the `@protobuf` repository; use `gazelle:map_kind` if you need to load this from somewhere else. |
 
 #### Directive: `python_root`:
 
@@ -484,6 +486,41 @@ def py_test(name, main=None, **kwargs):
 )
 ```
 
+#### Directive: `python_generate_proto`:
+
+When `# gazelle:python_generate_proto true`, Gazelle will generate one
+`py_proto_library` for each `proto_library`, generating Python clients for
+protobuf in each package. By default this is turned off. Gazelle will also
+generate a load statement for the `py_proto_library` - attempting to detect
+the configured name for the `@protobuf` / `@com_google_protobuf` repo in your
+`MODULE.bazel`, and otherwise falling back to `@com_google_protobuf` for
+compatibility with `WORKSPACE`.
+
+For example, in a package with `# gazelle:python_generate_proto true` and a
+`foo.proto`, if you have both the proto extension and the Python extension
+loaded into Gazelle, you'll get something like:
+
+```starlark
+load("@protobuf//bazel:py_proto_library.bzl", "py_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+# gazelle:python_generate_proto true
+
+proto_library(
+    name = "foo_proto",
+    srcs = ["foo.proto"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_proto_library(
+    name = "foo_py_pb2",
+    visibility = ["//:__subpackages__"],
+    deps = [":foo_proto"],
+)
+```
+
+When `false`, Gazelle will ignore any `py_proto_library`, including previously-generated or hand-created rules.
+
 ### Annotations
 
 *Annotations* refer to comments found _within Python files_ that configure how

gazelle/python/BUILD.bazel

@@ -34,6 +34,7 @@ go_library(
         "@bazel_gazelle//config:go_default_library",
         "@bazel_gazelle//label:go_default_library",
         "@bazel_gazelle//language:go_default_library",
+        "@bazel_gazelle//language/proto:go_default_library",
         "@bazel_gazelle//repo:go_default_library",
         "@bazel_gazelle//resolve:go_default_library",
         "@bazel_gazelle//rule:go_default_library",
@@ -91,7 +92,10 @@ gazelle_test(
 
 gazelle_binary(
     name = "gazelle_binary",
-    languages = [":python"],
+    languages = [
+        "@bazel_gazelle//language/proto",
+        ":python",
+    ],
     visibility = ["//visibility:public"],
 )
 

gazelle/python/configure.go

@@ -70,6 +70,7 @@ func (py *Configurer) KnownDirectives() []string {
 		pythonconfig.LabelNormalization,
 		pythonconfig.GeneratePyiDeps,
 		pythonconfig.ExperimentalAllowRelativeImports,
+		pythonconfig.GenerateProto,
 	}
 }
 
@@ -237,6 +238,12 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
 				log.Fatal(err)
 			}
 			config.SetGeneratePyiDeps(v)
+		case pythonconfig.GenerateProto:
+			v, err := strconv.ParseBool(strings.TrimSpace(d.Value))
+			if err != nil {
+				log.Fatal(err)
+			}
+			config.SetGenerateProto(v)
 		}
 	}
 

gazelle/python/generate.go

@@ -226,6 +226,10 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 	var result language.GenerateResult
 	result.Gen = make([]*rule.Rule, 0)
 
+	if cfg.GenerateProto() {
+		generateProtoLibraries(args, pythonProjectRoot, visibility, &result)
+	}
+
 	collisionErrors := singlylinkedlist.New()
 
 	appendPyLibrary := func(srcs *treeset.Set, pyLibraryTargetName string) {
@@ -551,3 +555,51 @@ func ensureNoCollision(file *rule.File, targetName, kind string) error {
 	}
 	return nil
 }
+
+func generateProtoLibraries(args language.GenerateArgs, pythonProjectRoot string, visibility []string, res *language.GenerateResult) {
+	// First, enumerate all the proto_library in this package.
+	var protoRuleNames []string
+	for _, r := range args.OtherGen {
+		if r.Kind() != "proto_library" {
+			continue
+		}
+		protoRuleNames = append(protoRuleNames, r.Name())
+	}
+	sort.Strings(protoRuleNames)
+
+	// Next, enumerate all the pre-existing py_proto_library in this package, so we can delete unnecessary rules later.
+	pyProtoRules := map[string]bool{}
+	if args.File != nil {
+		for _, r := range args.File.Rules {
+			if r.Kind() == "py_proto_library" {
+				pyProtoRules[r.Name()] = false
+			}
+		}
+	}
+
+	emptySiblings := treeset.Set{}
+	// Generate a py_proto_library for each proto_library.
+	for _, protoRuleName := range protoRuleNames {
+		pyProtoLibraryName := strings.TrimSuffix(protoRuleName, "_proto") + "_py_pb2"
+		pyProtoLibrary := newTargetBuilder(pyProtoLibraryKind, pyProtoLibraryName, pythonProjectRoot, args.Rel, &emptySiblings).
+			addVisibility(visibility).
+			addResolvedDependency(":" + protoRuleName).
+			generateImportsAttribute().build()
+
+		res.Gen = append(res.Gen, pyProtoLibrary)
+		res.Imports = append(res.Imports, pyProtoLibrary.PrivateAttr(config.GazelleImportsKey))
+		pyProtoRules[pyProtoLibrary.Name()] = true
+
+	}
+
+	// Finally, emit an empty rule for each pre-existing py_proto_library that we didn't already generate.
+	for ruleName, generated := range pyProtoRules {
+		if generated {
+			continue
+		}
+
+		emptyRule := newTargetBuilder(pyProtoLibraryKind, ruleName, pythonProjectRoot, args.Rel, &emptySiblings).build()
+		res.Empty = append(res.Empty, emptyRule)
+	}
+
+}

gazelle/python/kinds.go

@@ -15,13 +15,16 @@
 package python
 
 import (
+	"fmt"
+
 	"github.com/bazelbuild/bazel-gazelle/rule"
 )
 
 const (
-	pyBinaryKind  = "py_binary"
-	pyLibraryKind = "py_library"
-	pyTestKind    = "py_test"
+	pyBinaryKind       = "py_binary"
+	pyLibraryKind      = "py_library"
+	pyProtoLibraryKind = "py_proto_library"
+	pyTestKind         = "py_test"
 )
 
 // Kinds returns a map that maps rule names (kinds) and information on how to
@@ -32,7 +35,7 @@ func (*Python) Kinds() map[string]rule.KindInfo {
 
 var pyKinds = map[string]rule.KindInfo{
 	pyBinaryKind: {
-		MatchAny: false,
+		MatchAny:   false,
 		MatchAttrs: []string{"srcs"},
 		NonEmptyAttrs: map[string]bool{
 			"deps":    true,
@@ -45,7 +48,7 @@ var pyKinds = map[string]rule.KindInfo{
 			"srcs": true,
 		},
 		ResolveAttrs: map[string]bool{
-			"deps": true,
+			"deps":     true,
 			"pyi_deps": true,
 		},
 	},
@@ -62,10 +65,16 @@ var pyKinds = map[string]rule.KindInfo{
 			"srcs": true,
 		},
 		ResolveAttrs: map[string]bool{
-			"deps": true,
+			"deps":     true,
 			"pyi_deps": true,
 		},
 	},
+	pyProtoLibraryKind: {
+		NonEmptyAttrs: map[string]bool{
+			"deps": true,
+		},
+		ResolveAttrs: map[string]bool{"deps": true},
+	},
 	pyTestKind: {
 		MatchAny: false,
 		NonEmptyAttrs: map[string]bool{
@@ -79,26 +88,43 @@ var pyKinds = map[string]rule.KindInfo{
 			"srcs": true,
 		},
 		ResolveAttrs: map[string]bool{
-			"deps": true,
+			"deps":     true,
 			"pyi_deps": true,
 		},
 	},
 }
 
+func (py *Python) Loads() []rule.LoadInfo {
+	panic("ApparentLoads should be called instead")
+}
+
 // Loads returns .bzl files and symbols they define. Every rule generated by
 // GenerateRules, now or in the past, should be loadable from one of these
 // files.
-func (py *Python) Loads() []rule.LoadInfo {
-	return pyLoads
+func (py *Python) ApparentLoads(moduleToApparentName func(string) string) []rule.LoadInfo {
+	return apparentLoads(moduleToApparentName)
 }
 
-var pyLoads = []rule.LoadInfo{
-	{
-		Name: "@rules_python//python:defs.bzl",
-		Symbols: []string{
-			pyBinaryKind,
-			pyLibraryKind,
-			pyTestKind,
+func apparentLoads(moduleToApparentName func(string) string) []rule.LoadInfo {
+	protobuf := moduleToApparentName("protobuf")
+	if protobuf == "" {
+		protobuf = "com_google_protobuf"
+	}
+
+	return []rule.LoadInfo{
+		{
+			Name: "@rules_python//python:defs.bzl",
+			Symbols: []string{
+				pyBinaryKind,
+				pyLibraryKind,
+				pyTestKind,
+			},
 		},
-	},
+		{
+			Name: fmt.Sprintf("@%s//bazel:py_proto_library.bzl", protobuf),
+			Symbols: []string{
+				pyProtoLibraryKind,
+			},
+		},
+	}
 }

gazelle/python/testdata/directive_python_generate_proto/README.md

@@ -0,0 +1,9 @@
+# Directive: `python_generate_proto`
+
+This test case asserts that the `# gazelle:python_generate_proto` directive
+correctly:
+
+1.  Uses the default value when `python_generate_proto` is not set.
+2.  Generates (or not) `py_proto_library` when `python_generate_proto` is set, based on whether a proto is present.
+
+[gh-2994]: https://github.com/bazel-contrib/rules_python/issues/2994

gazelle/python/testdata/directive_python_generate_proto/WORKSPACE

@@ -0,0 +1 @@
+# This is a Bazel workspace for the Gazelle test data.

gazelle/python/testdata/directive_python_generate_proto/test.yaml

@@ -0,0 +1,3 @@
+---
+expect:
+  exit_code: 0

gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/BUILD.in

@@ -0,0 +1,9 @@
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+# python_generate_proto is not set, so py_proto_library is not generated.
+
+proto_library(
+    name = "foo_proto",
+    srcs = ["foo.proto"],
+    visibility = ["//:__subpackages__"],
+)

gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/BUILD.out

@@ -0,0 +1,9 @@
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+# python_generate_proto is not set, so py_proto_library is not generated.
+
+proto_library(
+    name = "foo_proto",
+    srcs = ["foo.proto"],
+    visibility = ["//:__subpackages__"],
+)

gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/foo.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+package foo;
+
+message Foo {
+    string bar = 1;
+}

gazelle/python/testdata/directive_python_generate_proto/test2_default_without_proto/BUILD.in

@@ -0,0 +1 @@
+# python_generate_proto is not set, so py_proto_library is not generated.

gazelle/python/testdata/directive_python_generate_proto/test2_default_without_proto/BUILD.out

@@ -0,0 +1 @@
+# python_generate_proto is not set, so py_proto_library is not generated.