feat: generate py_library per file (#1398)

fixes #1150
fixes #1323

you can no longer pre-define the name of the target by creating an empty
`py_library` (see 3c84655). I don't think this was being used and it's
straightforward to rename the generated per-project or per-package
target if you want

Author: raylu

CHANGELOG.md

@@ -35,6 +35,8 @@ A brief description of the categories of changes:
   the `py_binary` rule used to build it.
 * New Python versions available: `3.8.17`, `3.9.18`, `3.10.13`, `3.11.5` using
   https://github.com/indygreg/python-build-standalone/releases/tag/20230826.
+* (gazelle) New `# gazelle:python_generation_mode file` directive to support
+  generating one `py_library` per file.
 
 ### Removed
 

gazelle/README.md

@@ -189,9 +189,9 @@ Python-specific directives are as follows:
 | `# gazelle:python_validate_import_statements`| `true` |
 | Controls whether the Python import statements should be validated. Can be "true" or "false" | |
 | `# gazelle:python_generation_mode`| `package` |
-| Controls the target generation mode. Can be "package" or "project" | |
+| Controls the target generation mode. Can be "file", "package", or "project" | |
 | `# gazelle:python_library_naming_convention`| `$package_name$` |
-| Controls the `py_library` naming convention. It interpolates $package_name$ with the Bazel package name. E.g. if the Bazel package name is `foo`, setting this to `$package_name$_my_lib` would result in a generated target named `foo_my_lib`. | |
+| Controls the `py_library` naming convention. It interpolates \$package_name\$ with the Bazel package name. E.g. if the Bazel package name is `foo`, setting this to `$package_name$_my_lib` would result in a generated target named `foo_my_lib`. | |
 | `# gazelle:python_binary_naming_convention` | `$package_name$_bin` |
 | Controls the `py_binary` naming convention. Follows the same interpolation rules as `python_library_naming_convention`. | |
 | `# gazelle:python_test_naming_convention` | `$package_name$_test` |
@@ -206,11 +206,15 @@ Python source files are those ending in `.py` but not ending in `_test.py`.
 First, we look for the nearest ancestor BUILD file starting from the folder
 containing the Python source file.
 
-If there is no `py_library` in this BUILD file, one is created, using the
-package name as the target's name. This makes it the default target in the
-package.
+In package generation mode, if there is no `py_library` in this BUILD file, one
+is created using the package name as the target's name. This makes it the
+default target in the package. Next, all source files are collected into the
+`srcs` of the `py_library`.
 
-Next, all source files are collected into the `srcs` of the `py_library`.
+In project generation mode, all source files in subdirectories (that don't have
+BUILD files) are also collected.
+
+In file generation mode, each file is given its own target.
 
 Finally, the `import` statements in the source files are parsed, and
 dependencies are added to the `deps` attribute.

gazelle/python/configure.go

@@ -137,8 +137,13 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
 			switch pythonconfig.GenerationModeType(strings.TrimSpace(d.Value)) {
 			case pythonconfig.GenerationModePackage:
 				config.SetCoarseGrainedGeneration(false)
+				config.SetPerFileGeneration(false)
+			case pythonconfig.GenerationModeFile:
+				config.SetCoarseGrainedGeneration(false)
+				config.SetPerFileGeneration(true)
 			case pythonconfig.GenerationModeProject:
 				config.SetCoarseGrainedGeneration(true)
+				config.SetPerFileGeneration(false)
 			default:
 				err := fmt.Errorf("invalid value for directive %q: %s",
 					pythonconfig.GenerationMode, d.Value)

gazelle/python/generate.go

@@ -153,12 +153,17 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 				if entry.IsDir() {
 					// If we are visiting a directory, we determine if we should
 					// halt digging the tree based on a few criterias:
-					//   1. The directory has a BUILD or BUILD.bazel files. Then
+					//   1. We are using per-file generation.
+					//   2. The directory has a BUILD or BUILD.bazel files. Then
 					//       it doesn't matter at all what it has since it's a
 					//       separate Bazel package.
-					//   2. (only for fine-grained generation) The directory has
-					// 		 an __init__.py, __main__.py or __test__.py, meaning
-					// 		 a BUILD file will be generated.
+					//   3. (only for package generation) The directory has an
+					//       __init__.py, __main__.py or __test__.py, meaning a
+					//       BUILD file will be generated.
+					if cfg.PerFileGeneration() {
+						return fs.SkipDir
+					}
+
 					if isBazelPackage(path) {
 						boundaryPackages[path] = struct{}{}
 						return nil
@@ -213,15 +218,12 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 
 	collisionErrors := singlylinkedlist.New()
 
-	var pyLibrary *rule.Rule
-	if !pyLibraryFilenames.Empty() {
-		deps, err := parser.parse(pyLibraryFilenames)
+	appendPyLibrary := func(srcs *treeset.Set, pyLibraryTargetName string) {
+		deps, err := parser.parse(srcs)
 		if err != nil {
 			log.Fatalf("ERROR: %v\n", err)
 		}
 
-		pyLibraryTargetName := cfg.RenderLibraryName(packageName)
-
 		// Check if a target with the same name we are generating already
 		// exists, and if it is of a different kind from the one we are
 		// generating. If so, we have to throw an error since Gazelle won't
@@ -239,16 +241,34 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
 			}
 		}
 
-		pyLibrary = newTargetBuilder(pyLibraryKind, pyLibraryTargetName, pythonProjectRoot, args.Rel, pyFileNames).
+		pyLibrary := newTargetBuilder(pyLibraryKind, pyLibraryTargetName, pythonProjectRoot, args.Rel, pyFileNames).
 			addVisibility(visibility).
-			addSrcs(pyLibraryFilenames).
+			addSrcs(srcs).
 			addModuleDependencies(deps).
 			generateImportsAttribute().
 			build()
 
 		result.Gen = append(result.Gen, pyLibrary)
 		result.Imports = append(result.Imports, pyLibrary.PrivateAttr(config.GazelleImportsKey))
 	}
+	if cfg.PerFileGeneration() {
+		pyLibraryFilenames.Each(func(index int, filename interface{}) {
+			if filename == pyLibraryEntrypointFilename {
+				stat, err := os.Stat(filepath.Join(args.Dir, filename.(string)))
+				if err != nil {
+					log.Fatalf("ERROR: %v\n", err)
+				}
+				if stat.Size() == 0 {
+					return // ignore empty __init__.py
+				}
+			}
+			srcs := treeset.NewWith(godsutils.StringComparator, filename)
+			pyLibraryTargetName := strings.TrimSuffix(filepath.Base(filename.(string)), ".py")
+			appendPyLibrary(srcs, pyLibraryTargetName)
+		})
+	} else if !pyLibraryFilenames.Empty() {
+		appendPyLibrary(pyLibraryFilenames, cfg.RenderLibraryName(packageName))
+	}
 
 	if hasPyBinary {
 		deps, err := parser.parseSingle(pyBinaryEntrypointFilename)

gazelle/python/kinds.go

@@ -49,7 +49,8 @@ var pyKinds = map[string]rule.KindInfo{
 		},
 	},
 	pyLibraryKind: {
-		MatchAny: true,
+		MatchAny:   false,
+		MatchAttrs: []string{"srcs"},
 		NonEmptyAttrs: map[string]bool{
 			"deps":       true,
 			"srcs":       true,

gazelle/python/resolve.go

@@ -151,10 +151,10 @@ func (py *Resolver) Resolve(
 			for len(moduleParts) > 1 {
 				// Iterate back through the possible imports until
 				// a match is found.
-				// For example, "from foo.bar import baz" where bar is a variable, we should try
-				// `foo.bar.baz` first, then `foo.bar`, then `foo`. In the first case, the import could be file `baz.py`
-				// in the directory `foo/bar`.
-				// Or, the import could be variable `bar` in file `foo/bar.py`.
+				// For example, "from foo.bar import baz" where baz is a module, we should try `foo.bar.baz` first, then
+				// `foo.bar`, then `foo`.
+				// In the first case, the import could be file `baz.py` in the directory `foo/bar`.
+				// Or, the import could be variable `baz` in file `foo/bar.py`.
 				// The import could also be from a standard module, e.g. `six.moves`, where
 				// the dependency is actually `six`.
 				moduleParts = moduleParts[:len(moduleParts)-1]

gazelle/python/testdata/dont_rename_target/BUILD.in

@@ -2,4 +2,5 @@ load("@rules_python//python:defs.bzl", "py_library")
 
 py_library(
     name = "my_custom_target",
+    srcs = ["__init__.py"],
 )

gazelle/python/testdata/per_file/BUILD.in

@@ -0,0 +1,11 @@
+load("@rules_python//python:defs.bzl", "py_library")
+
+# gazelle:python_generation_mode file
+
+# This target should be kept unmodified by Gazelle.
+py_library(
+    name = "custom",
+    srcs = ["bar.py"],
+    visibility = ["//visibility:private"],
+	tags = ["cant_touch_this"],
+)

gazelle/python/testdata/per_file/BUILD.out

@@ -0,0 +1,24 @@
+load("@rules_python//python:defs.bzl", "py_library")
+
+# gazelle:python_generation_mode file
+
+# This target should be kept unmodified by Gazelle.
+py_library(
+    name = "custom",
+    srcs = ["bar.py"],
+    tags = ["cant_touch_this"],
+    visibility = ["//visibility:private"],
+)
+
+py_library(
+    name = "baz",
+    srcs = ["baz.py"],
+    visibility = ["//:__subpackages__"],
+)
+
+py_library(
+    name = "foo",
+    srcs = ["foo.py"],
+    visibility = ["//:__subpackages__"],
+    deps = [":custom"],
+)

gazelle/python/testdata/per_file/README.md

@@ -0,0 +1,5 @@
+# Per-file generation
+
+This test case generates one `py_library` per file.
+
+`__init__.py` is left empty so no target is generated for it.