feat(gazelle): Add type-checking only dependencies to pyi_deps (#3014)

#2538 added the
attribute `pyi_deps` to python rules, intended to be used for
dependencies that are only used for type-checking purposes. This PR adds
a new directive, `gazelle:python_generate_pyi_deps`, which, when
enabled:

- When a dependency is added only to satisfy type-checking only imports
(in a `if TYPE_CHECKING:` block), the dependency is added to `pyi_deps`
instead of `deps`;
- Third-party stub packages (eg. `boto3-stubs`) are now added to
`pyi_deps` instead of `deps`.

---------

Co-authored-by: Douglas Thor <[email protected]>

Author: amartani

CHANGELOG.md

@@ -70,6 +70,9 @@ END_UNRELEASED_TEMPLATE
 * (pypi) To configure the environment for `requirements.txt` evaluation, use the newly added
   developer preview of the `pip.default` tag class. Only `rules_python` and root modules can use
   this feature. You can also configure custom `config_settings` using `pip.default`.
+* (gazelle) New directive `gazelle:python_generate_pyi_deps`; when `true`,
+  dependencies added to satisfy type-only imports (`if TYPE_CHECKING`) and type
+  stub packages are added to `pyi_deps` instead of `deps`.
 
 {#v0-0-0-removed}
 ### Removed

gazelle/README.md

@@ -222,6 +222,8 @@ Python-specific directives are as follows:
 | Controls how distribution names in labels to third-party deps are normalized. Useful for using Gazelle plugin with other rules with different label conventions (e.g. `rules_pycross` uses PEP-503). Can be "snake_case", "none", or "pep503".                                                  |
 | `# gazelle:experimental_allow_relative_imports`          | `false` |
 | Controls whether Gazelle resolves dependencies for import statements that use paths relative to the current package. Can be "true" or "false".|
+| `# gazelle:python_generate_pyi_deps`                                                                                                                                                                                                                                                            | `false` |
+| Controls whether to generate a separate `pyi_deps` attribute for type-checking dependencies or merge them into the regular `deps` attribute. When `false` (default), type-checking dependencies are merged into `deps` for backward compatibility. When `true`, generates separate `pyi_deps`. Imports in blocks with the format `if typing.TYPE_CHECKING:`/`if TYPE_CHECKING:` and type-only stub packages (eg. boto3-stubs) are recognized as type-checking dependencies. |
 
 #### Directive: `python_root`:
 

gazelle/python/configure.go

@@ -68,6 +68,7 @@ func (py *Configurer) KnownDirectives() []string {
 		pythonconfig.TestFilePattern,
 		pythonconfig.LabelConvention,
 		pythonconfig.LabelNormalization,
+		pythonconfig.GeneratePyiDeps,
 		pythonconfig.ExperimentalAllowRelativeImports,
 	}
 }
@@ -230,6 +231,12 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
 					pythonconfig.ExperimentalAllowRelativeImports, rel, d.Value)
 			}
 			config.SetExperimentalAllowRelativeImports(v)
+		case pythonconfig.GeneratePyiDeps:
+			v, err := strconv.ParseBool(strings.TrimSpace(d.Value))
+			if err != nil {
+				log.Fatal(err)
+			}
+			config.SetGeneratePyiDeps(v)
 		}
 	}
 

gazelle/python/file_parser.go

@@ -47,9 +47,10 @@ type ParserOutput struct {
 }
 
 type FileParser struct {
-	code        []byte
-	relFilepath string
-	output      ParserOutput
+	code                 []byte
+	relFilepath          string
+	output               ParserOutput
+	inTypeCheckingBlock  bool
 }
 
 func NewFileParser() *FileParser {
@@ -158,6 +159,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool {
 				continue
 			}
 			m.Filepath = p.relFilepath
+			m.TypeCheckingOnly = p.inTypeCheckingBlock
 			if strings.HasPrefix(m.Name, ".") {
 				continue
 			}
@@ -178,6 +180,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool {
 			m.Filepath = p.relFilepath
 			m.From = from
 			m.Name = fmt.Sprintf("%s.%s", from, m.Name)
+			m.TypeCheckingOnly = p.inTypeCheckingBlock
 			p.output.Modules = append(p.output.Modules, m)
 		}
 	} else {
@@ -202,10 +205,43 @@ func (p *FileParser) SetCodeAndFile(code []byte, relPackagePath, filename string
 	p.output.FileName = filename
 }
 
+// isTypeCheckingBlock returns true if the given node is an `if TYPE_CHECKING:` block.
+func (p *FileParser) isTypeCheckingBlock(node *sitter.Node) bool {
+	if node.Type() != sitterNodeTypeIfStatement || node.ChildCount() < 2 {
+		return false
+	}
+
+	condition := node.Child(1)
+
+	// Handle `if TYPE_CHECKING:`
+	if condition.Type() == sitterNodeTypeIdentifier && condition.Content(p.code) == "TYPE_CHECKING" {
+		return true
+	}
+
+	// Handle `if typing.TYPE_CHECKING:`
+	if condition.Type() == "attribute" && condition.ChildCount() >= 3 {
+		object := condition.Child(0)
+		attr := condition.Child(2)
+		if object.Type() == sitterNodeTypeIdentifier && object.Content(p.code) == "typing" &&
+			attr.Type() == sitterNodeTypeIdentifier && attr.Content(p.code) == "TYPE_CHECKING" {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (p *FileParser) parse(ctx context.Context, node *sitter.Node) {
 	if node == nil {
 		return
 	}
+
+	// Check if this is a TYPE_CHECKING block
+	wasInTypeCheckingBlock := p.inTypeCheckingBlock
+	if p.isTypeCheckingBlock(node) {
+		p.inTypeCheckingBlock = true
+	}
+
 	for i := 0; i < int(node.ChildCount()); i++ {
 		if err := ctx.Err(); err != nil {
 			return
@@ -219,6 +255,9 @@ func (p *FileParser) parse(ctx context.Context, node *sitter.Node) {
 		}
 		p.parse(ctx, child)
 	}
+
+	// Restore the previous state
+	p.inTypeCheckingBlock = wasInTypeCheckingBlock
 }
 
 func (p *FileParser) Parse(ctx context.Context) (*ParserOutput, error) {

gazelle/python/file_parser_test.go

@@ -254,3 +254,40 @@ func TestParseFull(t *testing.T) {
 		FileName: "a.py",
 	}, *output)
 }
+
+func TestTypeCheckingImports(t *testing.T) {
+	code := `
+import sys
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import boto3
+    from rest_framework import serializers
+
+def example_function():
+    _ = sys.version_info
+`
+	p := NewFileParser()
+	p.SetCodeAndFile([]byte(code), "", "test.py")
+
+	result, err := p.Parse(context.Background())
+	if err != nil {
+		t.Fatalf("Failed to parse: %v", err)
+	}
+
+	// Check that we found the expected modules
+	expectedModules := map[string]bool{
+		"sys": false,
+		"typing.TYPE_CHECKING": false,
+		"boto3": true,
+		"rest_framework.serializers": true,
+	}
+
+	for _, mod := range result.Modules {
+		if expected, exists := expectedModules[mod.Name]; exists {
+			if mod.TypeCheckingOnly != expected {
+				t.Errorf("Module %s: expected TypeCheckingOnly=%v, got %v", mod.Name, expected, mod.TypeCheckingOnly)
+			}
+		}
+	}
+}

gazelle/python/parser.go

@@ -112,9 +112,9 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin
 				continue
 			}
 
-			modules.Add(m)
+			addModuleToTreeSet(modules, m)
 			if res.HasMain {
-				mainModules[res.FileName].Add(m)
+				addModuleToTreeSet(mainModules[res.FileName], m)
 			}
 		}
 
@@ -158,13 +158,24 @@ type Module struct {
 	// If this was a from import, e.g. from foo import bar, From indicates the module
 	// from which it is imported.
 	From string `json:"from"`
+	// Whether this import is type-checking only (inside if TYPE_CHECKING block).
+	TypeCheckingOnly bool `json:"type_checking_only"`
 }
 
 // moduleComparator compares modules by name.
 func moduleComparator(a, b interface{}) int {
 	return godsutils.StringComparator(a.(Module).Name, b.(Module).Name)
 }
 
+// addModuleToTreeSet adds a module to a treeset.Set, ensuring that a TypeCheckingOnly=false module is
+// prefered over a TypeCheckingOnly=true module.
+func addModuleToTreeSet(set *treeset.Set, mod Module) {
+	if mod.TypeCheckingOnly && set.Contains(mod) {
+		return
+	}
+	set.Add(mod)
+}
+
 // annotationKind represents Gazelle annotation kinds.
 type annotationKind string
 

gazelle/python/resolve.go

@@ -123,6 +123,16 @@ func (py *Resolver) Embeds(r *rule.Rule, from label.Label) []label.Label {
 	return make([]label.Label, 0)
 }
 
+// addDependency adds a dependency to either the regular deps or pyiDeps set based on
+// whether the module is type-checking only.
+func addDependency(dep string, mod Module, deps, pyiDeps *treeset.Set) {
+	if mod.TypeCheckingOnly {
+		pyiDeps.Add(dep)
+	} else {
+		deps.Add(dep)
+	}
+}
+
 // Resolve translates imported libraries for a given rule into Bazel
 // dependencies. Information about imported libraries is returned for each
 // rule generated by language.GenerateRules in
@@ -141,9 +151,11 @@ func (py *Resolver) Resolve(
 	// join with the main Gazelle binary with other rules. It may conflict with
 	// other generators that generate py_* targets.
 	deps := treeset.NewWith(godsutils.StringComparator)
+	pyiDeps := treeset.NewWith(godsutils.StringComparator)
+	cfgs := c.Exts[languageName].(pythonconfig.Configs)
+	cfg := cfgs[from.Pkg]
+
 	if modulesRaw != nil {
-		cfgs := c.Exts[languageName].(pythonconfig.Configs)
-		cfg := cfgs[from.Pkg]
 		pythonProjectRoot := cfg.PythonProjectRoot()
 		modules := modulesRaw.(*treeset.Set)
 		it := modules.Iterator()
@@ -228,7 +240,7 @@ func (py *Resolver) Resolve(
 							override.Repo = ""
 						}
 						dep := override.Rel(from.Repo, from.Pkg).String()
-						deps.Add(dep)
+						addDependency(dep, mod, deps, pyiDeps)
 						if explainDependency == dep {
 							log.Printf("Explaining dependency (%s): "+
 								"in the target %q, the file %q imports %q at line %d, "+
@@ -239,7 +251,7 @@ func (py *Resolver) Resolve(
 					}
 				} else {
 					if dep, distributionName, ok := cfg.FindThirdPartyDependency(moduleName); ok {
-						deps.Add(dep)
+						addDependency(dep, mod, deps, pyiDeps)
 						// Add the type and stub dependencies if they exist.
 						modules := []string{
 							fmt.Sprintf("%s_stubs", strings.ToLower(distributionName)),
@@ -249,7 +261,8 @@ func (py *Resolver) Resolve(
 						}
 						for _, module := range modules {
 							if dep, _, ok := cfg.FindThirdPartyDependency(module); ok {
-								deps.Add(dep)
+								// Type stub packages always go to pyiDeps
+								pyiDeps.Add(dep)
 							}
 						}
 						if explainDependency == dep {
@@ -308,7 +321,7 @@ func (py *Resolver) Resolve(
 						}
 						matchLabel := filteredMatches[0].Label.Rel(from.Repo, from.Pkg)
 						dep := matchLabel.String()
-						deps.Add(dep)
+						addDependency(dep, mod, deps, pyiDeps)
 						if explainDependency == dep {
 							log.Printf("Explaining dependency (%s): "+
 								"in the target %q, the file %q imports %q at line %d, "+
@@ -333,16 +346,41 @@ func (py *Resolver) Resolve(
 			os.Exit(1)
 		}
 	}
+
+	addResolvedDeps(r, deps)
+
+	if cfg.GeneratePyiDeps() {
+		if !deps.Empty() {
+			r.SetAttr("deps", convertDependencySetToExpr(deps))
+		}
+		if !pyiDeps.Empty() {
+			r.SetAttr("pyi_deps", convertDependencySetToExpr(pyiDeps))
+		}
+	} else {
+		// When generate_pyi_deps is false, merge both deps and pyiDeps into deps
+		combinedDeps := treeset.NewWith(godsutils.StringComparator)
+		combinedDeps.Add(deps.Values()...)
+		combinedDeps.Add(pyiDeps.Values()...)
+
+		if !combinedDeps.Empty() {
+			r.SetAttr("deps", convertDependencySetToExpr(combinedDeps))
+		}
+	}
+}
+
+// addResolvedDeps adds the pre-resolved dependencies from the rule's private attributes
+// to the provided deps set.
+func addResolvedDeps(
+	r *rule.Rule,
+	deps *treeset.Set,
+) {
 	resolvedDeps := r.PrivateAttr(resolvedDepsKey).(*treeset.Set)
 	if !resolvedDeps.Empty() {
 		it := resolvedDeps.Iterator()
 		for it.Next() {
 			deps.Add(it.Value())
 		}
 	}
-	if !deps.Empty() {
-		r.SetAttr("deps", convertDependencySetToExpr(deps))
-	}
 }
 
 // targetListFromResults returns a string with the human-readable list of

gazelle/python/target.go

@@ -15,11 +15,12 @@
 package python
 
 import (
+	"path/filepath"
+
 	"github.com/bazelbuild/bazel-gazelle/config"
 	"github.com/bazelbuild/bazel-gazelle/rule"
 	"github.com/emirpasic/gods/sets/treeset"
 	godsutils "github.com/emirpasic/gods/utils"
-	"path/filepath"
 )
 
 // targetBuilder builds targets to be generated by Gazelle.
@@ -79,7 +80,8 @@ func (t *targetBuilder) addModuleDependency(dep Module) *targetBuilder {
 		// dependency resolution easier
 		dep.Name = importSpecFromSrc(t.pythonProjectRoot, t.bzlPackage, fileName).Imp
 	}
-	t.deps.Add(dep)
+
+	addModuleToTreeSet(t.deps, dep)
 	return t
 }
 

gazelle/python/testdata/add_type_stub_packages/BUILD.in

@@ -0,0 +1 @@
+# gazelle:python_generate_pyi_deps true

gazelle/python/testdata/add_type_stub_packages/BUILD.out

@@ -1,14 +1,18 @@
 load("@rules_python//python:defs.bzl", "py_binary")
 
+# gazelle:python_generate_pyi_deps true
+
 py_binary(
     name = "add_type_stub_packages_bin",
     srcs = ["__main__.py"],
     main = "__main__.py",
+    pyi_deps = [
+        "@gazelle_python_test//boto3_stubs",
+        "@gazelle_python_test//django_types",
+    ],
     visibility = ["//:__subpackages__"],
     deps = [
         "@gazelle_python_test//boto3",
-        "@gazelle_python_test//boto3_stubs",
         "@gazelle_python_test//django",
-        "@gazelle_python_test//django_types",
     ],
 )