Merge branch 'main' into always-gen-buildfile

Author: dougthor42

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
 
 test --test_output=errors
 

CHANGELOG.md

@@ -66,12 +66,18 @@ END_UNRELEASED_TEMPLATE
 * (pypi) Fixes an issue where builds using a `bazel vendor` vendor directory
   would fail if the constraints file contained environment markers. Fixes
   [#2996](https://github.com/bazel-contrib/rules_python/issues/2996).
+* (pypi) Wheels with BUILD.bazel (or other special Bazel files) no longer
+  result in missing files at runtime
+  ([#2782](https://github.com/bazel-contrib/rules_python/issues/2782)).
 
 {#v0-0-0-added}
 ### Added
 * (pypi) To configure the environment for `requirements.txt` evaluation, use the newly added
   developer preview of the `pip.default` tag class. Only `rules_python` and root modules can use
   this feature. You can also configure custom `config_settings` using `pip.default`.
+* (gazelle) New directive `gazelle:python_generate_pyi_deps`; when `true`,
+  dependencies added to satisfy type-only imports (`if TYPE_CHECKING`) and type
+  stub packages are added to `pyi_deps` instead of `deps`.
 
 {#v0-0-0-removed}
 ### Removed

MODULE.bazel

@@ -103,6 +103,8 @@ use_repo(
     internal_dev_deps,
     "buildkite_config",
     "rules_python_runtime_env_tc_info",
+    "somepkg_with_build_files",
+    "whl_with_build_files",
 )
 
 # Add gazelle plugin so that we can run the gazelle example as an e2e integration

gazelle/README.md

@@ -24,7 +24,7 @@ The following documentation covers using bzlmod.
 
 ## Adding Gazelle to your project
 
-First, you'll need to add Gazelle to your `MODULES.bazel` file.
+First, you'll need to add Gazelle to your `MODULE.bazel` file.
 Get the current version of Gazelle from there releases here:  https://github.com/bazelbuild/bazel-gazelle/releases/.
 
 
@@ -222,6 +222,8 @@ Python-specific directives are as follows:
 | Controls how distribution names in labels to third-party deps are normalized. Useful for using Gazelle plugin with other rules with different label conventions (e.g. `rules_pycross` uses PEP-503). Can be "snake_case", "none", or "pep503".                                                  |
 | `# gazelle:experimental_allow_relative_imports`          | `false` |
 | Controls whether Gazelle resolves dependencies for import statements that use paths relative to the current package. Can be "true" or "false".|
+| `# gazelle:python_generate_pyi_deps`                                                                                                                                                                                                                                                            | `false` |
+| Controls whether to generate a separate `pyi_deps` attribute for type-checking dependencies or merge them into the regular `deps` attribute. When `false` (default), type-checking dependencies are merged into `deps` for backward compatibility. When `true`, generates separate `pyi_deps`. Imports in blocks with the format `if typing.TYPE_CHECKING:`/`if TYPE_CHECKING:` and type-only stub packages (eg. boto3-stubs) are recognized as type-checking dependencies. |
 
 #### Directive: `python_root`:
 

gazelle/python/configure.go

@@ -68,6 +68,7 @@ func (py *Configurer) KnownDirectives() []string {
 		pythonconfig.TestFilePattern,
 		pythonconfig.LabelConvention,
 		pythonconfig.LabelNormalization,
+		pythonconfig.GeneratePyiDeps,
 		pythonconfig.ExperimentalAllowRelativeImports,
 	}
 }
@@ -230,6 +231,12 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) {
 					pythonconfig.ExperimentalAllowRelativeImports, rel, d.Value)
 			}
 			config.SetExperimentalAllowRelativeImports(v)
+		case pythonconfig.GeneratePyiDeps:
+			v, err := strconv.ParseBool(strings.TrimSpace(d.Value))
+			if err != nil {
+				log.Fatal(err)
+			}
+			config.SetGeneratePyiDeps(v)
 		}
 	}
 

gazelle/python/file_parser.go

@@ -47,9 +47,10 @@ type ParserOutput struct {
 }
 
 type FileParser struct {
-	code        []byte
-	relFilepath string
-	output      ParserOutput
+	code                 []byte
+	relFilepath          string
+	output               ParserOutput
+	inTypeCheckingBlock  bool
 }
 
 func NewFileParser() *FileParser {
@@ -158,6 +159,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool {
 				continue
 			}
 			m.Filepath = p.relFilepath
+			m.TypeCheckingOnly = p.inTypeCheckingBlock
 			if strings.HasPrefix(m.Name, ".") {
 				continue
 			}
@@ -178,6 +180,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool {
 			m.Filepath = p.relFilepath
 			m.From = from
 			m.Name = fmt.Sprintf("%s.%s", from, m.Name)
+			m.TypeCheckingOnly = p.inTypeCheckingBlock
 			p.output.Modules = append(p.output.Modules, m)
 		}
 	} else {
@@ -202,10 +205,43 @@ func (p *FileParser) SetCodeAndFile(code []byte, relPackagePath, filename string
 	p.output.FileName = filename
 }
 
+// isTypeCheckingBlock returns true if the given node is an `if TYPE_CHECKING:` block.
+func (p *FileParser) isTypeCheckingBlock(node *sitter.Node) bool {
+	if node.Type() != sitterNodeTypeIfStatement || node.ChildCount() < 2 {
+		return false
+	}
+
+	condition := node.Child(1)
+
+	// Handle `if TYPE_CHECKING:`
+	if condition.Type() == sitterNodeTypeIdentifier && condition.Content(p.code) == "TYPE_CHECKING" {
+		return true
+	}
+
+	// Handle `if typing.TYPE_CHECKING:`
+	if condition.Type() == "attribute" && condition.ChildCount() >= 3 {
+		object := condition.Child(0)
+		attr := condition.Child(2)
+		if object.Type() == sitterNodeTypeIdentifier && object.Content(p.code) == "typing" &&
+			attr.Type() == sitterNodeTypeIdentifier && attr.Content(p.code) == "TYPE_CHECKING" {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (p *FileParser) parse(ctx context.Context, node *sitter.Node) {
 	if node == nil {
 		return
 	}
+
+	// Check if this is a TYPE_CHECKING block
+	wasInTypeCheckingBlock := p.inTypeCheckingBlock
+	if p.isTypeCheckingBlock(node) {
+		p.inTypeCheckingBlock = true
+	}
+
 	for i := 0; i < int(node.ChildCount()); i++ {
 		if err := ctx.Err(); err != nil {
 			return
@@ -219,6 +255,9 @@ func (p *FileParser) parse(ctx context.Context, node *sitter.Node) {
 		}
 		p.parse(ctx, child)
 	}
+
+	// Restore the previous state
+	p.inTypeCheckingBlock = wasInTypeCheckingBlock
 }
 
 func (p *FileParser) Parse(ctx context.Context) (*ParserOutput, error) {

gazelle/python/file_parser_test.go

@@ -254,3 +254,40 @@ func TestParseFull(t *testing.T) {
 		FileName: "a.py",
 	}, *output)
 }
+
+func TestTypeCheckingImports(t *testing.T) {
+	code := `
+import sys
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import boto3
+    from rest_framework import serializers
+
+def example_function():
+    _ = sys.version_info
+`
+	p := NewFileParser()
+	p.SetCodeAndFile([]byte(code), "", "test.py")
+
+	result, err := p.Parse(context.Background())
+	if err != nil {
+		t.Fatalf("Failed to parse: %v", err)
+	}
+
+	// Check that we found the expected modules
+	expectedModules := map[string]bool{
+		"sys": false,
+		"typing.TYPE_CHECKING": false,
+		"boto3": true,
+		"rest_framework.serializers": true,
+	}
+
+	for _, mod := range result.Modules {
+		if expected, exists := expectedModules[mod.Name]; exists {
+			if mod.TypeCheckingOnly != expected {
+				t.Errorf("Module %s: expected TypeCheckingOnly=%v, got %v", mod.Name, expected, mod.TypeCheckingOnly)
+			}
+		}
+	}
+}

gazelle/python/kinds.go

@@ -46,6 +46,7 @@ var pyKinds = map[string]rule.KindInfo{
 		},
 		ResolveAttrs: map[string]bool{
 			"deps": true,
+			"pyi_deps": true,
 		},
 	},
 	pyLibraryKind: {
@@ -62,6 +63,7 @@ var pyKinds = map[string]rule.KindInfo{
 		},
 		ResolveAttrs: map[string]bool{
 			"deps": true,
+			"pyi_deps": true,
 		},
 	},
 	pyTestKind: {
@@ -78,6 +80,7 @@ var pyKinds = map[string]rule.KindInfo{
 		},
 		ResolveAttrs: map[string]bool{
 			"deps": true,
+			"pyi_deps": true,
 		},
 	},
 }

gazelle/python/parser.go

@@ -112,9 +112,9 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin
 				continue
 			}
 
-			modules.Add(m)
+			addModuleToTreeSet(modules, m)
 			if res.HasMain {
-				mainModules[res.FileName].Add(m)
+				addModuleToTreeSet(mainModules[res.FileName], m)
 			}
 		}
 
@@ -158,13 +158,24 @@ type Module struct {
 	// If this was a from import, e.g. from foo import bar, From indicates the module
 	// from which it is imported.
 	From string `json:"from"`
+	// Whether this import is type-checking only (inside if TYPE_CHECKING block).
+	TypeCheckingOnly bool `json:"type_checking_only"`
 }
 
 // moduleComparator compares modules by name.
 func moduleComparator(a, b interface{}) int {
 	return godsutils.StringComparator(a.(Module).Name, b.(Module).Name)
 }
 
+// addModuleToTreeSet adds a module to a treeset.Set, ensuring that a TypeCheckingOnly=false module is
+// prefered over a TypeCheckingOnly=true module.
+func addModuleToTreeSet(set *treeset.Set, mod Module) {
+	if mod.TypeCheckingOnly && set.Contains(mod) {
+		return
+	}
+	set.Add(mod)
+}
+
 // annotationKind represents Gazelle annotation kinds.
 type annotationKind string