fix: delete BUILD et al files from pypi sourced dependencies (#3029)

Sometimes wheels mistakenly contain BUILD files (or other special Bazel
files).
When extracted, these interfere with how we expected the repo to look.
In particular,
globs no longer match correctly because BUILD files create a Bazel
package boundary.

To fix, delete these files. They aren't generally usable, since they
can't know what
version of Bazel, rule_python, or similar, is consuming them.

Fixes #2782

Author: rickeylev

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg
 
 test --test_output=errors
 

CHANGELOG.md

@@ -64,6 +64,9 @@ END_UNRELEASED_TEMPLATE
 * (pypi) Fixes an issue where builds using a `bazel vendor` vendor directory
   would fail if the constraints file contained environment markers. Fixes
   [#2996](https://github.com/bazel-contrib/rules_python/issues/2996).
+* (pypi) Wheels with BUILD.bazel (or other special Bazel files) no longer
+  result in missing files at runtime
+  ([#2782](https://github.com/bazel-contrib/rules_python/issues/2782)).
 
 {#v0-0-0-added}
 ### Added

MODULE.bazel

@@ -103,6 +103,8 @@ use_repo(
     internal_dev_deps,
     "buildkite_config",
     "rules_python_runtime_env_tc_info",
+    "somepkg_with_build_files",
+    "whl_with_build_files",
 )
 
 # Add gazelle plugin so that we can run the gazelle example as an e2e integration

python/private/internal_dev_deps.bzl

@@ -14,6 +14,8 @@
 """Module extension for internal dev_dependency=True setup."""
 
 load("@bazel_ci_rules//:rbe_repo.bzl", "rbe_preconfig")
+load("//python/private/pypi:whl_library.bzl", "whl_library")
+load("//tests/support/whl_from_dir:whl_from_dir_repo.bzl", "whl_from_dir_repo")
 load(":runtime_env_repo.bzl", "runtime_env_repo")
 
 def _internal_dev_deps_impl(mctx):
@@ -28,6 +30,17 @@ def _internal_dev_deps_impl(mctx):
     )
     runtime_env_repo(name = "rules_python_runtime_env_tc_info")
 
+    whl_from_dir_repo(
+        name = "whl_with_build_files",
+        root = "//tests/whl_with_build_files:testdata/BUILD.bazel",
+        output = "somepkg-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "somepkg_with_build_files",
+        whl_file = "@whl_with_build_files//:somepkg-1.0-any-none-any.whl",
+        requirement = "somepkg",
+    )
+
 internal_dev_deps = module_extension(
     implementation = _internal_dev_deps_impl,
     doc = "This extension creates internal rules_python dev dependencies.",

python/private/pypi/whl_library.bzl

@@ -249,6 +249,7 @@ def _whl_library_impl(rctx):
 
     whl_path = None
     if rctx.attr.whl_file:
+        rctx.watch(rctx.attr.whl_file)
         whl_path = rctx.path(rctx.attr.whl_file)
 
         # Simulate the behaviour where the whl is present in the current directory.
@@ -471,8 +472,26 @@ def _whl_library_impl(rctx):
             ],
         )
 
-    rctx.file("BUILD.bazel", build_file_contents)
+    # Delete these in case the wheel had them. They generally don't cause
+    # a problem, but let's avoid the chance of that happening.
+    rctx.file("WORKSPACE")
+    rctx.file("WORKSPACE.bazel")
+    rctx.file("MODULE.bazel")
+    rctx.file("REPO.bazel")
+
+    paths = list(rctx.path(".").readdir())
+    for _ in range(10000000):
+        if not paths:
+            break
+        path = paths.pop()
+
+        # BUILD files interfere with globbing and Bazel package boundaries.
+        if path.basename in ("BUILD", "BUILD.bazel"):
+            rctx.delete(path)
+        elif path.is_dir:
+            paths.extend(path.readdir())
 
+    rctx.file("BUILD.bazel", build_file_contents)
     return
 
 def _generate_entry_point_contents(

tests/support/support.bzl

@@ -19,6 +19,7 @@
 # rules_testing or as config_setting values, which don't support Label in some
 # places.
 
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")  # buildifier: disable=bzl-visibility
 load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")  # buildifier: disable=bzl-visibility
 
 MAC = Label("//tests/support:mac")
@@ -48,3 +49,8 @@ SUPPORTS_BOOTSTRAP_SCRIPT = select({
     "@platforms//os:windows": ["@platforms//:incompatible"],
     "//conditions:default": [],
 }) if IS_BAZEL_7_OR_HIGHER else ["@platforms//:incompatible"]
+
+SUPPORTS_BZLMOD_UNIXY = select({
+    "@platforms//os:windows": ["@platforms//:incompatible"],
+    "//conditions:default": [],
+}) if BZLMOD_ENABLED else ["@platforms//:incompatible"]

tests/support/whl_from_dir/BUILD.bazel

@@ -0,0 +1,50 @@
+"""Creates a whl file from a directory tree.
+
+Used to test wheels. Avoids checking in prebuilt files and their associated
+security risks.
+"""
+
+load("//python/private:repo_utils.bzl", "repo_utils")  # buildifier: disable=bzl-visibility
+
+def _whl_from_dir_repo(rctx):
+    root = rctx.path(rctx.attr.root).dirname
+    repo_utils.watch_tree(rctx, root)
+
+    output = rctx.path(rctx.attr.output)
+    repo_utils.execute_checked(
+        rctx,
+        # cd to root so zip recursively takes everything there.
+        working_directory = str(root),
+        op = "WhlFromDir",
+        arguments = [
+            "zip",
+            "-0",  # Skip compressing
+            "-X",  # Don't store file time or metadata
+            str(output),
+            "-r",
+            ".",
+        ],
+    )
+    rctx.file("BUILD.bazel", 'exports_files(glob(["*"]))')
+
+whl_from_dir_repo = repository_rule(
+    implementation = _whl_from_dir_repo,
+    attrs = {
+        "output": attr.string(
+            doc = """
+Output file name to write. Should match the wheel filename format:
+`pkg-version-pyversion-abi-platform.whl`. Typically a value like
+`mypkg-1.0-any-none-any.whl` is whats used for testing.
+
+For the full format, see
+https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-name-convention
+""",
+        ),
+        "root": attr.label(
+            doc = """
+A file whose directory will be put into the output wheel. All files
+are included verbatim.
+            """,
+        ),
+    },
+)

tests/support/whl_from_dir/whl_from_dir_repo.bzl

@@ -0,0 +1,9 @@
+load("//python:py_test.bzl", "py_test")
+load("//tests/support:support.bzl", "SUPPORTS_BZLMOD_UNIXY")
+
+py_test(
+    name = "verify_files_test",
+    srcs = ["verify_files_test.py"],
+    target_compatible_with = SUPPORTS_BZLMOD_UNIXY,
+    deps = ["@somepkg_with_build_files//:pkg"],
+)