chore: Migrate to rules_bazel_integration_test (#1598)

This patch deletes the old copy of `bazel_integration_test` that we
had vendored into the repo. It's replaced with the maintained
`rules_bazel_integration_test`.

Summary of changes:
* Increases minimum tested bazel version to 6.4.0.
  rules_bazel_integration_test depends on some features in 6.4
* Moves several CI jobs to be BIT tests. This free ups
  about 10 CI slots.
* Runs these tests under a separate CI job. This is so that feedback
  from the regular test jobs is faster.

Notes about these bazel-in-bazel integration tests:
* The tests are very heavy and easily overwhelm a system. Unfortunately,
  there doesn't appear to be a way to cap their parallelism; only
  disable it entirely using the `exclusive` tag. Some light
  testing shows there is some speedup to be gained on CI
  if we can, in the future, limit them to 2 or 4 concurrent
  executions.
* A special version named "self" is created that re-uses
  whatever the outer Bazel program is. This is mainly so that
  Bazel's "at head" testing pipelines (the one that runs tests
  with Bazel built from head) are able to affect the integration tests.
  It also comes in handy when locally testing a custom Bazel build.
* The globbing of child workspace files can be somewhat prone to
  following `bazel-*` symlinks, so its important the `.bazelignore`
  and deleted packages configs are up-to-date. Otherwise the
  globbing can turn into 30,000+ files and consume a system-freezing
  level of memory and CPU.

Fixes #1209

---------

Co-authored-by: Richard Levasseur <[email protected]>

Author: philsc

.bazelci/presubmit.yml

@@ -21,7 +21,7 @@ buildifier:
 .minimum_supported_version: &minimum_supported_version
   # For testing minimum supported version.
   # NOTE: Keep in sync with //:version.bzl
-  bazel: 6.2.0
+  bazel: 6.4.0
   skip_in_bazel_downstream_pipeline: "Bazel 6 required"
 .reusable_config: &reusable_config
   build_targets:
@@ -32,6 +32,7 @@ buildifier:
     - "@rules_python//examples/wheel/..."
   build_flags:
     - "--keep_going"
+    - "--build_tag_filters=-integration-test"
   test_targets:
     - "--"
     - "..."
@@ -42,6 +43,18 @@ buildifier:
     - "--noenable_bzlmod"
   build_flags:
     - "--noenable_bzlmod"
+.common_bazelinbazel_config: &common_bazelinbazel_config
+    build_flags:
+      - "--build_tag_filters=integration-test"
+    test_flags:
+      - "--test_tag_filters=integration-test"
+      - "--jobs=2"
+      # The integration tests are so expensive that only a few can be run concurrently
+      # without harming overall reliability and runtime.
+      - "--local_test_jobs=2"
+    build_targets: ["..."]
+    test_targets: ["..."]
+
 .reusable_build_test_all: &reusable_build_test_all
   build_targets: ["..."]
   test_targets: ["..."]
@@ -420,19 +433,14 @@ tasks:
     working_directory: examples/pip_repository_annotations
     platform: windows
 
-  integration_test_compile_pip_requirements_ubuntu_min_workspace:
-    <<: *minimum_supported_version
-    <<: *reusable_build_test_all
-    <<: *common_workspace_flags
-    name: "compile_pip_requirements: Ubuntu using minimum Bazel"
-    working_directory: tests/integration/compile_pip_requirements
-    platform: ubuntu2004
-  integration_test_compile_pip_requirements_ubuntu_min_bzlmod:
-    <<: *minimum_supported_version
-    <<: *reusable_build_test_all
-    name: "compile_pip_requirements: Ubuntu using minimum Bazel"
-    working_directory: tests/integration/compile_pip_requirements
+  integration_test_bazelinbazel_ubuntu:
+    <<: *common_bazelinbazel_config
+    name: "tests/integration bazel-in-bazel: Ubuntu"
     platform: ubuntu2004
+  integration_test_bazelinbazel_debian:
+    <<: *common_bazelinbazel_config
+    name: "tests/integration bazel-in-bazel: Debian"
+    platform: debian11
 
   integration_test_compile_pip_requirements_ubuntu:
     <<: *reusable_build_test_all
@@ -507,18 +515,6 @@ tasks:
     - "bazel run //:os_specific_requirements.update"
     - "git diff --exit-code"
 
-  integration_test_pip_repository_entry_points_ubuntu_workspace:
-    <<: *reusable_build_test_all
-    <<: *common_workspace_flags
-    name: "pip_repository_entry_points: Ubuntu, workspace"
-    working_directory: tests/integration/pip_repository_entry_points
-    platform: ubuntu2004
-  integration_test_pip_repository_entry_points_debian_workspace:
-    <<: *reusable_build_test_all
-    <<: *common_workspace_flags
-    name: "pip_repository_entry_points: Debian, workspace"
-    working_directory: tests/integration/pip_repository_entry_points
-    platform: debian11
   integration_test_pip_repository_entry_points_macos_workspace:
     <<: *reusable_build_test_all
     <<: *common_workspace_flags
@@ -532,18 +528,6 @@ tasks:
     working_directory: tests/integration/pip_repository_entry_points
     platform: windows
 
-  integration_test_ignore_root_user_error_ubuntu_workspace:
-    <<: *reusable_build_test_all
-    <<: *common_workspace_flags
-    name: "ignore_root_user_error: Ubuntu, workspace"
-    working_directory: tests/integration/ignore_root_user_error
-    platform: ubuntu2004
-  integration_test_ignore_root_user_error_debian_workspace:
-    <<: *reusable_build_test_all
-    <<: *common_workspace_flags
-    name: "ignore_root_user_error: Debian, workspace"
-    working_directory: tests/integration/ignore_root_user_error
-    platform: debian11
   integration_test_ignore_root_user_error_macos_workspace:
     <<: *reusable_build_test_all
     <<: *common_workspace_flags
@@ -602,4 +586,3 @@ tasks:
     shell_commands:
     # Assert that @compile_pip_requirements//:requirements_test does the right thing.
     - "bazel test @compile_pip_requirements//..."
-

.bazelignore

@@ -10,13 +10,19 @@ bazel-testlogs
 # treated as directories with valid BUILD files for the main repo.
 # Any directory with a WORKSPACE in it should be added here, with
 # an entry like `bazel-{workspacename}`
+examples/bzlmod/bazel-bin
 examples/bzlmod/bazel-bzlmod
+examples/bzlmod/bazel-out
+examples/bzlmod/bazel-testlogs
+examples/bzlmod/other_module/bazel-bin
 examples/bzlmod/other_module/bazel-other_module
+examples/bzlmod/other_module/bazel-out
+examples/bzlmod/other_module/bazel-testlogs
 examples/bzlmod_build_file_generation/bazel-bzlmod_build_file_generation
 examples/multi_python_versions/bazel-multi_python_versions
 examples/pip_parse/bazel-pip_parse
 examples/pip_parse_vendored/bazel-pip_parse_vendored
 examples/py_proto_library/bazel-py_proto_library
-tests/compile_pip_requirements/bazel-compile_pip_requirements
-tests/ignore_root_user_error/bazel-ignore_root_user_error
-tests/pip_repository_entry_points/bazel-pip_repository_entry_points
+tests/integration/compile_pip_requirements/bazel-compile_pip_requirements
+tests/integration/ignore_root_user_error/bazel-ignore_root_user_error
+tests/integration/pip_repository_entry_points/bazel-pip_repository_entry_points

.bazelrc

@@ -2,7 +2,8 @@
 # Trick bazel into treating BUILD files under examples/* as being regular files
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
-# To update these lines, run tools/update_deleted_packages.sh
+# To update these lines, execute
+# `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
 build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/pip_repository_entry_points
 query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/pip_repository_entry_points
 

.pre-commit-config.yaml

@@ -20,7 +20,7 @@ repos:
     rev: 6.1.0
     hooks:
       - id: buildifier
-        args: &args 
+        args: &args
           # Keep this argument in sync with .bazelci/presubmit.yaml
           - --warnings=all
       - id: buildifier-lint
@@ -30,7 +30,7 @@ repos:
     hooks:
       - id: isort
         name: isort (python)
-        args: 
+        args:
           - --profile
           - black
   - repo: https://github.com/psf/black
@@ -42,6 +42,6 @@ repos:
       - id: update-deleted-packages
         name: Update deleted packages
         language: script
-        entry: ./tools/update_deleted_packages.sh
-        files: ^((examples|tests)/*/(MODULE.bazel|WORKSPACE|WORKSPACE.bzlmod|BUILD.bazel)|.bazelrc|tools/update_deleted_packages.sh)$
+        entry: bazel run @rules_bazel_integration_test//tools:update_deleted_packages
+        files: ^((examples|tests)/.*/(MODULE.bazel|WORKSPACE|WORKSPACE.bzlmod|BUILD.bazel)|.bazelrc)$
         pass_filenames: false

BUILD.bazel

@@ -44,9 +44,7 @@ filegroup(
         "@rules_python_gazelle_plugin//:distribution",
     ],
     visibility = [
-        "//examples:__pkg__",
-        "//tests:__pkg__",
-        "//tests/toolchains:__pkg__",
+        "//:__subpackages__",
     ],
 )
 

MODULE.bazel

@@ -12,8 +12,6 @@ bazel_dep(name = "platforms", version = "0.0.4")
 bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
 bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
 
-bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
-
 internal_deps = use_extension("@rules_python//python/private/bzlmod:internal_deps.bzl", "internal_deps")
 internal_deps.install()
 use_repo(
@@ -55,7 +53,10 @@ use_repo(python, "pythons_hub")
 # This call registers the Python toolchains.
 register_toolchains("@pythons_hub//:all")
 
-# ===== DEV ONLY SETUP =====
+# ===== DEV ONLY DEPS AND SETUP BELOW HERE =====
+bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
+bazel_dep(name = "rules_bazel_integration_test", version = "0.20.0", dev_dependency = True)
+
 dev_pip = use_extension(
     "//python/extensions:pip.bzl",
     "pip",
@@ -76,3 +77,27 @@ dev_pip.parse(
     python_version = "3.11",
     requirements_lock = "//docs/sphinx:requirements.txt",
 )
+
+bazel_binaries = use_extension(
+    "@rules_bazel_integration_test//:extensions.bzl",
+    "bazel_binaries",
+    dev_dependency = True,
+)
+
+# Keep in sync with //:version.bzl
+bazel_binaries.local(
+    name = "self",
+    path = "tests/integration/bazel_from_env",
+)
+bazel_binaries.download(version = "6.4.0")
+bazel_binaries.download(version = "rolling")
+use_repo(
+    bazel_binaries,
+    "bazel_binaries",
+    # These don't appear necessary, but are reported as direct dependencies
+    # that should be use_repo()'d, so we add them as requested
+    "bazel_binaries_bazelisk",
+    "build_bazel_bazel_6_4_0",
+    "build_bazel_bazel_rolling",
+    "build_bazel_bazel_self",
+)

examples/BUILD.bazel

@@ -12,6 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-package(default_visibility = ["//visibility:public"])
-
 licenses(["notice"])  # Apache 2.0

gazelle/BUILD.bazel

@@ -28,6 +28,7 @@ filegroup(
     name = "distribution",
     srcs = [
         ":BUILD.bazel",
+        ":MODULE.bazel",
         ":README.md",
         ":WORKSPACE",
         ":def.bzl",

internal_deps.bzl

@@ -156,6 +156,23 @@ def rules_python_internal_deps():
         ],
     )
 
+    http_archive(
+        name = "rules_bazel_integration_test",
+        sha256 = "6e65d497c68f5794349bfa004369e144063686ce1ebd0227717cd23285be45ef",
+        urls = [
+            "https://github.com/bazel-contrib/rules_bazel_integration_test/releases/download/v0.20.0/rules_bazel_integration_test.v0.20.0.tar.gz",
+        ],
+    )
+
+    # Dependency of rules_bazel_integration_test.
+    http_archive(
+        name = "cgrindel_bazel_starlib",
+        sha256 = "9090280a9cff7322e7c22062506b3273a2e880ca464e520b5c77fdfbed4e8805",
+        urls = [
+            "https://github.com/cgrindel/bazel-starlib/releases/download/v0.18.1/bazel-starlib.v0.18.1.tar.gz",
+        ],
+    )
+
     http_archive(
         name = "rules_proto",
         sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd",

internal_setup.bzl

@@ -15,8 +15,12 @@
 """Setup for rules_python tests and tools."""
 
 load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+load("@cgrindel_bazel_starlib//:deps.bzl", "bazel_starlib_dependencies")
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
+load("@rules_bazel_integration_test//bazel_integration_test:deps.bzl", "bazel_integration_test_rules_dependencies")
+load("@rules_bazel_integration_test//bazel_integration_test:repo_defs.bzl", "bazel_binaries")
 load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
+load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
 load("//python/pip_install:repositories.bzl", "pip_install_dependencies")
 load("//python/private:internal_config_repo.bzl", "internal_config_repo")  # buildifier: disable=bzl-visibility
 
@@ -34,3 +38,7 @@ def rules_python_internal_setup():
     rules_proto_toolchains()
 
     protobuf_deps()
+
+    bazel_integration_test_rules_dependencies()
+    bazel_starlib_dependencies()
+    bazel_binaries(versions = SUPPORTED_BAZEL_VERSIONS)