Shard integration tests (#1350)

Author: restingbull

.bazelci/presubmit.yml

@@ -6,6 +6,10 @@ matrix:
   integration_platform:
     - ubuntu2404
     - ubuntu2204
+  integration_shard_flags:
+    - ["--test_tag_filters=shard_0", "--config=rbe"]
+    - ["--test_tag_filters=shard_1", "--config=rbe"]
+    - ["--test_tag_filters=shard_2", "--config=rbe"]
     # - macos
     # - windows re-enable when rules_bazel_integration_test can support custom test runner on windows.
   test_flags:
@@ -30,8 +34,7 @@ tasks:
   integration_tests:
     name: "Integration Tests"
     platform: ${{ integration_platform }}
-    test_flags:
-      - "--config=rbe"
+    test_flags: ${{ integration_shard_flags }}
     test_targets:
       - //examples:all
   rbe_ubuntu1604:

examples/BUILD

@@ -1,8 +1,4 @@
-load("@bazel_binaries//:defs.bzl", "bazel_binaries")
-load(
-    "@rules_bazel_integration_test//bazel_integration_test:defs.bzl",
-    "bazel_integration_tests",
-)
+load(":integration.bzl", "derive_metadata", "example_integration_test_suite")
 
 genrule(
     name = "update_bit_ignore",
@@ -21,54 +17,23 @@ genrule(
     executable = True,
 )
 
+SHARD_COUNT = 3
+
 [
-    bazel_integration_tests(
-        name = "%s_test" % example,
-        timeout = "eternal",
-        additional_env_inherit = [
-            "ANDROID_HOME",
-            "ANDROID_SDK_ROOT",
-            "ANDROID_NDK_HOME",
-        ],
-        bazel_versions = [
-            version
-            for version in bazel_binaries.versions.all
-            if version in metadata["only"] or (not metadata["only"] and version not in metadata["exclude"])
-        ],
-        tags = [],
-        test_runner = "//src/main/kotlin/io/bazel/kotlin/test:BazelIntegrationTestRunner",
-        workspace_files = glob(
-            ["%s/**/**" % example],
-            # exclude any bazel directories if existing
-            exclude = ["%s/bazel-*/**" % example],
-        ),
-        workspace_path = example,
+    example_integration_test_suite(
+        name = example,
+        metadata = metadata,
+        tags = ["shard_%s" % (idx % SHARD_COUNT)],
     )
-    for (example, metadata) in {
-        example: {
-            "exclude": [
-                # Cut to the file name, and use it as an excluded bazel version. For exclusion to work
-                # the file name in the `exclude` directory must match the bazel version in `bazel_binaries.versions.all`.
-                # This is done as a secondary loop for readability and avoiding over-globbing.
-                version.rpartition("/")[2]
-                for version in glob(
-                    ["%s/exclude/*" % example],
-                    allow_empty = True,
-                )
-            ],
-            "only": [
-                # Cut to the file name, and use it as an only bazel version. For exclusion to work
-                # the file name in the `only` directory must match the bazel version in `bazel_binaries.versions.all`.
-                # This is done as a secondary loop for readability and avoiding over-globbing.
-                version.rpartition("/")[2]
-                for version in glob(
-                    ["%s/only/*" % example],
-                    allow_empty = True,
-                )
-            ],
-        }
+    for (
+        idx,
+        (example, metadata),
+    ) in enumerate({
+        example: derive_metadata(
+            directory = example,
+        )
         for example in {
-            # Cut to the directory.
+            # Cut to the directory, de-duplicate via dict.
             file.partition("/")[0]: True
             for file in glob(
                 ["**/*"],
@@ -77,10 +42,10 @@ genrule(
                     "*",
                     # Node is currently broken.
                     "node/**",
-                    # Anvil is broken by a verison upgrade.
+                    # Anvil is broken by a version upgrade.
                     "anvil/**",
                 ],
             )
         }
-    }.items()
+    }.items())
 ]

examples/integration.bzl

@@ -0,0 +1,65 @@
+"""Macros for managing the integration test framework."""
+
+load("@bazel_binaries//:defs.bzl", "bazel_binaries")
+load(
+    "@rules_bazel_integration_test//bazel_integration_test:defs.bzl",
+    "bazel_integration_test",
+)
+
+def derive_metadata(directory):
+    return struct(
+        directory = directory,
+        workspace_files = native.glob(
+            ["%s/**/**" % directory],
+            # exclude any bazel directories if existing
+            exclude = ["%s/bazel-*/**" % directory],
+        ),
+        exclude = [
+            # Cut to the file name, and use it as an excluded bazel version. For exclusion to work
+            # the file name in the `exclude` directory must match the bazel version in `bazel_binaries.versions.all`.
+            # This is done as a secondary loop for readability and avoiding over-globbing.
+            version.rpartition("/")[2]
+            for version in native.glob(
+                ["%s/exclude/*" % directory],
+                allow_empty = True,
+            )
+        ],
+        only = [
+            # Cut to the file name, and use it as an only bazel version. For exclusion to work
+            # the file name in the `only` directory must match the bazel version in `bazel_binaries.versions.all`.
+            # This is done as a secondary loop for readability and avoiding over-globbing.
+            version.rpartition("/")[2]
+            for version in native.glob(
+                ["%s/only/*" % directory],
+                allow_empty = True,
+            )
+        ],
+    )
+
+def example_integration_test_suite(
+        name,
+        metadata,
+        tags):
+    for version in bazel_binaries.versions.all:
+        if version in metadata.only or (not metadata.only and version not in metadata.exclude):
+            clean_bazel_version = Label(version).name
+            test_name = "%s_%s_test" % (name, clean_bazel_version)
+            bazel_integration_test(
+                name = test_name,
+                timeout = "eternal",
+                additional_env_inherit = [
+                    "ANDROID_HOME",
+                    "ANDROID_SDK_ROOT",
+                    "ANDROID_NDK_HOME",
+                ],
+                bazel_version = version,
+                tags = tags + [clean_bazel_version, name],
+                test_runner = "//src/main/kotlin/io/bazel/kotlin/test:BazelIntegrationTestRunner",
+                workspace_files = metadata.workspace_files,
+                workspace_path = metadata.directory,
+            )
+
+    native.test_suite(
+        name = name,
+        tags = [name],
+    )