ci: add bazel_integration_test (#338)

This runs a py_test with a copy of bazel as a data dep.
It glob()s up the sources for each example and runs nested bazel test on them.
This detects whether the examples are fully working and self-contained.

Follow-up step is to replace the rules_python.tgz with a HEAD version so we detect
breakages.

Author: alexeagle

.bazelrc

@@ -1,8 +1,9 @@
 # For bazel-in-bazel testing
-# Trick bazel into treating BUILD files under examples/* and e2e/* as being regular files
+# Trick bazel into treating BUILD files under examples/* as being regular files
 # This lets us glob() up all the files inside the examples to make them inputs to tests
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
-# To update these lines, run this command:
-# sed -i.bak "/^[^#].*--deleted_packages/s#=.*#=$(find examples/*/* \( -name BUILD -or -name BUILD.bazel \) | xargs -n 1 dirname | paste -sd, -)#" .bazelrc && rm .bazelrc.bak
+# To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
 build --deleted_packages=examples/pip/boto,examples/pip/extras,examples/pip/helloworld
 query --deleted_packages=examples/pip/boto,examples/pip/extras,examples/pip/helloworld
+
+test --test_output=errors

examples/BUILD

@@ -11,6 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+load("//tools/bazel_integration_test:bazel_integration_test.bzl", "bazel_integration_test")
+
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
+
+bazel_integration_test(
+    name = "pip_example",
+    timeout = "long",
+)

examples/pip/.bazelrc

@@ -0,0 +1,2 @@
+# Watch Bazel run, and only run one at a time
+test --test_output=streamed

examples/pip/boto/BUILD

@@ -22,8 +22,10 @@ licenses(["notice"])  # Apache 2.0
 py_test(
     name = "boto_test",
     srcs = ["boto_test.py"],
+    python_version = "PY2",
     deps = [
         requirement("boto3"),
+        requirement("pip"),
         # six is a transitive dependency via python-dateutil. Explicitly depend
         # on it to work around issue #70; see issue #98.
         requirement("six"),

examples/pip/helloworld/helloworld_test.py

@@ -12,10 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import helloworld
 import unittest
 
-from examples.helloworld import helloworld
-
 
 class HelloWorldTest(unittest.TestCase):
 

internal_deps.bzl

@@ -110,3 +110,13 @@ def rules_python_internal_deps():
         name = "piptool_deps",
         requirements = "@rules_python//python:requirements.txt",
     )
+
+    maybe(
+        http_archive,
+        name = "build_bazel_integration_testing",
+        urls = [
+            "https://github.com/bazelbuild/bazel-integration-testing/archive/165440b2dbda885f8d1ccb8d0f417e6cf8c54f17.zip",
+        ],
+        strip_prefix = "bazel-integration-testing-165440b2dbda885f8d1ccb8d0f417e6cf8c54f17",
+        sha256 = "2401b1369ef44cc42f91dc94443ef491208dbd06da1e1e10b702d8c189f098e3",
+    )

internal_setup.bzl

@@ -1,13 +1,20 @@
 """Setup for rules_python tests and tools."""
 
+load("@build_bazel_integration_testing//tools:repositories.bzl", "bazel_binaries")
+
 # Requirements for building our piptool.
 load(
     "@piptool_deps//:requirements.bzl",
     _piptool_install = "pip_install",
 )
 
+load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
+
 def rules_python_internal_setup():
     """Setup for rules_python tests and tools."""
 
     # Requirements for building our piptool.
     _piptool_install()
+
+    # Depend on the Bazel binaries for running bazel-in-bazel tests
+    bazel_binaries(versions = SUPPORTED_BAZEL_VERSIONS)

tools/bazel_integration_test/BUILD

@@ -0,0 +1 @@
+exports_files(["test_runner.py"])

tools/bazel_integration_test/bazel_integration_test.bzl

@@ -0,0 +1,111 @@
+"Define a rule for running bazel test under Bazel"
+
+load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
+load("//python:defs.bzl", "py_test")
+
+BAZEL_BINARY = "@build_bazel_bazel_%s//:bazel_binary" % SUPPORTED_BAZEL_VERSIONS[0].replace(".", "_")
+
+_ATTRS = {
+    "bazel_binary": attr.label(
+        default = BAZEL_BINARY,
+        doc = """The bazel binary files to test against.
+
+It is assumed by the test runner that the bazel binary is found at label_workspace/bazel (wksp/bazel.exe on Windows)""",
+    ),
+    "bazel_commands": attr.string_list(
+        default = ["info", "test ..."],
+        doc = """The list of bazel commands to run. Defaults to `["info", "test ..."]`.
+
+Note that if a command contains a bare `--` argument, the --test_arg passed to Bazel will appear before it.
+""",
+    ),
+    "workspace_files": attr.label(
+        doc = """A filegroup of all files in the workspace-under-test necessary to run the test.""",
+    ),
+}
+
+# Avoid using non-normalized paths (workspace/../other_workspace/path)
+def _to_manifest_path(ctx, file):
+    if file.short_path.startswith("../"):
+        return file.short_path[3:]
+    else:
+        return ctx.workspace_name + "/" + file.short_path
+
+def _config_impl(ctx):
+    if len(SUPPORTED_BAZEL_VERSIONS) > 1:
+        fail("""
+        bazel_integration_test doesn't support multiple Bazel versions to test against yet.
+        """)
+    if len(ctx.files.workspace_files) == 0:
+        fail("""
+No files were found to run under integration testing. See comment in /.bazelrc.
+You probably need to run 
+    tools/bazel_integration_test/update_deleted_packages.sh
+""")
+
+    # Serialize configuration file for test runner
+    config = ctx.actions.declare_file("%s.json" % ctx.attr.name)
+    ctx.actions.write(
+        output = config,
+        content = """
+{{
+    "workspaceRoot": "{TMPL_workspace_root}",
+    "bazelBinaryWorkspace": "{TMPL_bazel_binary_workspace}",
+    "bazelCommands": [ {TMPL_bazel_commands} ]
+}}
+""".format(
+            TMPL_workspace_root = ctx.files.workspace_files[0].dirname,
+            TMPL_bazel_binary_workspace = ctx.attr.bazel_binary.label.workspace_name,
+            TMPL_bazel_commands = ", ".join(["\"%s\"" % s for s in ctx.attr.bazel_commands]),
+        ),
+    )
+
+    return [DefaultInfo(
+        files = depset([config]),
+        runfiles = ctx.runfiles(files = [config]),
+    )]
+
+_config = rule(
+    implementation = _config_impl,
+    doc = "Configures an integration test that runs a specified version of bazel against an external workspace.",
+    attrs = _ATTRS,
+)
+
+def bazel_integration_test(name, **kwargs):
+    """Wrapper macro to set default srcs and run a py_test with config
+
+    Args:
+        name: name of the resulting py_test
+        **kwargs: additional attributes like timeout and visibility
+    """
+    # By default, we assume sources for "pip_example" are in examples/pip/**/*
+    dirname = name[:-len("_example")]
+    native.filegroup(
+        name = "_%s_sources" % name,
+        srcs = native.glob(
+            ["%s/**/*" % dirname],
+            exclude = ["%s/bazel-*/**" % dirname],
+        ),
+    )
+    workspace_files = kwargs.pop("workspace_files", "_%s_sources" % name)
+
+    _config(
+        name = "_%s_config" % name,
+        workspace_files = workspace_files,
+    )
+
+    py_test(
+        name = name,
+        srcs = [Label("//tools/bazel_integration_test:test_runner.py")],
+        main = "test_runner.py",
+        args = [native.package_name() + "/_%s_config.json" % name],
+        deps = [Label("//python/runfiles")],
+        data = [
+            BAZEL_BINARY,
+            "_%s_config" % name,
+            workspace_files,
+        ],
+        **kwargs,
+    )
+
+    

tools/bazel_integration_test/test_runner.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+import json
+import os
+import platform
+from subprocess import Popen
+import sys
+
+from rules_python.python.runfiles import runfiles
+
+def main(conf_file):
+    with open(conf_file) as j:
+        config = json.load(j)
+    r = runfiles.Create()
+
+    isWindows = platform.system() == 'Windows'
+    bazelBinary = r.Rlocation(os.path.join(config['bazelBinaryWorkspace'], 'bazel.exe' if isWindows else 'bazel'))
+    
+    workspacePath = config['workspaceRoot']
+    # Canonicalize bazel external/some_repo/foo
+    if workspacePath.startswith('external/'):
+        workspacePath = '..' + workspacePath[len('external'):]
+
+    for command in config['bazelCommands']:
+        bazel_args = command.split(' ')
+        try:
+            doubleHyphenPos = bazel_args.index('--')
+            print("patch that in ", doubleHyphenPos)
+        except ValueError:
+            pass
+
+
+        # Bazel's wrapper script needs this or you get 
+        # 2020/07/13 21:58:11 could not get the user's cache directory: $HOME is not defined
+        os.environ['HOME'] = str(Path.home())
+
+        bazel_args.insert(0, bazelBinary)
+        bazel_process = Popen(bazel_args, cwd = workspacePath)
+        bazel_process.wait()
+        if bazel_process.returncode != 0:
+            # Test failure in Bazel is exit 3
+            # https://github.com/bazelbuild/bazel/blob/486206012a664ecb20bdb196a681efc9a9825049/src/main/java/com/google/devtools/build/lib/util/ExitCode.java#L44
+            sys.exit(3)
+
+if __name__ == '__main__':
+  main(sys.argv[1])