tests: add integration test for build_python_zip (bazel-contrib#2045)

This is a more comprehensive regression test for verifying
`--build_python_zip` is
actually working
(bazel-contrib#1840)

This also creates a small framework to make it easier to write
integration tests that
need to customize the environment bazel runs in and check the output of
bazel itself.
I figure this will be helpful for writing simple verification tests for
repository/bzlmod
phase logic (i.e. set the debug env vars and grep the output). While we
should avoid heavy
usage of these bazel-in-bazel tests, a bit of grepping logs would go a
long way for covering
edge cases that examples don't cover.

Author: rickeylev

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

tests/integration/BUILD.bazel

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("@rules_bazel_integration_test//bazel_integration_test:defs.bzl", "default_test_runner")
+load("//python:py_library.bzl", "py_library")
 load(":integration_test.bzl", "rules_python_integration_test")
 
 licenses(["notice"])
@@ -102,3 +103,14 @@ rules_python_integration_test(
     bzlmod = False,
     workspace_path = "py_cc_toolchain_registered",
 )
+
+rules_python_integration_test(
+    name = "custom_commands_test",
+    py_main = "custom_commands_test.py",
+)
+
+py_library(
+    name = "runner_lib",
+    srcs = ["runner.py"],
+    imports = ["../../"],
+)

tests/integration/custom_commands/BUILD.bazel

@@ -0,0 +1,20 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@rules_python//python:py_binary.bzl", "py_binary")
+
+py_binary(
+    name = "bin",
+    srcs = ["bin.py"],
+)

tests/integration/custom_commands/MODULE.bazel

@@ -0,0 +1,21 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module(name = "module_under_test")
+
+bazel_dep(name = "rules_python", version = "0.0.0")
+local_path_override(
+    module_name = "rules_python",
+    path = "../../..",
+)

tests/integration/custom_commands/WORKSPACE

@@ -0,0 +1,13 @@
+local_repository(
+    name = "rules_python",
+    path = "../../..",
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
+
+py_repositories()
+
+python_register_toolchains(
+    name = "python_3_11",
+    python_version = "3.11",
+)

tests/integration/custom_commands/WORKSPACE.bzlmod

@@ -0,0 +1,16 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+print("Hello, world")
+print(__file__)

tests/integration/custom_commands/bin.py

@@ -0,0 +1,31 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import unittest
+
+from tests.integration import runner
+
+
+class CustomCommandsTest(runner.TestCase):
+    # Regression test for https://github.com/bazelbuild/rules_python/issues/1840
+    def test_run_build_python_zip_false(self):
+        result = self.run_bazel("run", "--build_python_zip=false", "//:bin")
+        self.assert_result_matches(result, "bazel-out")
+
+
+if __name__ == "__main__":
+    # Enabling this makes the runner log subprocesses as the test goes along.
+    # logging.basicConfig(level = "INFO")
+    unittest.main()

tests/integration/custom_commands_test.py

@@ -19,13 +19,15 @@ load(
     "bazel_integration_tests",
     "integration_test_utils",
 )
+load("//python:py_test.bzl", "py_test")
 
 def rules_python_integration_test(
         name,
         workspace_path = None,
         bzlmod = True,
         gazelle_plugin = False,
         tags = None,
+        py_main = None,
         **kwargs):
     """Runs a bazel-in-bazel integration test.
 
@@ -37,10 +39,24 @@ def rules_python_integration_test(
             disable bzlmod.
         gazelle_plugin: Whether the test uses the gazelle plugin.
         tags: Test tags.
+        py_main: Optional `.py` file to run tests using. When specified, a
+            python based test runner is used, and this source file is the main
+            entry point and responsible for executing tests.
         **kwargs: Passed to the upstream `bazel_integration_tests` rule.
     """
     workspace_path = workspace_path or name.removesuffix("_test")
-    if bzlmod:
+    if py_main:
+        test_runner = name + "_py_runner"
+        py_test(
+            name = test_runner,
+            srcs = [py_main],
+            main = py_main,
+            deps = [":runner_lib"],
+            # Hide from ... patterns; should only be run as part
+            # of the bazel integration test
+            tags = ["manual"],
+        )
+    elif bzlmod:
         if gazelle_plugin:
             test_runner = "//tests/integration:test_runner_gazelle_plugin"
         else:

tests/integration/integration_test.bzl

@@ -0,0 +1,131 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import os.path
+import pathlib
+import re
+import shlex
+import subprocess
+import unittest
+
+_logger = logging.getLogger(__name__)
+
+class ExecuteError(Exception):
+    def __init__(self, result):
+        self.result = result
+    def __str__(self):
+        return self.result.describe()
+
+class ExecuteResult:
+    def __init__(
+        self,
+        args: list[str],
+        env: dict[str, str],
+        cwd: pathlib.Path,
+        proc_result: subprocess.CompletedProcess,
+    ):
+        self.args = args
+        self.env = env
+        self.cwd = cwd
+        self.exit_code = proc_result.returncode
+        self.stdout = proc_result.stdout
+        self.stderr = proc_result.stderr
+
+    def describe(self) -> str:
+        env_lines = [
+            "  " + shlex.quote(f"{key}={value}")
+            for key, value in sorted(self.env.items())
+        ]
+        env = " \\\n".join(env_lines)
+        args = shlex.join(self.args)
+        maybe_stdout_nl = "" if self.stdout.endswith("\n") else "\n"
+        maybe_stderr_nl = "" if self.stderr.endswith("\n") else "\n"
+        return f"""\
+COMMAND:
+cd {self.cwd} && \\
+env \\
+{env} \\
+  {args}
+RESULT: exit_code: {self.exit_code}
+===== STDOUT START =====
+{self.stdout}{maybe_stdout_nl}===== STDOUT END   =====
+===== STDERR START =====
+{self.stderr}{maybe_stderr_nl}===== STDERR END   =====
+"""
+
+
+class TestCase(unittest.TestCase):
+    def setUp(self):
+        super().setUp()
+        self.repo_root = pathlib.Path(os.environ["BIT_WORKSPACE_DIR"])
+        self.bazel = pathlib.Path(os.environ["BIT_BAZEL_BINARY"])
+        outer_test_tmpdir = pathlib.Path(os.environ["TEST_TMPDIR"])
+        self.test_tmp_dir = outer_test_tmpdir / "bit_test_tmp"
+        # Put the global tmp not under the test tmp to better match how a real
+        # execution has entirely different directories for these.
+        self.tmp_dir = outer_test_tmpdir / "bit_tmp"
+        self.bazel_env = {
+            "PATH": os.environ["PATH"],
+            "TEST_TMPDIR": str(self.test_tmp_dir),
+            "TMP": str(self.tmp_dir),
+            # For some reason, this is necessary for Bazel 6.4 to work.
+            # If not present, it can't find some bash helpers in @bazel_tools
+            "RUNFILES_DIR": os.environ["TEST_SRCDIR"]
+        }
+
+    def run_bazel(self, *args: str, check: bool = True) -> ExecuteResult:
+        """Run a bazel invocation.
+
+        Args:
+            *args: The args to pass to bazel; the leading `bazel` command is
+                added automatically
+            check: True if the execution must succeed, False if failure
+                should raise an error.
+        Returns:
+            An `ExecuteResult` from running Bazel
+        """
+        args = [str(self.bazel), *args]
+        env = self.bazel_env
+        _logger.info("executing: %s", shlex.join(args))
+        cwd = self.repo_root
+        proc_result = subprocess.run(
+            args=args,
+            text=True,
+            capture_output=True,
+            cwd=cwd,
+            env=env,
+            check=False,
+        )
+        exec_result = ExecuteResult(args, env, cwd, proc_result)
+        if check and exec_result.exit_code:
+            raise ExecuteError(exec_result)
+        else:
+            return exec_result
+
+    def assert_result_matches(self, result: ExecuteResult, regex: str) -> None:
+        """Assert stdout/stderr of an invocation matches a regex.
+
+        Args:
+            result: ExecuteResult from `run_bazel` whose stdout/stderr will
+                be checked.
+            regex: Pattern to match, using `re.search` semantics.
+        """
+        if not re.search(regex, result.stdout + result.stderr):
+            self.fail(
+                "Bazel output did not match expected pattern\n"
+                + f"expected pattern: {regex}\n"
+                + f"invocation details:\n{result.describe()}"
+            )