BCR bazel compatibility test: Add generate_report.py (#2091)

This will be run in the last step for a
https://buildkite.com/bazel/bcr-bazel-compatibility-test build without
`USE_BAZELISK_MIGRATE` and generate a report in markdown that can be
used for filing a GitHub issue report for broken modules.

e.g.:
<img width="685" alt="image"
src="https://github.com/user-attachments/assets/3e888dd0-601c-4b38-bce0-047eb80be9bb">

Author: meteorcloudy

buildkite/bazel-central-registry/bcr_compatibility.py

@@ -23,6 +23,7 @@
 import os
 import sys
 import subprocess
+import time
 
 import bazelci
 import bcr_presubmit
@@ -47,6 +48,14 @@
 # Default to use only 30% of CI resources for each type of machines.
 CI_RESOURCE_PERCENTAGE = int(os.environ.get('CI_RESOURCE_PERCENTAGE', 30))
 
+SCRIPT_URL = "https://raw.githubusercontent.com/bazelbuild/continuous-integration/{}/buildkite/bazel-central-registry/generate_report.py?{}".format(
+    bazelci.GITHUB_BRANCH, int(time.time())
+)
+
+
+def fetch_generate_report_py_command():
+    return "curl -s {0} -o generate_report.py".format(SCRIPT_URL)
+
 
 def select_modules_from_env_vars():
     """
@@ -108,6 +117,26 @@ def create_step_for_report_flags_results():
         ),
     ]
 
+def create_step_for_generate_report():
+    parts = [
+        bazelci.PLATFORMS[bazelci.DEFAULT_PLATFORM]["python"],
+        "generate_report.py",
+        "--build_number=%s" % os.getenv("BUILDKITE_BUILD_NUMBER"),
+    ]
+    return [
+        {"wait": "~", "continue_on_failure": "true"},
+        bazelci.create_step(
+            label="Generate report in markdown",
+            commands=[
+                bazelci.fetch_bazelcipy_command(),
+                bcr_presubmit.fetch_bcr_presubmit_py_command(),
+                fetch_generate_report_py_command(),
+                " ".join(parts),
+            ],
+            platform=bazelci.DEFAULT_PLATFORM,
+        ),
+    ]
+
 def main():
     modules = get_target_modules()
     pipeline_steps = []
@@ -127,6 +156,8 @@ def main():
             pipeline_steps.insert(0, {"block": "Please review generated jobs before proceeding", "blocked_state": "running"})
         if bazelci.use_bazelisk_migrate():
             pipeline_steps += create_step_for_report_flags_results()
+        else:
+            pipeline_steps += create_step_for_generate_report()
 
     bcr_presubmit.upload_jobs_to_pipeline(pipeline_steps)
 

buildkite/bazel-central-registry/generate_report.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+#
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# pylint: disable=line-too-long
+# pylint: disable=missing-function-docstring
+# pylint: disable=unspecified-encoding
+# pylint: disable=invalid-name
+"""The CI script for generate report for BCR Bazel Compatibility Test pipeline."""
+
+
+import argparse
+import collections
+import os
+import json
+import re
+import sys
+
+import bazelci
+import bcr_presubmit
+
+BUILDKITE_ORG = os.environ["BUILDKITE_ORGANIZATION_SLUG"]
+
+PIPELINE = os.environ["BUILDKITE_PIPELINE_SLUG"]
+
+MODULE_VERSION_PATTERN = re.compile(r'(?P<module_version>[a-z](?:[a-z0-9._-]*[a-z0-9])?@[^\s]+)')
+
+def extract_module_version(line):
+    match = MODULE_VERSION_PATTERN.search(line)
+    if match:
+        return match.group("module_version")
+
+
+def get_github_maintainer(module_name):
+    metadata = json.load(open(bcr_presubmit.get_metadata_json(module_name), "r"))
+    github_maintainers = []
+    for maintainer in metadata["maintainers"]:
+        if "github" in maintainer:
+            github_maintainers.append(maintainer["github"])
+
+    if not github_maintainers:
+        github_maintainers.append("bazelbuild/bcr-maintainers")
+    return github_maintainers
+
+
+def print_report_in_markdown(failed_jobs_per_module, pipeline_url):
+    bazel_version = os.environ.get("USE_BAZEL_VERSION")
+    print("\n")
+    print("## The following modules are broken%s:" % (f" with Bazel@{bazel_version}" if bazel_version else ""))
+    print("BCR Bazel Compatibility Test: ", pipeline_url)
+    for module, jobs in failed_jobs_per_module.items():
+        module_name = module.strip().split("@")[0]
+        github_maintainers = get_github_maintainer(module_name)
+        print(f"### {module}")
+        print("Maintainers: ", ", ".join(f"@{maintainer}" for maintainer in github_maintainers))
+        for job in jobs:
+            print(f"- [{job['name']}]({job['web_url']})")
+    print("\n")
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    parser = argparse.ArgumentParser(description="Script to report BCR Bazel Compatibility Test result.")
+    parser.add_argument("--build_number", type=str)
+
+    args = parser.parse_args(argv)
+    if not args.build_number:
+        parser.print_help()
+        return 2
+
+    client = bazelci.BuildkiteClient(org=BUILDKITE_ORG, pipeline=PIPELINE)
+    build_info = client.get_build_info(args.build_number)
+    failed_jobs_per_module = collections.defaultdict(list)
+    for job in build_info["jobs"]:
+        if job.get("state") == "failed" and "name" in job:
+            module = extract_module_version(job["name"])
+            if not module:
+                continue
+            failed_jobs_per_module[module].append(job)
+
+    print_report_in_markdown(failed_jobs_per_module, build_info["web_url"])
+
+if __name__ == "__main__":
+    sys.exit(main())