Add bin script for joining buildkite logs

Author: Bencodes

bin/combine-buildkite

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+import os
+import re
+import requests
+
+_API_TOKEN = os.environ["BUILDKITE_API_TOKEN"]
+_BASE_URL = "https://api.buildkite.com/v2/organizations/"
+_BUILDKITE_AUTH_HEADERS = {"Authorization": f"Bearer {_API_TOKEN}"}
+
+ansi_escape_8bit = re.compile(
+    r"(?:\x1B[@-Z\\-_]|[\x80-\x9A\x9C-\x9F]|(?:\x1B\[|\x9B)[0-?]*[ -/]*[@-~])"
+)
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Combine buildkite logs")
+    parser.add_argument(
+        "pipeline", help="Pipeline to combine logs for, ex: bazelbuild/bazel"
+    )
+    parser.add_argument(
+        "build_number", help="Build number to combine logs for"
+    )
+    return parser
+
+
+def _get_job_log(
+    org: str, pipeline: str, build_number: str, job_id: str
+) -> str:
+    response = requests.get(
+        f"{_BASE_URL}/{org}/pipelines/{pipeline}/builds/{build_number}/jobs/{job_id}/log",
+        headers=_BUILDKITE_AUTH_HEADERS,
+    )
+    response.raise_for_status()
+    return ansi_escape_8bit.sub("", response.json()["content"]).replace(
+        "\r", ""
+    )
+
+
+async def _main(org: str, pipeline: str, build_number: str) -> None:
+    response = requests.get(
+        f"{_BASE_URL}/{org}/pipelines/{pipeline}/builds/{build_number}",
+        headers=_BUILDKITE_AUTH_HEADERS,
+    )
+    response.raise_for_status()
+
+    combined_log = ""
+    build = response.json()
+    loop = asyncio.get_event_loop()
+    futures = []
+
+    for job in build["jobs"]:
+        job_id = str(job["id"])
+        futures.append(
+            loop.run_in_executor(
+                None, _get_job_log, org, pipeline, build_number, job_id
+            )
+        )
+
+    for future in futures:
+        combined_log += await future
+        combined_log += "\n"
+
+    with open(f"buildkite-build-{build_number}.log", "w") as f:
+        f.write(combined_log)
+        f.write("\n")
+
+
+if __name__ == "__main__":
+    args = _build_parser().parse_args()
+    org, pipeline = args.pipeline.split("/", 1)
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(_main(org, pipeline, args.build_number))