|
| 1 | +"""Dispatches a job to the k8s cluster. |
| 2 | +
|
| 3 | +This script takes in a commit SHA to test along with the platform, spawns a job |
| 4 | +to test it, and then streams the logs from the job. We read logs from the job |
| 5 | +every so often using the kuberntes logging API rather than directly executing |
| 6 | +commands inside the container and streaming the output. This is to work |
| 7 | +around https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/748. |
| 8 | +""" |
| 9 | + |
| 10 | +import sys |
| 11 | +import logging |
| 12 | +import time |
| 13 | +import dateutil |
| 14 | +import datetime |
| 15 | +import json |
| 16 | + |
| 17 | +import kubernetes |
| 18 | + |
| 19 | +PLATFORM_TO_NAMESPACE = {"Linux": "llvm-premerge-linux-buildbot"} |
| 20 | +LOG_SECONDS_TO_QUERY = 10 |
| 21 | +SECONDS_QUERY_LOGS_EVERY = 5 |
| 22 | + |
| 23 | + |
| 24 | +def start_build_linux(commit_sha: str, k8s_client) -> str: |
| 25 | + """Spawns a pod to build/test LLVM at the specified SHA. |
| 26 | +
|
| 27 | + Args: |
| 28 | + commit_sha: The commit SHA to build/run the tests at. |
| 29 | + k8s_client: The kubernetes client instance to use for spawning the pod. |
| 30 | +
|
| 31 | + Returns: |
| 32 | + A string containing the name of the pod. |
| 33 | + """ |
| 34 | + pod_name = f"build-{commit_sha}" |
| 35 | + commands = [ |
| 36 | + "git clone --depth 100 https://github.com/llvm/llvm-project", |
| 37 | + "cd llvm-project", |
| 38 | + f"git checkout ${commit_sha}", |
| 39 | + "export CC=clang", |
| 40 | + "export CXX=clang++", |
| 41 | + './.ci/monolithic-linux.sh "bolt;clang;clang-tools-extra;flang;libclc;lld;lldb;llvm;mlir;polly" "check-bolt check-clang check-clang-cir check-clang-tools check-flang check-lld check-lldb check-llvm check-mlir check-polly" "compiler-rt;libc;libcxx;libcxxabi;libunwind" "check-compiler-rt check-libc" "check-cxx check-cxxabi check-unwind" "OFF"' |
| 42 | + "echo BUILD FINISHED", |
| 43 | + ] |
| 44 | + pod_definition = { |
| 45 | + "apiVersion": "v1", |
| 46 | + "kind": "Pod", |
| 47 | + "metadata": { |
| 48 | + "name": pod_name, |
| 49 | + "namespace": PLATFORM_TO_NAMESPACE["Linux"], |
| 50 | + }, |
| 51 | + "spec": { |
| 52 | + "containers": [ |
| 53 | + { |
| 54 | + "name": "build", |
| 55 | + "image": "ghcr.io/llvm/ci-ubuntu-24.04", |
| 56 | + "command": ["/bin/bash", "-c", ";".join(commands)], |
| 57 | + } |
| 58 | + ], |
| 59 | + "restartPolicy": "Never", |
| 60 | + }, |
| 61 | + } |
| 62 | + kubernetes.utils.create_from_dict(k8s_client, pod_definition) |
| 63 | + return pod_name |
| 64 | + |
| 65 | + |
| 66 | +def read_logs(pod_name: str, namespace: str, v1_api) -> list[str]: |
| 67 | + """Reads logs from the specified pod. |
| 68 | +
|
| 69 | + Reads logs using the k8s API and returns a nicely formatted list of |
| 70 | + strings. |
| 71 | +
|
| 72 | + Args: |
| 73 | + pod_name: The name of the pod to read logs from. |
| 74 | + namespace: The namespace the pod is in. |
| 75 | + v1_api: The kubernetes API instance to use for querying logs. |
| 76 | +
|
| 77 | + Returns: |
| 78 | + A list of strings representing the log lines. |
| 79 | + """ |
| 80 | + logs = v1_api.read_namespaced_pod_log( |
| 81 | + name=pod_name, |
| 82 | + namespace=namespace, |
| 83 | + timestamps=True, |
| 84 | + since_seconds=LOG_SECONDS_TO_QUERY, |
| 85 | + ) |
| 86 | + return logs.split("\n")[:-1] |
| 87 | + |
| 88 | + |
| 89 | +def get_logs_to_print( |
| 90 | + logs: list[str], latest_time: datetime.datetime |
| 91 | +) -> tuple[datetime.datetime, list[str]]: |
| 92 | + """Get the logs that we should be printing. |
| 93 | +
|
| 94 | + This function takes in a raw list of logs along with the timestamp of the |
| 95 | + last log line to be printed and returns the new log lines that should be |
| 96 | + printed. |
| 97 | +
|
| 98 | + Args: |
| 99 | + logs: The raw list of log lines. |
| 100 | + latest_time: The timestamp from the last log line that was printed. |
| 101 | +
|
| 102 | + Returns: |
| 103 | + A tuple containing the timestamp of the last log line returned and a list |
| 104 | + of strings containing the log lines that should be printed. |
| 105 | + """ |
| 106 | + first_new_index = 0 |
| 107 | + time_stamp = latest_time |
| 108 | + for log_line in logs: |
| 109 | + time_stamp_str = log_line.split(" ")[0] |
| 110 | + time_stamp = dateutil.parser.parse(time_stamp_str[:-1]) |
| 111 | + if time_stamp > latest_time: |
| 112 | + break |
| 113 | + first_new_index += 1 |
| 114 | + last_time_stamp = latest_time |
| 115 | + if logs: |
| 116 | + last_time_stamp_str = logs[-1].split(" ")[0] |
| 117 | + last_time_stamp = dateutil.parser.parse(last_time_stamp_str[:-1]) |
| 118 | + return (last_time_stamp, logs[first_new_index:]) |
| 119 | + |
| 120 | + |
| 121 | +def print_logs( |
| 122 | + pod_name: str, namespace: str, v1_api, lastest_time: datetime.datetime |
| 123 | +) -> tuple[bool, datetime.datetime]: |
| 124 | + """Queries the pod and prints the relevant log lines. |
| 125 | +
|
| 126 | + Args: |
| 127 | + pod_name: The pod to print the logs for. |
| 128 | + namespace: The namespace the log is in. |
| 129 | + v1_api: The kubernetes API client instance to use for querying the logs. |
| 130 | + latest_time: The timestamp of the last log line to be printed. |
| 131 | +
|
| 132 | + Returns: |
| 133 | + A tuple containing a boolean representing whether or not the pod has |
| 134 | + finished executing and the timestamp of the last log line printed. |
| 135 | + """ |
| 136 | + logs = read_logs(pod_name, namespace, v1_api) |
| 137 | + new_time_stamp, logs_to_print = get_logs_to_print(logs, lastest_time) |
| 138 | + pod_finished = False |
| 139 | + for log_line in logs_to_print: |
| 140 | + print(log_line.split("\r")[-1]) |
| 141 | + if "BUILD FINISHED" in log_line: |
| 142 | + pod_finished = True |
| 143 | + |
| 144 | + return (pod_finished, new_time_stamp) |
| 145 | + |
| 146 | + |
| 147 | +def main(commit_sha: str, platform: str): |
| 148 | + kubernetes.config.load_kube_config() |
| 149 | + k8s_client = kubernetes.client.ApiClient() |
| 150 | + if platform == "Linux": |
| 151 | + pod_name = start_build_linux(commit_sha, k8s_client) |
| 152 | + else: |
| 153 | + raise ValueError("Unrecognized platform.") |
| 154 | + namespace = PLATFORM_TO_NAMESPACE[platform] |
| 155 | + latest_time = datetime.datetime.min |
| 156 | + v1_api = kubernetes.client.CoreV1Api() |
| 157 | + while True: |
| 158 | + try: |
| 159 | + pod_finished, latest_time = print_logs( |
| 160 | + pod_name, namespace, v1_api, latest_time |
| 161 | + ) |
| 162 | + if pod_finished: |
| 163 | + break |
| 164 | + except kubernetes.client.exceptions.ApiException as log_exception: |
| 165 | + if "ContainerCreating" in json.loads(log_exception.body)["message"]: |
| 166 | + logging.warning( |
| 167 | + "Cannot yet read logs from the pod: waiting for the container to start." |
| 168 | + ) |
| 169 | + else: |
| 170 | + logging.warning(f"Failed to get logs from the pod: {log_exception}") |
| 171 | + time.sleep(SECONDS_QUERY_LOGS_EVERY) |
| 172 | + v1_api.delete_namespaced_pod(pod_name, namespace) |
| 173 | + |
| 174 | + |
| 175 | +if __name__ == "__main__": |
| 176 | + if len(sys.argv) != 3: |
| 177 | + logging.fatal("Expected usage is dispatch_job.py {commit SHA} {platform}") |
| 178 | + sys.exit(1) |
| 179 | + main(sys.argv[1], sys.argv[2]) |
0 commit comments