Skip to content

Commit 0a87d5b

Browse files
[CI] Wait for Pod to Schedule before Requesting Logs
Otherwise we run into 500 errors if the pod takes too long to schedule.
1 parent a0946e7 commit 0a87d5b

File tree

2 files changed

+14
-2
lines changed

2 files changed

+14
-2
lines changed

premerge/premerge_resources/main.tf

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ resource "kubernetes_role" "linux_buildbot_role" {
287287

288288
rule {
289289
api_groups = [""]
290-
resources = ["pods", "pods/log"]
290+
resources = ["pods", "pods/log", "pods/status"]
291291
verbs = ["create", "delete", "get"]
292292
}
293293

@@ -338,7 +338,7 @@ resource "kubernetes_role" "windows_2022_buildbot_role" {
338338

339339
rule {
340340
api_groups = [""]
341-
resources = ["pods", "pods/log"]
341+
resources = ["pods", "pods/log", "pods/status"]
342342
verbs = ["create", "delete", "get"]
343343
}
344344

zorg/buildbot/builders/annotated/premerge/dispatch_job.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,13 @@ def read_logs(pod_name: str, namespace: str, v1_api) -> list[str]:
149149
return logs.split("\n")[:-1]
150150

151151

152+
def get_pod_status(pod_name: str, namespace: str, v1_api) -> str:
153+
"""Gets the status of a pod."""
154+
return v1_api.read_namespaced_pod_status(
155+
name=pod_name, namespace=namespace
156+
).status.phase
157+
158+
152159
def get_logs_to_print(
153160
logs: list[str], latest_time: datetime.datetime
154161
) -> tuple[datetime.datetime, list[str]]:
@@ -220,6 +227,11 @@ def main(commit_sha: str, platform: str):
220227
latest_time = datetime.datetime.min
221228
v1_api = kubernetes.client.CoreV1Api()
222229
print("@@@BUILD_STEP Build/Test@@@")
230+
pod_status = "Pending"
231+
while pod_status == "Pending":
232+
print("Waiting for the pod to schedule onto a machine.")
233+
time.sleep(SECONDS_QUERY_LOGS_EVERY)
234+
pod_status = get_pod_status(pod_name, namespace, v1_api)
223235
while True:
224236
try:
225237
pod_finished, latest_time = print_logs(

0 commit comments

Comments
 (0)