Skip to content

Commit 01450ce

Browse files
authored
Merge pull request #1525 from manics/builder-failed-fail
If build pod is deleted check status instead of assuming Succeeded
2 parents e80f841 + 3450e52 commit 01450ce

File tree

4 files changed

+57
-17
lines changed

4 files changed

+57
-17
lines changed

binderhub/build.py

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,11 @@ class BuildStatus(Enum):
4242
Used when `kind` is `Kind.BUILD_STATUS_CHANGE`
4343
"""
4444

45-
PENDING = 1
46-
RUNNING = 2
47-
COMPLETED = 3
48-
FAILED = 4
49-
UNKNOWN = 5
45+
PENDING = "pending"
46+
RUNNING = "running"
47+
BUILT = "built"
48+
FAILED = "failed"
49+
UNKNOWN = "unknown"
5050

5151
def __init__(self, kind: Kind, payload: Union[str, BuildStatus]):
5252
self.kind = kind
@@ -460,11 +460,22 @@ def submit(self):
460460
_request_timeout=KUBE_REQUEST_TIMEOUT,
461461
):
462462
if f["type"] == "DELETED":
463-
# Assume this is a successful completion
464-
self.progress(
465-
ProgressEvent.Kind.BUILD_STATUS_CHANGE,
466-
ProgressEvent.BuildStatus.COMPLETED,
463+
phase = f["object"].status.phase
464+
app_log.debug(
465+
"Pod %s was deleted with phase %s",
466+
f["object"].metadata.name,
467+
phase,
467468
)
469+
if phase == "Succeeded":
470+
self.progress(
471+
ProgressEvent.Kind.BUILD_STATUS_CHANGE,
472+
ProgressEvent.BuildStatus.BUILT,
473+
)
474+
else:
475+
self.progress(
476+
ProgressEvent.Kind.BUILD_STATUS_CHANGE,
477+
ProgressEvent.BuildStatus.FAILED,
478+
)
468479
return
469480
self.pod = f["object"]
470481
if not self.stop_event.is_set():
@@ -699,7 +710,7 @@ def stream_logs(self):
699710
),
700711
)
701712
self.progress(
702-
ProgressEvent.Kind.BUILD_STATUS_CHANGE, ProgressEvent.BuildStatus.COMPLETED
713+
ProgressEvent.Kind.BUILD_STATUS_CHANGE, ProgressEvent.BuildStatus.BUILT
703714
)
704715
self.progress(
705716
"log",

binderhub/build_local.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ def break_callback():
140140
self._handle_log(line)
141141
self.progress(
142142
ProgressEvent.Kind.BUILD_STATUS_CHANGE,
143-
ProgressEvent.BuildStatus.COMPLETED,
143+
ProgressEvent.BuildStatus.BUILT,
144144
)
145145
except subprocess.CalledProcessError:
146146
self.progress(

binderhub/builder.py

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -509,12 +509,13 @@ def _check_result(future):
509509
# FIXME: If pod goes into an unrecoverable stage, such as ImagePullBackoff or
510510
# whatever, we should fail properly.
511511
if progress.kind == ProgressEvent.Kind.BUILD_STATUS_CHANGE:
512+
phase = progress.payload.value
512513
if progress.payload == ProgressEvent.BuildStatus.PENDING:
513514
# nothing to do, just waiting
514515
continue
515-
elif progress.payload == ProgressEvent.BuildStatus.COMPLETED:
516+
elif progress.payload == ProgressEvent.BuildStatus.BUILT:
516517
event = {
517-
"phase": "built",
518+
"phase": phase,
518519
"message": "Built image, launching...\n",
519520
"imageName": image_name,
520521
}
@@ -525,13 +526,17 @@ def _check_result(future):
525526
log_future = pool.submit(build.stream_logs)
526527
log_future.add_done_callback(_check_result)
527528
continue
528-
elif progress.payload == ProgressEvent.BuildStatus.COMPLETED:
529+
elif progress.payload == ProgressEvent.BuildStatus.BUILT:
529530
# Do nothing, is ok!
530531
continue
531532
elif progress.payload == ProgressEvent.BuildStatus.FAILED:
532-
event = {"phase": "failure"}
533+
event = {"phase": phase}
533534
elif progress.payload == ProgressEvent.BuildStatus.UNKNOWN:
534-
event = {"phase": "unknown"}
535+
event = {"phase": phase}
536+
else:
537+
raise ValueError(
538+
f"Found unknown phase {phase} in ProgressEvent"
539+
)
535540
elif progress.kind == ProgressEvent.Kind.LOG_MESSAGE:
536541
# The logs are coming out of repo2docker, so we expect
537542
# them to be JSON structured anyway

binderhub/tests/test_build.py

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,30 @@ async def test_build(app, needs_build, needs_launch, always_build, slug, pytestc
9494
assert r.url.startswith(final["url"])
9595

9696

97+
@pytest.mark.asyncio(timeout=120)
98+
@pytest.mark.remote
99+
async def test_build_fail(app, needs_build, needs_launch, always_build, pytestconfig):
100+
"""
101+
Test build a repo that should fail immediately.
102+
"""
103+
slug = "gh/binderhub-ci-repos/minimal-dockerfile/failed"
104+
build_url = f"{app.url}/build/{slug}"
105+
r = await async_requests.get(build_url, stream=True)
106+
r.raise_for_status()
107+
failed_events = 0
108+
async for line in async_requests.iter_lines(r):
109+
line = line.decode("utf8", "replace")
110+
if line.startswith("data:"):
111+
event = json.loads(line.split(":", 1)[1])
112+
assert event.get("phase") not in ("launching", "ready")
113+
if event.get("phase") == "failed":
114+
failed_events += 1
115+
break
116+
r.close()
117+
118+
assert failed_events > 0, "Should have seen phase 'failed'"
119+
120+
97121
def _list_dind_pods_mock():
98122
"""Mock list of DIND pods"""
99123
mock_response = mock.MagicMock()
@@ -235,7 +259,7 @@ async def test_local_repo2docker_build():
235259
event = await q.get(10)
236260
if (
237261
event.kind == ProgressEvent.Kind.BUILD_STATUS_CHANGE
238-
and event.payload == ProgressEvent.BuildStatus.COMPLETED
262+
and event.payload == ProgressEvent.BuildStatus.BUILT
239263
):
240264
break
241265
events.append(event)

0 commit comments

Comments
 (0)