Skip to content

Commit fed9171

Browse files
committed
fix: test
1 parent 453b4f6 commit fed9171

File tree

1 file changed

+69
-0
lines changed

1 file changed

+69
-0
lines changed

tests/e2e/rayjob_existing_cluster_kind_test.py

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,75 @@ def monitor_rayjob_completion(self, rayjob: RayJob, timeout: int = 360):
257257
print(
258258
f"📋 RayJob YAML (Unknown status debug):\n{result.stdout}"
259259
)
260+
261+
# Also check for job pods that might be stuck
262+
job_pods_result = subprocess.run(
263+
[
264+
"kubectl",
265+
"get",
266+
"pods",
267+
"-n",
268+
self.namespace,
269+
"-l",
270+
f"ray.io/group=rayjob",
271+
"-o",
272+
"wide",
273+
],
274+
capture_output=True,
275+
text=True,
276+
timeout=10,
277+
)
278+
if job_pods_result.returncode == 0:
279+
print(f"🔍 RayJob-related pods:\n{job_pods_result.stdout}")
280+
281+
# Check for any pending pods in the namespace
282+
pending_pods_result = subprocess.run(
283+
[
284+
"kubectl",
285+
"get",
286+
"pods",
287+
"-n",
288+
self.namespace,
289+
"--field-selector=status.phase=Pending",
290+
"-o",
291+
"wide",
292+
],
293+
capture_output=True,
294+
text=True,
295+
timeout=10,
296+
)
297+
if (
298+
pending_pods_result.returncode == 0
299+
and pending_pods_result.stdout.strip()
300+
):
301+
print(
302+
f"⏸️ Pending pods in namespace:\n{pending_pods_result.stdout}"
303+
)
304+
305+
# Get events for the entire namespace to see scheduling issues
306+
namespace_events_result = subprocess.run(
307+
[
308+
"kubectl",
309+
"get",
310+
"events",
311+
"-n",
312+
self.namespace,
313+
"--sort-by=.metadata.creationTimestamp",
314+
"-o",
315+
"wide",
316+
],
317+
capture_output=True,
318+
text=True,
319+
timeout=10,
320+
)
321+
if (
322+
namespace_events_result.returncode == 0
323+
and namespace_events_result.stdout.strip()
324+
):
325+
print(
326+
f"📅 Recent namespace events:\n{namespace_events_result.stdout}"
327+
)
328+
260329
except Exception as e:
261330
print(f"❌ Error getting debug info: {e}")
262331

0 commit comments

Comments
 (0)