fix: test

Author: pawelpaszki

.github/workflows/e2e_tests.yaml

@@ -124,7 +124,8 @@ jobs:
           pip install poetry
           poetry install --with test,docs
           echo "Running e2e tests..."
-          poetry run pytest -v -s ./tests/e2e -m 'kind and nvidia_gpu' > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
+          # poetry run pytest -v -s ./tests/e2e -m 'kind and nvidia_gpu' > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
+          poetry run pytest -v -s ./tests/e2e/rayjob_existing_cluster_kind_test.py::TestRayJobExistingClusterKind::test_rayjob_ray_cluster_sdk_kind_nvidia_gpu > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
         env:
           GRPC_DNS_RESOLVER: "native"
 

src/codeflare_sdk/ray/rayjobs/rayjob.py

@@ -56,6 +56,7 @@ def __init__(
         active_deadline_seconds: Optional[int] = None,
         entrypoint_num_cpus: Optional[int] = None,
         entrypoint_num_gpus: Optional[int] = None,
+        backoff_limit: int = 3,
     ):
         """
         Initialize a RayJob instance.
@@ -104,6 +105,7 @@ def __init__(
         self.active_deadline_seconds = active_deadline_seconds
         self.entrypoint_num_cpus = entrypoint_num_cpus
         self.entrypoint_num_gpus = entrypoint_num_gpus
+        self.backoff_limit = backoff_limit
 
         # Auto-set shutdown_after_job_finishes based on cluster_config presence
         # If cluster_config is provided, we want to clean up the cluster after job finishes
@@ -186,6 +188,7 @@ def _build_rayjob_cr(self) -> Dict[str, Any]:
                 "entrypoint": self.entrypoint,
                 "shutdownAfterJobFinishes": self.shutdown_after_job_finishes,
                 "ttlSecondsAfterFinished": self.ttl_seconds_after_finished,
+                "backoffLimit": self.backoff_limit,
             },
         }
 

tests/e2e/rayjob_existing_cluster_kind_test.py

@@ -157,7 +157,7 @@ def monitor_rayjob_completion(self, rayjob: RayJob, timeout: int = 900):
                     if result.returncode == 0:
                         print(f"📋 RayJob YAML details:\n{result.stdout}")
 
-                    # Also try to get pod logs
+                    # Try to get job submitter pod logs (these pods may be cleaned up quickly)
                     pod_result = subprocess.run(
                         [
                             "kubectl",
@@ -169,6 +169,7 @@ def monitor_rayjob_completion(self, rayjob: RayJob, timeout: int = 900):
                             f"ray.io/rayjob={rayjob.name}",
                             "-o",
                             "name",
+                            "--sort-by=.metadata.creationTimestamp",
                         ],
                         capture_output=True,
                         text=True,
@@ -196,6 +197,26 @@ def monitor_rayjob_completion(self, rayjob: RayJob, timeout: int = 900):
                     else:
                         print(f"❌ Could not find pods for RayJob: {pod_result.stderr}")
 
+                    # Also try to get events related to the RayJob
+                    events_result = subprocess.run(
+                        [
+                            "kubectl",
+                            "get",
+                            "events",
+                            "-n",
+                            self.namespace,
+                            "--field-selector",
+                            f"involvedObject.name={rayjob.name}",
+                            "-o",
+                            "wide",
+                        ],
+                        capture_output=True,
+                        text=True,
+                        timeout=10,
+                    )
+                    if events_result.returncode == 0 and events_result.stdout.strip():
+                        print(f"📅 Events for RayJob:\n{events_result.stdout}")
+
                 except Exception as e:
                     print(f"❌ Error getting failure details: {e}")