fix: test

Author: pawelpaszki

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.languageServer": "None"
+}

tests/e2e/rayjob_existing_cluster_kind_test.py

@@ -105,6 +105,10 @@ def assert_rayjob_submit_against_existing_cluster(
         ), f"Job submission failed, expected {job_name}, got {submission_result}"
         print(f"✅ Successfully submitted RayJob '{job_name}' against existing cluster")
 
+        # Wait a moment for the RayJob resource to be created in Kubernetes
+        print("⏳ Waiting for RayJob resource to be processed by KubeRay operator...")
+        sleep(5)
+
         # Monitor the job status until completion
         self.monitor_rayjob_completion(rayjob, timeout=900)
 
@@ -122,10 +126,15 @@ def monitor_rayjob_completion(self, rayjob: RayJob, timeout: int = 900):
 
         elapsed_time = 0
         check_interval = 10  # Check every 10 seconds
+        job_found = False  # Track if we've seen the job at least once
 
         while elapsed_time < timeout:
             status, ready = rayjob.status(print_to_console=True)
 
+            # Track if we've found the job (not UNKNOWN status)
+            if status != CodeflareRayJobStatus.UNKNOWN:
+                job_found = True
+
             # Check if job has completed (either successfully or failed)
             if status == CodeflareRayJobStatus.COMPLETE:
                 print(f"✅ RayJob '{rayjob.name}' completed successfully!")
@@ -135,15 +144,30 @@ def monitor_rayjob_completion(self, rayjob: RayJob, timeout: int = 900):
             elif status == CodeflareRayJobStatus.RUNNING:
                 print(f"🏃 RayJob '{rayjob.name}' is still running...")
             elif status == CodeflareRayJobStatus.UNKNOWN:
-                print(f"❓ RayJob '{rayjob.name}' status is unknown")
+                if job_found:
+                    # If we've seen the job before but now it's unknown, that's concerning
+                    print(
+                        f"⚠️  RayJob '{rayjob.name}' status became unknown after being found"
+                    )
+                else:
+                    # Job hasn't appeared yet, this is normal initially
+                    print(
+                        f"⏳ Waiting for RayJob '{rayjob.name}' to appear in Kubernetes..."
+                    )
 
             # Wait before next check
             sleep(check_interval)
             elapsed_time += check_interval
 
         # If we reach here, the job has timed out
         final_status, _ = rayjob.status(print_to_console=True)
-        raise TimeoutError(
-            f"⏰ RayJob '{rayjob.name}' did not complete within {timeout} seconds. "
-            f"Final status: {final_status}"
-        )
+        if not job_found:
+            raise TimeoutError(
+                f"⏰ RayJob '{rayjob.name}' was never found in Kubernetes within {timeout} seconds. "
+                f"Check if the RayJob resource was created successfully."
+            )
+        else:
+            raise TimeoutError(
+                f"⏰ RayJob '{rayjob.name}' did not complete within {timeout} seconds. "
+                f"Final status: {final_status}"
+            )

tests/e2e/rayjob_existing_cluster_oauth_test.py

@@ -90,6 +90,10 @@ def run_rayjob_against_existing_cluster_oauth(self):
         ), f"Job submission failed, expected {job_name}, got {submission_result}"
         print(f"✅ Successfully submitted RayJob '{job_name}'")
 
+        # Wait a moment for the RayJob resource to be created in Kubernetes
+        print("⏳ Waiting for RayJob resource to be processed by KubeRay operator...")
+        sleep(5)
+
         # Monitor the job status until completion
         self.monitor_rayjob_completion(rayjob)
 
@@ -103,16 +107,21 @@ def monitor_rayjob_completion(self, rayjob: RayJob, timeout: int = 300):
 
         Args:
             rayjob: The RayJob instance to monitor
-            timeout: Maximum time to wait in seconds (default: 15 minutes)
+            timeout: Maximum time to wait in seconds (default: 5 minutes)
         """
         print(f"⏳ Monitoring RayJob '{rayjob.name}' status...")
 
         elapsed_time = 0
         check_interval = 10  # Check every 10 seconds
+        job_found = False  # Track if we've seen the job at least once
 
         while elapsed_time < timeout:
             status, ready = rayjob.status(print_to_console=True)
 
+            # Track if we've found the job (not UNKNOWN status)
+            if status != CodeflareRayJobStatus.UNKNOWN:
+                job_found = True
+
             # Check if job has completed (either successfully or failed)
             if status == CodeflareRayJobStatus.COMPLETE:
                 print(f"✅ RayJob '{rayjob.name}' completed successfully!")
@@ -122,15 +131,30 @@ def monitor_rayjob_completion(self, rayjob: RayJob, timeout: int = 300):
             elif status == CodeflareRayJobStatus.RUNNING:
                 print(f"🏃 RayJob '{rayjob.name}' is still running...")
             elif status == CodeflareRayJobStatus.UNKNOWN:
-                print(f"❓ RayJob '{rayjob.name}' status is unknown")
+                if job_found:
+                    # If we've seen the job before but now it's unknown, that's concerning
+                    print(
+                        f"⚠️  RayJob '{rayjob.name}' status became unknown after being found"
+                    )
+                else:
+                    # Job hasn't appeared yet, this is normal initially
+                    print(
+                        f"⏳ Waiting for RayJob '{rayjob.name}' to appear in Kubernetes..."
+                    )
 
             # Wait before next check
             sleep(check_interval)
             elapsed_time += check_interval
 
         # If we reach here, the job has timed out
         final_status, _ = rayjob.status(print_to_console=True)
-        raise TimeoutError(
-            f"⏰ RayJob '{rayjob.name}' did not complete within {timeout} seconds. "
-            f"Final status: {final_status}"
-        )
+        if not job_found:
+            raise TimeoutError(
+                f"⏰ RayJob '{rayjob.name}' was never found in Kubernetes within {timeout} seconds. "
+                f"Check if the RayJob resource was created successfully."
+            )
+        else:
+            raise TimeoutError(
+                f"⏰ RayJob '{rayjob.name}' did not complete within {timeout} seconds. "
+                f"Final status: {final_status}"
+            )