Merge branch 'main' into fix/add-extra-things-job-submit

Author: deep1401

src/lattice/routes/instances/utils.py

@@ -640,6 +640,7 @@ def launch_cluster_with_skypilot(
             cluster_name=cluster_name,
             idle_minutes_to_autostop=idle_minutes_to_autostop,
             credentials=credentials,
+            retry_until_up=True,
         )
         print(f"REQUEST ID: {request_id}")
 

src/lattice/routes/jobs/utils.py

@@ -187,7 +187,11 @@ def submit_job_to_existing_cluster(
         # Only do sky.launch for file_mounts otherwise skypilot wont honour it
         if file_mounts is not None:
             request_id = sky.launch(
-                task, cluster_name=cluster_name, fast=True, no_setup=True
+                task,
+                cluster_name=cluster_name,
+                fast=True,
+                no_setup=True,
+                retry_until_up=True,
             )
         else:
             request_id = sky.exec(task, cluster_name=cluster_name)