@@ -978,42 +978,47 @@ async def _timeout_if_waiting_for_cluster_too_long(
978978        comp_run : CompRunsAtDB ,
979979        comp_tasks : dict [NodeIDStr , CompTaskAtDB ],
980980    ) ->  dict [NodeIDStr , CompTaskAtDB ]:
981-         if  comp_run .result  is  RunningState .WAITING_FOR_CLUSTER :
982-             tasks_waiting_for_cluster  =  [
983-                 t 
984-                 for  t  in  comp_tasks .values ()
985-                 if  t .state  is  RunningState .WAITING_FOR_CLUSTER 
986-             ]
987-             # get latest modified task 
988-             latest_modified_of_all_tasks  =  max (
989-                 tasks_waiting_for_cluster , key = lambda  task : task .modified 
990-             ).modified 
991- 
992-             if  (
993-                 arrow .utcnow ().datetime  -  latest_modified_of_all_tasks 
994-             ) >  self .settings .COMPUTATIONAL_BACKEND_MAX_WAITING_FOR_CLUSTER_TIMEOUT :
995-                 await  CompTasksRepository .instance (
996-                     self .db_engine 
997-                 ).update_project_tasks_state (
998-                     project_id ,
999-                     comp_run .run_id ,
1000-                     [task .node_id  for  task  in  tasks_waiting_for_cluster ],
1001-                     RunningState .FAILED ,
1002-                     optional_progress = 1.0 ,
1003-                     optional_stopped = arrow .utcnow ().datetime ,
1004-                 )
1005-                 for  task  in  tasks_waiting_for_cluster :
1006-                     task .state  =  RunningState .FAILED 
1007-                 msg  =  user_message (
1008-                     "The system has timed out while waiting for computational resources. Please try running your project again or contact oSparc support if this issue persists." ,
1009-                     _version = 1 ,
1010-                 )
1011-                 _logger .error (msg )
1012-                 await  publish_project_log (
1013-                     self .rabbitmq_client ,
1014-                     user_id ,
1015-                     project_id ,
1016-                     log = msg ,
1017-                     log_level = logging .ERROR ,
1018-                 )
981+         if  comp_run .result  is  not RunningState .WAITING_FOR_CLUSTER :
982+             return  comp_tasks 
983+ 
984+         tasks_waiting_for_cluster  =  [
985+             t 
986+             for  t  in  comp_tasks .values ()
987+             if  t .state  is  RunningState .WAITING_FOR_CLUSTER 
988+         ]
989+         if  not  tasks_waiting_for_cluster :
990+             return  comp_tasks 
991+ 
992+         # get latest modified task 
993+         latest_modified_of_all_tasks  =  max (
994+             tasks_waiting_for_cluster , key = lambda  task : task .modified 
995+         ).modified 
996+ 
997+         if  (
998+             arrow .utcnow ().datetime  -  latest_modified_of_all_tasks 
999+         ) >  self .settings .COMPUTATIONAL_BACKEND_MAX_WAITING_FOR_CLUSTER_TIMEOUT :
1000+             await  CompTasksRepository .instance (
1001+                 self .db_engine 
1002+             ).update_project_tasks_state (
1003+                 project_id ,
1004+                 comp_run .run_id ,
1005+                 [task .node_id  for  task  in  tasks_waiting_for_cluster ],
1006+                 RunningState .FAILED ,
1007+                 optional_progress = 1.0 ,
1008+                 optional_stopped = arrow .utcnow ().datetime ,
1009+             )
1010+             for  task  in  tasks_waiting_for_cluster :
1011+                 task .state  =  RunningState .FAILED 
1012+             msg  =  user_message (
1013+                 "The system has timed out while waiting for computational resources. Please try running your project again or contact oSparc support if this issue persists." ,
1014+                 _version = 1 ,
1015+             )
1016+             _logger .error (msg )
1017+             await  publish_project_log (
1018+                 self .rabbitmq_client ,
1019+                 user_id ,
1020+                 project_id ,
1021+                 log = msg ,
1022+                 log_level = logging .ERROR ,
1023+             )
10191024        return  comp_tasks 
0 commit comments