fix inf. loop

Author: sbryngelson

.github/scripts/monitor_slurm_job.sh

@@ -48,9 +48,12 @@ tail_pid=$!
 
 # Wait for job to complete with retry logic for transient squeue failures
 squeue_failures=0
+no_sacct_retries=0
+max_no_sacct_retries=5
 while true; do
   if squeue -j "$job_id" &>/dev/null; then
     squeue_failures=0
+    no_sacct_retries=0
   else
     squeue_failures=$((squeue_failures + 1))
     # Check if job actually completed using sacct (if available)
@@ -68,7 +71,14 @@ while true; do
             ;;
         esac
       else
-        # No sacct: avoid false positive by doing an extra check cycle
+        # No sacct: track retries to avoid infinite loop
+        no_sacct_retries=$((no_sacct_retries + 1))
+        if [ $no_sacct_retries -ge $max_no_sacct_retries ]; then
+          echo "Warning: sacct unavailable and job not in squeue after $((no_sacct_retries * 5)) seconds"
+          echo "Assuming job completed (output file exists and squeue consistently fails)"
+          break
+        fi
+        # Reset squeue_failures to retry (but track no_sacct_retries to prevent infinite loop)
         squeue_failures=2
       fi
     fi