fix preempt and add walltime (#954)

Author: sbryngelson

.github/workflows/frontier/submit-bench.sh

@@ -32,7 +32,7 @@ sbatch <<EOT
 #SBATCH -A CFD154                  # charge account
 #SBATCH -N 1                       # Number of nodes required
 $sbatch_device_opts
-#SBATCH -t 01:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -t 02:59:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -o$job_slug.out            # Combined output and error messages file
 #SBATCH -p extended                # Extended partition for shorter queues
 #SBATCH -W                         # Do not exit until the submitted job terminates.

.github/workflows/frontier/submit.sh

@@ -33,7 +33,7 @@ sbatch <<EOT
 #SBATCH -A CFD154                  # charge account
 #SBATCH -N 1                       # Number of nodes required
 $sbatch_device_opts
-#SBATCH -t 01:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -t 02:59:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -o$job_slug.out            # Combined output and error messages file
 #SBATCH -p extended                # Extended partition for shorter queues
 #SBATCH -W                         # Do not exit until the submitted job terminates.

.github/workflows/phoenix/submit-bench.sh

@@ -69,7 +69,7 @@ JOBID=$(sbatch <<-EOT | awk '{print $4}'
 EOT
 )
 
-echo "🚀 Submitted SLURM job $JOBID"
+echo "Submitted: SLURM job $JOBID"
 
 # if this wrapper is killed/canceled, make sure SLURM job is cleaned up
 trap '[[ -n "${JOBID:-}" ]] && scancel "$JOBID" >/dev/null 2>&1 || :' EXIT
@@ -86,22 +86,22 @@ while :; do
 
   # If it’s one of SLURM’s terminal states, break immediately
   case "$STATE" in
-    COMPLETED|FAILED|CANCELLED|TIMEOUT)
-      echo "✅ SLURM job $JOBID reached terminal state: $STATE"
+    COMPLETED|FAILED|CANCELLED|TIMEOUT|PREEMPTED)
+      echo "Completed: SLURM job $JOBID reached terminal state: $STATE"
       break
       ;;
     "")
-      echo "✅ SLURM job $JOBID no longer in queue; assuming finished"
+      echo "Completed: SLURM job $JOBID no longer in queue; assuming finished"
       break
       ;;
     *)
-      echo "⏳ SLURM job $JOBID state: $STATE"
+      echo "Waiting: SLURM job $JOBID state: $STATE"
       sleep 10
       ;;
   esac
 done
 
 # Now retrieve the exit code and exit with it
 EXIT_CODE=$(sacct -j "$JOBID" --noheader --format=ExitCode | head -1 | cut -d: -f1)
-echo "🔚 SLURM job $JOBID exit code: $EXIT_CODE"
+echo "Completed: SLURM job $JOBID exit code: $EXIT_CODE"
 exit "$EXIT_CODE"

.github/workflows/phoenix/submit.sh

@@ -62,7 +62,7 @@ JOBID=$(sbatch <<-EOT | awk '{print $4}'
 EOT
 )
 
-echo "🚀 Submitted SLURM job $JOBID"
+echo "Submitted: SLURM job $JOBID"
 
 # if this wrapper is killed/canceled, make sure SLURM job is cleaned up
 trap '[[ -n "${JOBID:-}" ]] && scancel "$JOBID" >/dev/null 2>&1 || :' EXIT
@@ -79,22 +79,22 @@ while :; do
 
   # If it’s one of SLURM’s terminal states, break immediately
   case "$STATE" in
-    COMPLETED|FAILED|CANCELLED|TIMEOUT)
-      echo "✅ SLURM job $JOBID reached terminal state: $STATE"
+    COMPLETED|FAILED|CANCELLED|TIMEOUT|PREEMPTED)
+      echo "Completed: SLURM job $JOBID reached terminal state: $STATE"
       break
       ;;
     "")
-      echo "✅ SLURM job $JOBID no longer in queue; assuming finished"
+      echo "Completed: SLURM job $JOBID no longer in queue; assuming finished"
       break
       ;;
     *)
-      echo "⏳ SLURM job $JOBID state: $STATE"
+      echo "Waiting: SLURM job $JOBID state: $STATE"
       sleep 10
       ;;
   esac
 done
 
 # Now retrieve the exit code and exit with it
 EXIT_CODE=$(sacct -j "$JOBID" --noheader --format=ExitCode | head -1 | cut -d: -f1)
-echo "🔚 SLURM job $JOBID exit code: $EXIT_CODE"
+echo "Completed: SLURM job $JOBID exit code: $EXIT_CODE"
 exit "$EXIT_CODE"