clean up

Author: nefrathenrici

src/backends.jl

@@ -98,6 +98,9 @@ function module_load_string(::Type{DerechoBackend})
     module purge
     module load climacommon
     module list
+    export TMPDIR=\${SCRATCH}/temp
+    mkdir -p \${TMPDIR}
+    echo \$TMPDIR
     """
 end
 

src/pbs.jl

@@ -86,13 +86,11 @@ duplicating the job snippet in tests.
 """
 function pbs_trap_block()
     return """
-    # Self-requeue on preemption or near-walltime signals
-    # - Many PBS deployments send SIGTERM shortly before walltime or on preemption;
-    #   some may send SIGUSR1 as a warning.
-    # - We trap these signals and call `qrerun` to requeue the same job ID so it can
-    #   continue later with the same submission parameters.
-    # - Exiting with status 0 prevents the scheduler from marking the job as failed
-    #   due to the trap.
+    # Self-requeue on preemption or near-walltime signals:
+    # Trap SIGTERM on job termination and call `qrerun` to requeue the same job ID 
+    # so it can continue later with the same submission parameters.
+    # Exiting with status 0 prevents the scheduler from marking the job as failed
+    # due to the trap.
     handle_preterminate() {
         sig="\$1"
         echo "[ClimaCalibrate] Received \$sig on PBS job \${PBS_JOBID:-unknown}, attempting qrerun"
@@ -104,7 +102,6 @@ function pbs_trap_block()
         exit 0
     }
     trap 'handle_preterminate TERM' TERM
-    trap 'handle_preterminate USR1' USR1
     """
 end
 

src/slurm.jl

@@ -250,10 +250,10 @@ function generate_sbatch_script(
     # Self-requeue on pre-timeout or termination signals
     # `#SBATCH --signal=B:USR1@300` sends SIGUSR1 to the batch script 300 seconds before
     # the job time limit (B means send to the batch script). Sites may also deliver TERM.
-    # We trap USR1/TERM and call `scontrol requeue $SLURM_JOB_ID` so the job returns to
+    # We trap USR1/TERM and call `scontrol requeue \$SLURM_JOB_ID` so the job returns to
     # the queue and can continue later with the same submission parameters.
     # Exiting with status 0 prevents a false failure due to the trap itself.
-    trap 'echo "[ClimaCalibrate] Pre-timeout/TERM on job $SLURM_JOB_ID, requeuing"; scontrol requeue $SLURM_JOB_ID; exit 0' USR1 TERM
+    trap 'echo "[ClimaCalibrate] Pre-timeout/TERM on job \$SLURM_JOB_ID, requeuing"; scontrol requeue \$SLURM_JOB_ID; exit 0' USR1 TERM
 
     $module_load_str
     export CLIMACOMMS_DEVICE="$climacomms_device"

test/pbs_unit_tests.jl

@@ -46,13 +46,11 @@ expected_pbs_contents = """
 #PBS -l walltime=01:30:00
 #PBS -l select=2:ncpus=16:ngpus=2:mpiprocs=2
 
-# Self-requeue on preemption or near-walltime signals
-# - Many PBS deployments send SIGTERM shortly before walltime or on preemption;
-#   some may send SIGUSR1 as a warning.
-# - We trap these signals and call `qrerun` to requeue the same job ID so it can
-#   continue later with the same submission parameters.
-# - Exiting with status 0 prevents the scheduler from marking the job as failed
-#   due to the trap.
+# Self-requeue on preemption or near-walltime signals:
+# Trap SIGTERM on job termination and call `qrerun` to requeue the same job ID 
+# so it can continue later with the same submission parameters.
+# Exiting with status 0 prevents the scheduler from marking the job as failed
+# due to the trap.
 handle_preterminate() {
     sig="\$1"
     echo "[ClimaCalibrate] Received \$sig on PBS job \${PBS_JOBID:-unknown}, attempting qrerun"
@@ -64,8 +62,6 @@ handle_preterminate() {
     exit 0
 }
 trap 'handle_preterminate TERM' TERM
-trap 'handle_preterminate USR1' USR1
-
 
 export MODULEPATH="/glade/campaign/univ/ucit0011/ClimaModules-Derecho:\$MODULEPATH" 
 module purge