Updates for supporting running zppy on Dane (#789)

* Fix E3SM Diags SLURM resources on Dane

* update ncremap command for vertical remapping

* increase open file descriptor limit

* other updates needed supporting mpas-a on dane

* use srun for ncremap on dane

Author: chengzhuzhang

zppy/templates/e3sm_diags.bash

@@ -1,5 +1,16 @@
 #!/bin/bash
 {% include 'inclusions/slurm_header.bash' %}
+
+{# For SLURM, match CPU allocation to E3SM Diags multiprocessing. #}
+{% set cpus_per_task = (num_workers if multiprocessing else 1) %}
+
+{# Dane enforces step-level CPU/memory limits unless explicitly requested. #}
+{% if machine == 'dane' %}
+#SBATCH  --ntasks=1
+#SBATCH  --cpus-per-task={{ cpus_per_task }}
+#SBATCH  --mem=0
+{% endif %}
+
 {% include 'inclusions/boilerplate.bash' %}
 set -e
 {{ environment_commands }}
@@ -456,10 +467,19 @@ EOF
 cat > e3sm_diags.cfg << EOF
 {% include cfg %}
 EOF
+{% if machine == 'dane' %}
+command="srun --ntasks=1 --cpus-per-task={{ cpus_per_task }} --cpu-bind=cores python -u e3sm.py -d e3sm_diags.cfg"
+{% else %}
 command="srun -n 1 python -u e3sm.py -d e3sm_diags.cfg"
+{% endif %}
+{% else %}
+{% if machine == 'dane' %}
+command="srun --ntasks=1 --cpus-per-task={{ cpus_per_task }} --cpu-bind=cores python -u e3sm.py"
 {% else %}
 command="srun -n 1 python -u e3sm.py"
 {% endif %}
+{% endif %}
+
 
 # Run diagnostics
 time ${command}

zppy/templates/e3sm_to_cmip.bash

@@ -38,7 +38,7 @@ EOF
     do
       if [ -f ${file} ]; then
         #ncks --rgr xtr_mth=mss_val --vrt_fl='{{cmip_plevdata}}' ${file} ${file}.plev
-        ncremap -p mpi --vrt_ntp=log --vrt_xtr=mss_val --vrt_out='{{cmip_plevdata}}' ${file} ${file}.plev
+        ncremap {% if machine == 'dane' %}--mpi_pfx='srun -n {{ nodes }}'{% else %}-p mpi{% endif %} --vrt_ntp=log --vrt_xtr=mss_val --vrt_out='{{cmip_plevdata}}' ${file} ${file}.plev
         if [ $? != 0 ]; then
           cd {{ scriptDir }}
           echo 'ERROR (1)' > {{ prefix }}.status

zppy/templates/mpas_analysis.bash

@@ -9,6 +9,12 @@ set +e
 export OMP_NUM_THREADS=1
 export HDF5_USE_FILE_LOCKING=FALSE
 
+{% if machine == 'dane' %}
+# MPAS-Analysis workaround on dane: avoid "Too many open files" failures by
+# increasing the per-process open file descriptor limit (within the job's hard limit).
+ulimit -n 65536 2>/dev/null || true
+{% endif %}
+
 # Basic definitions
 case="{{ case }}"
 www="{{ www }}"
@@ -132,7 +138,7 @@ mapMpiTasks = {{ mapMpiTasks }}
 # "None" if ESMF should perform remapping in serial without a command, or one of
 # "srun" or "mpirun" if it should be run in parallel  (or in serial but with a
 # command)
-{% if machine in ['pm-cpu', 'pm-gpu', 'anvil', 'chrysalis'] %}
+{% if machine in ['pm-cpu', 'pm-gpu', 'anvil', 'chrysalis', 'dane'] %}
 mapParallelExec = srun
 {% elif machine in ['compy'] %}
 mapParallelExec = srun --mpi=pmi2
@@ -142,7 +148,7 @@ mapParallelExec = srun --mpi=pmi2
 # possibly with some flags if it should be run with that command
 {% if machine in ['pm-cpu', 'pm-gpu'] %}
 ncremapParallelExec = None
-{% elif machine in ['anvil', 'chrysalis'] %}
+{% elif machine in ['anvil', 'chrysalis', 'dane'] %}
 ncremapParallelExec = srun -n 1
 {% elif machine in ['compy'] %}
 ncremapParallelExec = srun --mpi=pmi2 -n 1