Revert "Fix Phoenix template for real (#824)" (#825)

Author: sbryngelson

src/simulation/m_time_steppers.fpp

@@ -1148,7 +1148,7 @@ contains
             if (proc_rank == 0) print *, 'RKCK 5th time-stage at', rkck_time_tmp
 #endif
             call s_compute_rhs(q_cons_ts(2)%vf, q_T_sf, q_prim_vf, rhs_ts_rkck(5)%vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
-            call s_compute_EL_coupled_solver(q_cons_ts(2)%vf, q_prim_vf, rhs_ts_rkck(5)%vf, RKstep)
+            call s_compute_EL_coupled_solver(q_cons_ts(2)%vf, q_prim_vf, rhs_ts_rkck(5)%vf, 5)
             call s_update_tmp_rkck(5, q_cons_ts, rhs_ts_rkck, lag_largestep)
             if (lag_largestep > 0._wp) call s_compute_rkck_dt(lag_largestep, restart_rkck_step)
             if (restart_rkck_step) cycle
@@ -1162,7 +1162,7 @@ contains
             if (proc_rank == 0) print *, 'RKCK 6th time-stage at', rkck_time_tmp
 #endif
             call s_compute_rhs(q_cons_ts(2)%vf, q_T_sf, q_prim_vf, rhs_ts_rkck(6)%vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
-            call s_compute_EL_coupled_solver(q_cons_ts(2)%vf, q_prim_vf, rhs_ts_rkck(6)%vf, RKstep)
+            call s_compute_EL_coupled_solver(q_cons_ts(2)%vf, q_prim_vf, rhs_ts_rkck(6)%vf, 6)
             call s_update_tmp_rkck(6, q_cons_ts, rhs_ts_rkck, lag_largestep)
             if (lag_largestep > 0._wp) call s_compute_rkck_dt(lag_largestep, restart_rkck_step)
             if (restart_rkck_step) cycle

toolchain/templates/phoenix.mako

@@ -18,7 +18,8 @@
 #SBATCH --qos=${quality_of_service}
 % endif
 % if gpu:
-#SBATCH -G${tasks_per_node}
+#SBATCH --gres=gpu:V100:${tasks_per_node}
+#SBATCH --mem-per-gpu=16G\
 % endif
 % if email:
 #SBATCH --mail-user=${email}
@@ -40,15 +41,10 @@ echo
     % if not mpi:
         (set -x; ${profiler} "${target.get_install_binpath(case)}")
     % else:
-         (set -x; srun \
-            % if engine == 'interactive':
-                --nodes ${nodes} --ntasks-per-node ${tasks_per_node} \
-                --cpus-per-task 1                                    \
-                % if gpu:
-                    --gpus-per-task 1 --gpu-bind closest                 \
-                % endif
-            % endif
-            ${profiler} "${target.get_install_binpath(case)}")
+        (set -x; ${profiler}    \
+            mpirun -np ${nodes*tasks_per_node}            \
+                   --bind-to none                         \
+                   "${target.get_install_binpath(case)}")
     % endif
 
     ${helpers.run_epilogue(target)}