Merge branch 'master' into debug-testhang

Author: sbryngelson

.cursor/rules/mfc-agent-rules.mdc

@@ -16,7 +16,7 @@ Written primarily for Fortran/Fypp; the OpenACC and style sections matter only w
   - Most sources are `.fpp`; CMake transpiles them to `.f90`.  
 - **Fypp macros** live in `src/<subprogram>/include/` you should scan these first.  
   `<subprogram>` ∈ {`simulation`,`common`,`pre_process`,`post_process`}.  
-- Only `simulation` (+ its `common` calls) is GPU-accelerated via **OpenACC**.  
+- Only `simulation` (+ its `common` calls) is GPU-accelerated via **OpenACC** or **OpenMP**.  
 - Assume free-form Fortran 2008+, `implicit none`, explicit `intent`, and modern
   intrinsics.  
 - Prefer `module … contains … subroutine foo()`; avoid `COMMON` blocks and
@@ -56,27 +56,29 @@ Written primarily for Fortran/Fypp; the OpenACC and style sections matter only w
 * Every variable: `intent(in|out|inout)` + appropriate `dimension` / `allocatable`
   / `pointer`.
 * Use `s_mpi_abort(<msg>)` for errors, not `stop`.
-* Mark OpenACC-callable helpers that are called from OpenACC parallel loops immediately after declaration:
+* Mark GPU-callable helpers that are called from GPU parallel loops immediately after declaration:
   ```fortran
   subroutine s_flux_update(...)
-    !$acc routine seq
+    $:GPU_ROUTINE(function_name='s_flux_update', parallelism='[seq]')
     ...
   end subroutine
   ```
 
 ---
 
-# 3  OpenACC Programming Guidelines (for kernels)
+# 3  FYPP Macros for GPU acceleration Pogramming Guidelines (for kernels)
+
+Do not directly use OpenACC or OpenMP directives directly. Instead, use the FYPP macros contained in src/common/include/parallel_macros.fpp
 
 Wrap tight loops with
 
 ```fortran
-!$acc parallel loop gang vector default(present) reduction(...)
+$:GPU_PARALLEL_FOR(private='[...]', copy='[...]')
 ```
-* Add `collapse(n)` to merge nested loops when safe.
-* Declare loop-local variables with `private(...)`.
+* Add `collapse=n` to merge nested loops when safe.
+* Declare loop-local variables with `private='[...]'`.
 * Allocate large arrays with `managed` or move them into a persistent
-  `!$acc enter data` region at start-up.
+  `$:GPU_ENTER_DATA(...)` region at start-up.
 * **Do not** place `stop` / `error stop` inside device code.
 * Must compile with Cray `ftn` and NVIDIA `nvfortran` for GPU offloading; also build CPU-only with
   GNU `gfortran` and Intel `ifx`/`ifort`.

.fortlsrc

@@ -29,7 +29,7 @@
         "MFC_DOUBLE_PRECISION": 1
     },
     "lowercase_intrinsics": true,
-    "debug_log": true,
+    "debug_log": false,
     "disable_diagnostics": false,
     "use_signature_help": true,
     "variable_hover": true,
@@ -93,4 +93,4 @@
         "**/m_nvtx*",
         "**/syscheck.fpp"
     ]
-} 
+} 

.github/workflows/frontier/submit-bench.sh

@@ -32,7 +32,7 @@ sbatch <<EOT
 #SBATCH -A CFD154                  # charge account
 #SBATCH -N 1                       # Number of nodes required
 $sbatch_device_opts
-#SBATCH -t 01:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -t 02:59:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -o$job_slug.out            # Combined output and error messages file
 #SBATCH -p extended                # Extended partition for shorter queues
 #SBATCH -W                         # Do not exit until the submitted job terminates.

.github/workflows/frontier/submit.sh

@@ -33,7 +33,7 @@ sbatch <<EOT
 #SBATCH -A CFD154                  # charge account
 #SBATCH -N 1                       # Number of nodes required
 $sbatch_device_opts
-#SBATCH -t 01:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -t 02:59:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -o$job_slug.out            # Combined output and error messages file
 #SBATCH -p extended                # Extended partition for shorter queues
 #SBATCH -W                         # Do not exit until the submitted job terminates.

.github/workflows/phoenix/submit-bench.sh

@@ -69,7 +69,7 @@ JOBID=$(sbatch <<-EOT | awk '{print $4}'
 EOT
 )
 
-echo "🚀 Submitted SLURM job $JOBID"
+echo "Submitted: SLURM job $JOBID"
 
 # if this wrapper is killed/canceled, make sure SLURM job is cleaned up
 trap '[[ -n "${JOBID:-}" ]] && scancel "$JOBID" >/dev/null 2>&1 || :' EXIT
@@ -86,22 +86,22 @@ while :; do
 
   # If it’s one of SLURM’s terminal states, break immediately
   case "$STATE" in
-    COMPLETED|FAILED|CANCELLED|TIMEOUT)
-      echo "✅ SLURM job $JOBID reached terminal state: $STATE"
+    COMPLETED|FAILED|CANCELLED|TIMEOUT|PREEMPTED)
+      echo "Completed: SLURM job $JOBID reached terminal state: $STATE"
       break
       ;;
     "")
-      echo "✅ SLURM job $JOBID no longer in queue; assuming finished"
+      echo "Completed: SLURM job $JOBID no longer in queue; assuming finished"
       break
       ;;
     *)
-      echo "⏳ SLURM job $JOBID state: $STATE"
+      echo "Waiting: SLURM job $JOBID state: $STATE"
       sleep 10
       ;;
   esac
 done
 
 # Now retrieve the exit code and exit with it
 EXIT_CODE=$(sacct -j "$JOBID" --noheader --format=ExitCode | head -1 | cut -d: -f1)
-echo "🔚 SLURM job $JOBID exit code: $EXIT_CODE"
+echo "Completed: SLURM job $JOBID exit code: $EXIT_CODE"
 exit "$EXIT_CODE"

.github/workflows/phoenix/submit.sh

@@ -62,7 +62,7 @@ JOBID=$(sbatch <<-EOT | awk '{print $4}'
 EOT
 )
 
-echo "🚀 Submitted SLURM job $JOBID"
+echo "Submitted: SLURM job $JOBID"
 
 # if this wrapper is killed/canceled, make sure SLURM job is cleaned up
 trap '[[ -n "${JOBID:-}" ]] && scancel "$JOBID" >/dev/null 2>&1 || :' EXIT
@@ -79,22 +79,22 @@ while :; do
 
   # If it’s one of SLURM’s terminal states, break immediately
   case "$STATE" in
-    COMPLETED|FAILED|CANCELLED|TIMEOUT)
-      echo "✅ SLURM job $JOBID reached terminal state: $STATE"
+    COMPLETED|FAILED|CANCELLED|TIMEOUT|PREEMPTED)
+      echo "Completed: SLURM job $JOBID reached terminal state: $STATE"
       break
       ;;
     "")
-      echo "✅ SLURM job $JOBID no longer in queue; assuming finished"
+      echo "Completed: SLURM job $JOBID no longer in queue; assuming finished"
       break
       ;;
     *)
-      echo "⏳ SLURM job $JOBID state: $STATE"
+      echo "Waiting: SLURM job $JOBID state: $STATE"
       sleep 10
       ;;
   esac
 done
 
 # Now retrieve the exit code and exit with it
 EXIT_CODE=$(sacct -j "$JOBID" --noheader --format=ExitCode | head -1 | cut -d: -f1)
-echo "🔚 SLURM job $JOBID exit code: $EXIT_CODE"
+echo "Completed: SLURM job $JOBID exit code: $EXIT_CODE"
 exit "$EXIT_CODE"

CMakeLists.txt

@@ -135,17 +135,17 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
         add_compile_options(
             -Wall
-            -Wextra	
+            -Wextra
             -fcheck=all,no-array-temps
             -fbacktrace
             -fimplicit-none
             -fsignaling-nans
             -finit-real=snan
             -finit-integer=-99999999
-            -Wintrinsic-shadow	
-            -Wunderflow	
-            -Wrealloc-lhs	
-            -Wsurprising	
+            -Wintrinsic-shadow
+            -Wunderflow
+            -Wrealloc-lhs
+            -Wsurprising
 	    )
     endif()
 
@@ -163,6 +163,7 @@ elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
         "SHELL:-h acc_model=auto_async_none"
         "SHELL: -h acc_model=no_fast_addr"
         "SHELL: -h list=adm"
+        "SHELL: -munsafe-fp-atomics" # Not unsafe for operations we do
     )
 
     add_link_options("SHELL:-hkeepfiles")
@@ -172,7 +173,6 @@ elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
                 "SHELL:-h acc_model=auto_async_none"
                 "SHELL: -h acc_model=no_fast_addr"
                 "SHELL: -K trap=fp" "SHELL: -G2"
-
         )
         add_link_options("SHELL: -K trap=fp" "SHELL: -G2")
     endif()
@@ -200,10 +200,10 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
         add_compile_options(
             $<$<COMPILE_LANGUAGE:Fortran>:-O0>
-            $<$<COMPILE_LANGUAGE:Fortran>:-C> 
+            $<$<COMPILE_LANGUAGE:Fortran>:-C>
             $<$<COMPILE_LANGUAGE:Fortran>:-g>
-            $<$<COMPILE_LANGUAGE:Fortran>:-traceback> 
-            $<$<COMPILE_LANGUAGE:Fortran>:-Minform=inform> 
+            $<$<COMPILE_LANGUAGE:Fortran>:-traceback>
+            $<$<COMPILE_LANGUAGE:Fortran>:-Minform=inform>
             $<$<COMPILE_LANGUAGE:Fortran>:-Mbounds>
         )
     endif()

README.md

@@ -155,6 +155,8 @@ They are organized below.
 * Runge-Kutta orders 1-3 (SSP TVD), adaptive time stepping
 * RK4-5 operator splitting for Euler-Lagrange modeling
 * Interface sharpening (THINC-like)
+* Information geometric regularization (IGR)
+    * Shock capturing without WENO and Riemann solvers
 
 ### Large-scale and accelerated simulation
 

benchmarks/5eq_rk3_weno3_hllc/case.py

@@ -188,8 +188,8 @@
             "cyl_coord": "F",
             "dt": dt,
             "t_step_start": 0,
-            "t_step_stop": int(30 * (95 * size + 5)),
-            "t_step_save": int(30 * (95 * size + 5)),
+            "t_step_stop": int(20 * (5 * size + 5)),
+            "t_step_save": int(20 * (5 * size + 5)),
             # Simulation Algorithm Parameters
             "num_patches": 3,
             "model_eqns": 2,

benchmarks/hypo_hll/case.py

@@ -41,8 +41,8 @@
             "p": Nz,
             "dt": 1e-8,
             "t_step_start": 0,
-            "t_step_stop": int(30 * (95 * size + 5)),
-            "t_step_save": int(30 * (95 * size + 5)),
+            "t_step_stop": int(20 * (5 * size + 5)),
+            "t_step_save": int(20 * (5 * size + 5)),
             # Simulation Algorithm Parameters
             "num_patches": 2,
             "model_eqns": 2,