Merge branch 'master' into 2/3D_Extend_IC

Author: sbryngelson

.cursor/rules/mfc-agent-rules.mdc

@@ -0,0 +1,82 @@
+---
+description: Full MFC project rules – consolidated for Agent Mode
+alwaysApply: true
+---
+
+# 0  Purpose & Scope
+Consolidated guidance for the MFC exascale, many-physics solver.  
+Written primarily for Fortran/Fypp; the OpenACC and style sections matter only when
+`.fpp` / `.f90` files are in view.
+
+---
+
+# 1  Global Project Context (always)
+- **Project**: *MFC* is modern Fortran 2008+ generated with **Fypp**.  
+  - Sources `src/`, tests `tests/`, examples `examples/`.  
+  - Most sources are `.fpp`; CMake transpiles them to `.f90`.  
+- **Fypp macros** live in `src/<subprogram>/include/` you should scan these first.  
+  `<subprogram>` ∈ {`simulation`,`common`,`pre_process`,`post_process`}.  
+- Only `simulation` (+ its `common` calls) is GPU-accelerated via **OpenACC**.  
+- Assume free-form Fortran 2008+, `implicit none`, explicit `intent`, and modern
+  intrinsics.  
+- Prefer `module … contains … subroutine foo()`; avoid `COMMON` blocks and
+  file-level `include` files.  
+- **Read the full codebase and docs *before* changing code.**  
+  Docs: <https://mflowcode.github.io/documentation/md_readme.html> and the respository root `README.md`.  
+
+### Incremental-change workflow
+1. Draft a step-by-step plan.  
+2. After each step, build:  
+   ```bash
+   ./mfc.sh build -t pre_process simulation -j $(nproc)
+    ```
+3. If it compiles, run focused tests:
+   ```bash
+   ./mfc.sh test -j $(nproc) -f EA8FA07E -t 9E2CA336
+   ```
+4. Roll back & fix if a step fails.
+
+* Do not run ./mfc.sh test -j $(nproc) without any other arguments (it takes too long to run all tests).
+
+---
+
+# 2  Style & Naming Conventions (for \*.fpp / \*.f90)
+
+* **Indent 2 spaces**; continuation lines align under `&`.
+* Lower-case keywords and intrinsics (`do`, `end subroutine`, …).
+* **Modules**: `m_<feature>` (e.g. `m_transport`).
+* **Public procedures**:
+  * Subroutine → `s_<verb>_<noun>` (e.g. `s_compute_flux`)
+  * Function   → `f_<verb>_<noun>`
+* Private helpers stay in the module; avoid nested procedures.
+* **Size limits**: subroutine ≤ 500 lines, helper ≤ 150, function ≤ 100,
+  module/file ≤ 1000.
+* ≤ 6 arguments per routine; otherwise pass a derived-type “params” struct.
+* No `goto` (except unavoidable legacy); no global state (`COMMON`, `save`).
+* Every variable: `intent(in|out|inout)` + appropriate `dimension` / `allocatable`
+  / `pointer`.
+* Use `s_mpi_abort(<msg>)` for errors, not `stop`.
+* Mark OpenACC-callable helpers that are called from OpenACC parallel loops immediately after declaration:
+  ```fortran
+  subroutine s_flux_update(...)
+    !$acc routine seq
+    ...
+  end subroutine
+  ```
+
+---
+
+# 3  OpenACC Programming Guidelines (for kernels)
+
+Wrap tight loops with
+
+```fortran
+!$acc parallel loop gang vector default(present) reduction(...)
+```
+* Add `collapse(n)` to merge nested loops when safe.
+* Declare loop-local variables with `private(...)`.
+* Allocate large arrays with `managed` or move them into a persistent
+  `!$acc enter data` region at start-up.
+* **Do not** place `stop` / `error stop` inside device code.
+* Must compile with Cray `ftn` and NVIDIA `nvfortran` for GPU offloading; also build CPU-only with
+  GNU `gfortran` and Intel `ifx`/`ifort`.

.github/workflows/frontier/submit.sh

@@ -33,10 +33,9 @@ sbatch <<EOT
 #SBATCH -A CFD154                  # charge account
 #SBATCH -N 1                       # Number of nodes required
 $sbatch_device_opts
-#SBATCH -t 01:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -t 03:59:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -o$job_slug.out            # Combined output and error messages file
 #SBATCH -p extended                # Extended partition for shorter queues
-#SBATCH -q debug                   # Use debug QOS - only one job per user allowed in queue!
 #SBATCH -W                         # Do not exit until the submitted job terminates.
 
 set -e

.github/workflows/phoenix/bench.sh

@@ -8,8 +8,13 @@ if [ "$job_device" == "gpu" ]; then
     device_opts="--gpu -g $gpu_ids"
 fi
 
+mkdir -p /storage/scratch1/6/sbryngelson3/mytmp_build
+export TMPDIR=/storage/scratch1/6/sbryngelson3/mytmp_build
+
 if ["$job_device" == "gpu"]; then
     ./mfc.sh bench --mem 12 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks
 else
     ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks
 fi
+
+unset TMPDIR

.github/workflows/pmd.yml

@@ -0,0 +1,36 @@
+name: Source DRYness
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:    
+  pmd:
+      name: PMD
+      runs-on: "ubuntu-latest"
+      env:
+          pr_everything: 0
+      steps:
+          - name: Clone - PR
+            uses: actions/checkout@v4
+
+          - name: Set up Java
+            uses: actions/setup-java@v4
+            with:
+              distribution: temurin
+              java-version: '17'
+
+          - name: Run CPD for Fortran
+            continue-on-error: true
+            run: |
+              # Get latest PMD version from GitHub API
+              PMD_VERSION=$(curl -s https://api.github.com/repos/pmd/pmd/releases/latest | grep '"tag_name":' | cut -d'"' -f4 | sed 's/pmd_releases\///')
+              echo "Using PMD version: $PMD_VERSION"
+
+              curl -sSL -o pmd.zip \
+                "https://github.com/pmd/pmd/releases/download/pmd_releases/${PMD_VERSION}/pmd-dist-${PMD_VERSION}-bin.zip"
+              unzip -q pmd.zip
+              PMD_HOME="pmd-bin-${PMD_VERSION}"
+          
+              "${PMD_HOME}/bin/pmd" cpd \
+                --dir src \
+                --language fortran \
+                --minimum-tokens=40

CMakeLists.txt

@@ -135,13 +135,17 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
         add_compile_options(
             -Wall
+            -Wextra	
             -fcheck=all,no-array-temps
             -fbacktrace
             -fimplicit-none
-            #-ffpe-trap=invalid,zero,denormal,overflow
             -fsignaling-nans
             -finit-real=snan
             -finit-integer=-99999999
+            -Wintrinsic-shadow	
+            -Wunderflow	
+            -Wrealloc-lhs	
+            -Wsurprising	
 	    )
     endif()
 

misc/length-subroutines.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Use gawk if available, otherwise fall back to awk
+if command -v gawk > /dev/null; then
+    AWK_CMD="gawk"
+    IGNORECASE_BLOCK='BEGIN { IGNORECASE = 1 }'
+else
+    AWK_CMD="awk"
+    IGNORECASE_BLOCK=''
+    echo "Warning: gawk not found. Case-insensitive matching may not work as expected." >&2
+fi
+
+find . -type f \( -name "*.f90" -o -name "*.fpp" \) | while read file; do
+    "$AWK_CMD" "
+    $IGNORECASE_BLOCK
+    /^[ \t]*((pure|elemental|impure)[ \t]+)*subroutine[ \t]+[a-zA-Z_][a-zA-Z0-9_]*[ \t]*\\(/ {
+        in_sub = 1
+        match(\$0, /subroutine[ \t]+([a-zA-Z_][a-zA-Z0-9_]*)/, arr)
+        sub_name = arr[1]
+        start_line = NR
+        next
+    }
+    /^[ \t]*end[ \t]+subroutine[ \t]*([a-zA-Z_][a-zA-Z0-9_]*)?[ \t]*\$/ && in_sub {
+        end_line = NR
+        print (end_line - start_line + 1) \"\t\" FILENAME \": \" sub_name
+        in_sub = 0
+    }
+    " "$file"
+done | sort -nr | awk -F'\t' '{print $2 " : " $1 " lines"}' | head -20

src/common/include/macros.fpp

@@ -18,8 +18,8 @@
 
 #:def DEALLOCATE(*args)
     @:LOG({'@:DEALLOCATE(${re.sub(' +', ' ', ', '.join(args))}$)'})
-    deallocate (${', '.join(args)}$)
     !$acc exit data delete(${', '.join(args)}$)
+    deallocate (${', '.join(args)}$)
 #:enddef DEALLOCATE
 
 #:def ACC_SETUP_VFs(*args)