Merge branch 'MFlowCode:master' into source

Author: mrodrig6

.github/workflows/frontier/build.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 . ./mfc.sh load -c f -m g
-./mfc.sh build -j 8 --gpu
+./mfc.sh build -j 8 --gpu --sys-hdf5 --sys-fftw

.github/workflows/frontier/test.sh

@@ -1,3 +1,6 @@
 #!/bin/bash
 
-./mfc.sh test -j 4 -a -- -c frontier
+gpus=`rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' '`
+ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
+
+./mfc.sh test -j $ngpus --sys-hdf5 --sys-fftw -- -c frontier

.github/workflows/spelling.yml

@@ -11,6 +11,4 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Spell Check
-      uses: crate-ci/typos@master
-      with:
-        config: .typos.toml
+      run:  ./mfc.sh spelling

.gitignore

@@ -3,6 +3,7 @@ package.json
 yarn.lock
 docker-compose.yml
 
+.venv/
 /build/
 .vscode/
 src/*/include/case.fpp
@@ -35,26 +36,27 @@ docs/documentation/*-example.png
 docs/documentation/examples.md
 
 examples/*batch/*/
-examples/*/D/*
-examples/*/p*
-examples/*/D_*
-examples/*/*.inf
-examples/*/*.inp
-examples/*/*.dat
-examples/*/*.o*
-examples/*/silo*
-examples/*/restart_data*
-examples/*/*.out
-examples/*/binary
-examples/*/fort.1
-examples/*/*.sh
-examples/*/*.err
-examples/*/viz/
+examples/**/D/*
+examples/**/p*
+examples/**/D_*
+examples/**/*.inf
+examples/**/*.inp
+examples/**/*.dat
+examples/**/*.o*
+examples/**/silo*
+examples/**/restart_data*
+examples/**/*.out
+examples/**/binary
+examples/**/fort.1
+examples/**/*.sh
+examples/**/*.err
+examples/**/viz/
 examples/*.jpg
 examples/*.png
 examples/*/workloads/
 examples/*/run-*/
 examples/*/logs/
+examples/**/*.f90
 workloads/
 
 benchmarks/*batch/*/

.vscode/settings.json

@@ -7,8 +7,8 @@
         "**/.git":      true,
         "**/.DS_Store": true,
         "**/build":     true,
-        "*.o":          true,
-        "*.mod":        true
+        "**/*.o":       true,
+        "**/*.mod":     true,
     },
 
     "cmake.configureOnOpen": false,
@@ -17,7 +17,7 @@
 
     "files.associations": {
         "*.f90": "FortranFreeForm",
-        "*.fpp": "FortranFreeForm"
+        "*.fpp": "FortranFreeForm",
     },
 
     "fortran.preferredCase":       "lowercase",
@@ -27,5 +27,5 @@
         "${workspacefolder}/src/common",
         "${workspacefolder}/src/common/include"
     ],
-    "fortran.linter.extraArgs":    ["--free-form"]
+    "fortran.linter.extraArgs":    ["--free-form"],
 }

CMakeLists.txt

@@ -55,24 +55,25 @@ compiler (or a different version thereof), please:\n\
  - Install the compiler or load its module. (e.g. module load gcc/10.1)\n\
  - Set/Export the C, CXX, and FC environment variables. (e.g. 'export CC=gcc', \
  'export CXX=g++', and 'export FC=gfortran'.\n\
- - If using mfc.sh, delete the build/<code name> directory and try again. (e.g. 'rm -rf build/pre_process')\n")
+ - If using mfc.sh, delete the build/<code name> directory and try again. (e.g. 'rm -rf build/pre_process')")
 
 if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
     if (CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 5)
-        message(FATAL_ERROR "${__err_msg}ERROR: GNU v5.0 or newer is required to build MFC.")
+        message(FATAL_ERROR "ERROR: GNU v5.0 or newer is required to build MFC.\n${__err_msg}")
     endif()
     if (MFC_OpenACC)
-        message(FATAL_ERROR "${__err_msg}ERROR: MFC with GPU processing is not currently compatible with GNU compilers. Please use NVIDIA or Cray compilers.")
+        message(FATAL_ERROR "ERROR: MFC with GPU processing is not currently compatible with GNU compilers. Please use NVIDIA or Cray compilers.\n${__err_msg}")
     endif()
 elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_ID STREQUAL "PGI"))
     if (CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 21.7)
-        message(FATAL_ERROR "${__err_msg}ERROR: When using NVHPC, v21.7 or newer is required to build MFC.")
+        message(FATAL_ERROR "ERROR: When using NVHPC, v21.7 or newer is required to build MFC.\n${__err_msg}")
     endif()
-    if ((CMAKE_BUILD_TYPE STREQUAL "Debug") AND MFC_OpenACC)
-        message(FATAL_ERROR "${__err_msg}ERROR: When using NVHPC, MFC with Debug and GPU options is unavailable.")
+
+    if ((CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 23.11) AND (CMAKE_BUILD_TYPE STREQUAL "Debug") AND MFC_OpenACC)
+        message(FATAL_ERROR "ERROR: When using NVHPC, MFC with Debug and GPU options requires NVHPC v23.11 or newer.\n${__err_msg}")
     endif()
 elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_C_COMPILER_ID STREQUAL "AppleClang")
-    message(FATAL_ERROR "${__err_msg}ERROR: MFC does not support the Apple Clang compilers. Please consult the documentation.")
+    message(FATAL_ERROR "ERROR: MFC does not support the Apple Clang compilers. Please consult the documentation.\n${__err_msg}")
 endif()
 
 # Fypp is required to build MFC. We try to locate it. The path to the binary is
@@ -127,13 +128,15 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
             -fbacktrace
             -fimplicit-none
             #-ffpe-trap=invalid,zero,denormal,overflow
-        )
+            -fsignaling-nans
+	    )
     endif()
 
     if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER 10)
         add_compile_options(
             $<$<COMPILE_LANGUAGE:Fortran>:-fallow-invalid-boz>
             $<$<COMPILE_LANGUAGE:Fortran>:-fallow-argument-mismatch>
+            $<$<COMPILE_LANGUAGE:Fortran>:-fcheck=bounds>            
         )
     endif()
 elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
@@ -183,7 +186,7 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_
     endif()
 
     if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-        add_compile_options(-C -g -O0 -traceback -Mchkptr -Minform=inform -Mbounds)
+        add_compile_options(-C -g -O0 -traceback -Minform=inform -Mbounds)
     endif()
 
     if (DEFINED ENV{MFC_CUDA_CC})

README.md

@@ -84,7 +84,7 @@ The shock-droplet interaction case above was run via
 ```shell
 ./mfc.sh run -n $(nproc) ./examples/3d_shockdroplet/case.py 
 ```
-where `8` is the number of cores the example will run on.
+where `$(nproc)` is the number of cores the example will run on (and the number of physical cores on your CPU device).
 You can visualize the output data in `examples/3d_shockdroplet/silo_hdf5` via Paraview, Visit, or your favorite software.
 
 ## Is this _really_ exascale?

docs/documentation/expectedPerformance.md

@@ -30,28 +30,31 @@ These are reported as (X/Y cores), where X is the used cores, and Y is the total
 | NVIDIA A30                |                           | GPU           | 1 GPU           | 1.1              | NVHPC 24.1           | GT Rogues Gallery  |
 | AMD MI250X                |                           | GPU           | 1 _GCD_*        | 1.1              | CCE 16.0.1           | OLCF Frontier |
 | AMD MI100                 |                           | GPU           | 1 GPU           | 1.4              | CCE 16.0.1           | Cray internal system |
-| NVIDIA L40S               | Single-precision GPU                    | GPU           | 1 GPU           | 1.7              | NVHPC 24.5           | GT ICE  |
+| NVIDIA L40S               | Single-precision GPU      | GPU           | 1 GPU           | 1.7              | NVHPC 24.5           | GT ICE  |
+| AMD EPYC 9654             | Genoa                     | CPU           | 96/96 cores     | 1.7              | Intel oneAPI 2021.9  | DOD Carpenter  |
 | NVIDIA P100               |                           | GPU           | 1 GPU           | 2.4              | NVHPC 23.5           | GT CSE Internal  |
 | AMD EPYC 9534             | Genoa                     | CPU           | 64/64 cores     | 2.7              | GNU 12.3.0           | GT Phoenix  |
 | NVIDIA A40                | Single-precision GPU      | GPU           | 1 GPU           | 3.3              | NVHPC 22.11          | NCSA Delta  |
 | NVIDIA Grace CPU          | Arm, Neoverse V2          | CPU           | 72/72 cores     | 3.7              | NVHPC 24.1           | GT Rogues Gallery  |
-| NVIDIA RTX6000            | Single-precision GPU                    | GPU           | 1 GPU           | 3.9              | NVHPC 22.11          | GT Phoenix  |
+| NVIDIA RTX6000            | Single-precision GPU      | GPU           | 1 GPU           | 3.9              | NVHPC 22.11          | GT Phoenix  |
 | AMD EPYC 7763             | Milan                     | CPU           | 64/64 cores     | 4.1              | GNU 11.4.0           | NCSA Delta  |
 | AMD EPYC 7713             | Milan                     | CPU           | 64/64 cores     | 5.0              | GNU 12.3.0           | GT Phoenix  |
-| Intel Xeon 8480CL         | Platinum, Sapphire Rapids | CPU           | 56/56 cores     | 5.0              | NVHPC 24.5           | GT Phoenix  |
-| Intel Xeon 6454S          | Gold, Sapphire Rapids     | CPU           | 32/32 cores     | 5.6              | NVHPC 24.5           | GT Rogues Gallery  |
-| Intel Xeon 8462Y+         | Platinum, Sapphire Rapids | CPU           | 32/32 cores     | 6.2              | GNU 12.3.0           | GT ICE  |
-| Intel Xeon 6548Y+         | Gold, Emerald Rapids      | CPU           | 32/32 cores     | 6.6              | Intel oneAPI 2021.9  | GT ICE  |
-| Intel Xeon 8352Y          | Platinum, Ice Lake        | CPU           | 32/32 cores     | 6.6              | NVHPC 24.5           | GT Rogues Gallery  |
+| Intel Xeon 8480CL         | Sapphire Rapids           | CPU           | 56/56 cores     | 5.0              | NVHPC 24.5           | GT Phoenix  |
+| Intel Xeon 6454S          | Sapphire Rapids           | CPU           | 32/32 cores     | 5.6              | NVHPC 24.5           | GT Rogues Gallery  |
+| Intel Xeon 8462Y+         | Sapphire Rapids           | CPU           | 32/32 cores     | 6.2              | GNU 12.3.0           | GT ICE  |
+| Intel Xeon 6548Y+         | Emerald Rapids            | CPU           | 32/32 cores     | 6.6              | Intel oneAPI 2021.9  | GT ICE  |
+| Intel Xeon 8352Y          | Ice Lake                  | CPU           | 32/32 cores     | 6.6              | NVHPC 24.5           | GT Rogues Gallery  |
 | Ampere Altra Q80-28       | Arm, Neoverse-N1          | CPU           | 80/80 cores     | 6.8              | GNU 12.2.0           | OLCF Wombat  | 
 | AMD EPYC 7513             | Milan                     | CPU           | 32/32 cores     | 7.4              | GNU 12.3.0           | GT ICE  |
 | AMD EPYC 7452             | Rome                      | CPU           | 32/32 cores     | 8.4              | GNU 12.3.0           | GT ICE  |
+| IBM Power10               |                           | CPU           | 24/24 cores     | 10               | GNU 13.3.1           | GT Rogues Gallery |
 | AMD EPYC 7401             | Naples                    | CPU           | 24/24 cores     | 10               | GNU 10.3.1           | LLNL Corona  |
 | Apple M1 Pro              |                           | CPU           |  8/10 cores     | 14               | GNU 13.2.0           | N/A     |
-| Intel Xeon Gold 6226      | Cascade Lake              | CPU           | 12/12 cores     | 17               | GNU 12.3.0           | GT ICE  |
+| Intel Xeon 6226           | Cascade Lake              | CPU           | 12/12 cores     | 17               | GNU 12.3.0           | GT ICE  |
 | Apple M1 Max              |                           | CPU           |  8/10 cores     | 18               | GNU 14.1.0           | N/A     |
 | IBM Power9                |                           | CPU           | 20/21 cores     | 21               | GNU 9.1.0            | OLCF Summit |
 | Intel Xeon E5-2650V4      | Broadwell                 | CPU           | 12/12 cores     | 27               | NVHPC 23.5           | GT CSE Internal  |
+| Intel Xeon E7-4850V3      | Haswell                   | CPU           | 14/14 cores     | 34               | GNU 9.4.0            | GT CSE Internal  |
 
 __All grind times are in nanoseconds (ns) per grid point (gp) per equation (eq) per right-hand side (rhs) evaluation, so X ns/gp/eq/rhs. Lower is better.__
 

examples/3D_sphbubcollapse/case.py

@@ -45,7 +45,7 @@
     # ==========================================================================
 
     # Simulation Algorithm Parameters ==========================================
-    # Only two patches are necesssary, the background liquid and the
+    # Only two patches are necessary, the background liquid and the
     # gas bubble
     'num_patches'                  : 2,
     # Use the 6 equation model

mfc.sh

@@ -38,6 +38,10 @@ elif [ "$1" '==' "venv" ]; then
     shift; . "$(pwd)/toolchain/bootstrap/python.sh"  $@; return
 elif [ "$1" '==' "clean" ]; then
     rm -rf "$(pwd)/build"; exit 0
+elif [ "$1" '==' "spelling" ]; then
+    . "$(pwd)/toolchain/bootstrap/python.sh"
+
+    shift; . "$(pwd)/toolchain/bootstrap/spelling.sh" $@; exit 0
 fi
 
 mkdir -p "$(pwd)/build"