Merge branch 'MFlowCode:master' into source

Author: mrodrig6

.github/workflows/bench.yml

@@ -52,8 +52,8 @@ jobs:
 
       - name: Generate & Post Comment
         run: |
-          . ./mfc.sh load -c p -m g
-          ./mfc.sh bench_diff master/bench-${{ matrix.device }}.yaml pr/bench-${{ matrix.device }}.yaml
+          (cd pr && . ./mfc.sh load -c p -m g)
+          (cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}.yaml ../pr/bench-${{ matrix.device }}.yaml)
 
       - name: Archive Logs
         uses: actions/upload-artifact@v3

CMakeLists.txt

@@ -172,16 +172,13 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_
     add_compile_options(
         $<$<COMPILE_LANGUAGE:Fortran>:-Mfreeform>
         $<$<COMPILE_LANGUAGE:Fortran>:-cpp>
-        -Minfo=accel
+	    $<$<COMPILE_LANGUAGE:Fortran>:-Minfo=inline>
+        $<$<COMPILE_LANGUAGE:Fortran>:-Minfo=accel>
     )
 
-    if (CMAKE_BUILD_TYPE STREQUAL "Release")
-        add_compile_options(
-            $<$<COMPILE_LANGUAGE:Fortran:-minline>
-        )
-    elseif (CMAKE_BUILD_TYPE STREQUAL "Debug")
+    if (CMAKE_BUILD_TYPE STREQUAL "Debug")
         add_compile_options(
-            $<$<COMPILE_LANGUAGE:Fortran:-O0>
+            $<$<COMPILE_LANGUAGE:Fortran>:-O0>
         )
     endif()
 
@@ -208,13 +205,22 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release")
     endif()
 
     # Enable LTO/IPO if supported
-    CHECK_IPO_SUPPORTED(RESULT SUPPORTS_IPO OUTPUT IPO_ERROR)
-    if (SUPPORTS_IPO)
-        message(STATUS "Enabled IPO / LTO")
-        set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+    if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC")
+        if (MFC_Unified)
+            message(STATUS "IPO is not available with NVHPC using Unified Memory")
+        else()
+            message(STATUS "Performing IPO using -Mextract followed by -Minline")
+            set(NVHPC_USE_TWO_PASS_IPO TRUE)
+        endif()
     else()
-        message(STATUS "IPO / LTO is NOT available")
-    endif()
+        CHECK_IPO_SUPPORTED(RESULT SUPPORTS_IPO OUTPUT IPO_ERROR)
+        if (SUPPORTS_IPO)
+            message(STATUS "Enabled IPO / LTO")
+            set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+	    else()
+            message(STATUS "IPO / LTO is NOT available")
+        endif()
+    endif() 
 endif()
 
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -365,124 +371,139 @@ function(MFC_SETUP_TARGET)
     cmake_parse_arguments(ARGS "OpenACC;MPI;SILO;HDF5;FFTW" "TARGET" "SOURCES" ${ARGN})
 
     add_executable(${ARGS_TARGET} ${ARGS_SOURCES})
-
-    set_target_properties(${ARGS_TARGET} PROPERTIES Fortran_PREPROCESS ON)
-
-    target_include_directories(${ARGS_TARGET} PRIVATE
-        "${CMAKE_SOURCE_DIR}/src/common"
-        "${CMAKE_SOURCE_DIR}/src/common/include"
-        "${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}")
-
-    if (EXISTS "${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}/include")
-        target_include_directories(${ARGS_TARGET} PRIVATE
-            "${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}/include")
+    set(IPO_TARGETS ${ARGS_TARGET})
+    # Here we need to split into "library" and "executable" to perform IPO on the NVIDIA compiler.
+    # A little hacky, but it *is* an edge-case for *one* compiler.
+    if (NVHPC_USE_TWO_PASS_IPO)
+        add_library(${ARGS_TARGET}_lib OBJECT ${ARGS_SOURCES})
+        target_compile_options(${ARGS_TARGET}_lib PRIVATE 
+		    $<$<COMPILE_LANGUAGE:Fortran>:-Mextract=lib:${ARGS_TARGET}_lib>
+	        $<$<COMPILE_LANGUAGE:Fortran>:-Minline>
+    	)
+        add_dependencies(${ARGS_TARGET} ${ARGS_TARGET}_lib)
+        target_compile_options(${ARGS_TARGET} PRIVATE -Minline=lib:${ARGS_TARGET}_lib)
+        list(PREPEND IPO_TARGETS ${ARGS_TARGET}_lib)
     endif()
 
-    string(TOUPPER "${ARGS_TARGET}" ${ARGS_TARGET}_UPPER)
-    target_compile_definitions(
-        ${ARGS_TARGET} PRIVATE MFC_${CMAKE_Fortran_COMPILER_ID}
-                               MFC_${${ARGS_TARGET}_UPPER}
-    )
+    foreach (a_target ${IPO_TARGETS})
+        set_target_properties(${a_target} PROPERTIES Fortran_PREPROCESS ON)
 
-    if (MFC_MPI AND ARGS_MPI)
-        find_package(MPI COMPONENTS Fortran REQUIRED)
+        target_include_directories(${a_target} PRIVATE
+            "${CMAKE_SOURCE_DIR}/src/common"
+            "${CMAKE_SOURCE_DIR}/src/common/include"
+            "${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}")
 
-        target_compile_definitions(${ARGS_TARGET} PRIVATE MFC_MPI)
-        target_link_libraries     (${ARGS_TARGET} PRIVATE MPI::MPI_Fortran)
-    endif()
+        if (EXISTS "${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}/include")
+            target_include_directories(${a_target} PRIVATE
+                "${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}/include")
+        endif()
 
-    if (ARGS_SILO)
-        find_package(SILO REQUIRED)
-        target_link_libraries(${ARGS_TARGET} PRIVATE SILO::SILO)
-    endif()
+        string(TOUPPER "${ARGS_TARGET}" ${ARGS_TARGET}_UPPER)
+        target_compile_definitions(
+            ${a_target} PRIVATE MFC_${CMAKE_Fortran_COMPILER_ID}
+                                MFC_${${ARGS_TARGET}_UPPER}
+        )
 
-    if (ARGS_HDF5)
-        find_package(HDF5 REQUIRED)
-        target_link_libraries(${ARGS_TARGET} PRIVATE HDF5::HDF5)
-    endif()
+        if (MFC_MPI AND ARGS_MPI)
+            find_package(MPI COMPONENTS Fortran REQUIRED)
 
-    if (ARGS_FFTW)
-        if (MFC_OpenACC AND ARGS_OpenACC)
-            if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
-                find_package(CUDAToolkit REQUIRED)
-                target_link_libraries(${ARGS_TARGET} PRIVATE CUDA::cudart CUDA::cufft)
-            else()
-                find_package(hipfort COMPONENTS hipfft CONFIG REQUIRED)
-                target_link_libraries(${ARGS_TARGET} PRIVATE hipfort::hipfft)
-            endif()
-        else()
-            find_package(FFTW REQUIRED)
-            target_link_libraries(${ARGS_TARGET} PRIVATE FFTW::FFTW)
+            target_compile_definitions(${a_target} PRIVATE MFC_MPI)
+            target_link_libraries     (${a_target} PRIVATE MPI::MPI_Fortran)
         endif()
-    endif()
 
-    if (MFC_OpenACC AND ARGS_OpenACC)
-        find_package(OpenACC)
+        if (ARGS_SILO)
+            find_package(SILO REQUIRED)
+            target_link_libraries(${a_target} PRIVATE SILO::SILO)
+        endif()
 
-        # This should be equivalent to if (NOT OpenACC_FC_FOUND)
-        if (NOT TARGET OpenACC::OpenACC_Fortran)
-            message(FATAL_ERROR "OpenACC + Fortran is unsupported.")
+        if (ARGS_HDF5)
+            find_package(HDF5 REQUIRED)
+            target_link_libraries(${a_target} PRIVATE HDF5::HDF5)
         endif()
 
-        target_link_libraries(${ARGS_TARGET} PRIVATE OpenACC::OpenACC_Fortran)
-        target_compile_definitions(${ARGS_TARGET} PRIVATE MFC_OpenACC)
-
-        if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-            # FIXME: This should work with other cards than gfx90a ones.
-            target_compile_options(${ARGS_TARGET} PRIVATE
-                "-foffload=amdgcn-amdhsa='-march=gfx90a'"
-                "-foffload-options=-lgfortran\ -lm"
-                "-fno-exceptions")
-        elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
-            find_package(cuTENSOR)
-            if (NOT cuTENSOR_FOUND)
-                message(WARNING
-                    "Failed to locate the NVIDIA cuTENSOR library. MFC will be "
-                    "built without support for it, disallowing the use of "
-                    "cu_tensor=T. This can result in degraded performance.")
+        if (ARGS_FFTW)
+            if (MFC_OpenACC AND ARGS_OpenACC)
+                if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
+                    find_package(CUDAToolkit REQUIRED)
+                    target_link_libraries(${a_target} PRIVATE CUDA::cudart CUDA::cufft)
+                else()
+                    find_package(hipfort COMPONENTS hipfft CONFIG REQUIRED)
+                    target_link_libraries(${a_target} PRIVATE hipfort::hipfft)
+                endif()
             else()
-                target_link_libraries     (${ARGS_TARGET} PRIVATE cuTENSOR::cuTENSOR)
-                target_compile_definitions(${ARGS_TARGET} PRIVATE MFC_cuTENSOR)
+                find_package(FFTW REQUIRED)
+                target_link_libraries(${a_target} PRIVATE FFTW::FFTW)
             endif()
+        endif()
 
-            foreach (cc ${MFC_CUDA_CC})
-                target_compile_options(${ARGS_TARGET}
-                    PRIVATE -gpu=cc${cc}
-                )
-            endforeach()
-
-            target_compile_options(${ARGS_TARGET}
-                PRIVATE -gpu=keep,ptxinfo,lineinfo
-            )
+        if (MFC_OpenACC AND ARGS_OpenACC)
+            find_package(OpenACC)
 
-            # GH-200 Unified Memory Support
-            if (MFC_Unified)
-                target_compile_options(${ARGS_TARGET}
-                    PRIVATE -gpu=unified
-                )
-                # "This option must appear in both the compile and link lines" -- NVHPC Docs
-                target_link_options(${ARGS_TARGET}
-                    PRIVATE -gpu=unified
-                )
+            # This should be equivalent to if (NOT OpenACC_FC_FOUND)
+            if (NOT TARGET OpenACC::OpenACC_Fortran)
+                message(FATAL_ERROR "OpenACC + Fortran is unsupported.")
             endif()
 
-            if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-                target_compile_options(${ARGS_TARGET}
-                    PRIVATE -gpu=autocompare,debug
+            target_link_libraries(${a_target} PRIVATE OpenACC::OpenACC_Fortran)
+            target_compile_definitions(${a_target} PRIVATE MFC_OpenACC)
+
+            if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+                # FIXME: This should work with other cards than gfx90a ones.
+                target_compile_options(${a_target} PRIVATE
+                    "-foffload=amdgcn-amdhsa='-march=gfx90a'"
+                    "-foffload-options=-lgfortran\ -lm"
+                    "-fno-exceptions")
+            elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
+                find_package(cuTENSOR)
+                if (NOT cuTENSOR_FOUND)
+                    message(WARNING
+                        "Failed to locate the NVIDIA cuTENSOR library. MFC will be "
+                        "built without support for it, disallowing the use of "
+                        "cu_tensor=T. This can result in degraded performance.")
+                else()
+                    target_link_libraries     (${a_target} PRIVATE cuTENSOR::cuTENSOR)
+                    target_compile_definitions(${a_target} PRIVATE MFC_cuTENSOR)
+                endif()
+
+                foreach (cc ${MFC_CUDA_CC})
+                    target_compile_options(${a_target}
+                        PRIVATE -gpu=cc${cc}
+                    )
+                endforeach()
+
+                target_compile_options(${a_target}
+                    PRIVATE -gpu=keep,ptxinfo,lineinfo
                 )
+
+                # GH-200 Unified Memory Support
+                if (MFC_Unified)
+                    target_compile_options(${ARGS_TARGET}
+                        PRIVATE -gpu=unified
+                    )
+                    # "This option must appear in both the compile and link lines" -- NVHPC Docs
+                    target_link_options(${ARGS_TARGET}
+                        PRIVATE -gpu=unified
+                    )
+                endif()
+
+                if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+                    target_compile_options(${a_target}
+                        PRIVATE -gpu=autocompare,debug
+                    )
+                endif()
+            elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
+                find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
+                target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn)
             endif()
-        elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
-            find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
-            target_link_libraries(${ARGS_TARGET} PRIVATE hipfort::hip hipfort::hipfort-amdgcn)
+        elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
+            target_compile_options(${a_target} PRIVATE "SHELL:-h noacc" "SHELL:-x acc")
         endif()
-    elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
-        target_compile_options(${ARGS_TARGET} PRIVATE "SHELL:-h noacc" "SHELL:-x acc")
-    endif()
 
-    if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
-        find_package(CUDAToolkit REQUIRED)
-        target_link_libraries(${ARGS_TARGET} PRIVATE CUDA::nvToolsExt)
-    endif()
+        if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
+            find_package(CUDAToolkit REQUIRED)
+            target_link_libraries(${a_target} PRIVATE CUDA::nvToolsExt)
+        endif()
+    endforeach()
 
     install(TARGETS ${ARGS_TARGET} RUNTIME DESTINATION bin)
 endfunction()

README.md

@@ -24,11 +24,11 @@
 
 Welcome to the home of MFC!
 MFC simulates compressible multi-component and multi-phase flows, [amongst other things](#what-else-can-this-thing-do). 
-MFC is written in Fortran and makes use of metaprogramming to keep the code short (about 20K lines).
+MFC is written in Fortran and uses metaprogramming to keep the code short (about 20K lines).
 
 MFC is used on the latest leadership-class supercomputers.
 It scales <b>ideally to exascale</b>; [tens of thousands of GPUs on NVIDIA- and AMD-GPU machines](#is-this-really-exascale) on Oak Ridge Summit and Frontier.
-MFC is a SPEChpc benchmark candidate, part of the JSC JUPITER Early Access Program, and made use of OLCF Frontier and LLNL El Capitan early access systems.
+MFC is a SPEChpc benchmark candidate, part of the JSC JUPITER Early Access Program, and used OLCF Frontier and LLNL El Capitan early access systems.
 
 Get in touch with <a href="mailto:[email protected]">Spencer</a> if you have questions!
 We have an [active Slack channel](https://join.slack.com/t/mflowcode/shared_invite/zt-y75wibvk-g~zztjknjYkK1hFgCuJxVw) and development team.
@@ -56,7 +56,7 @@ Another example is the high-Mach flow over an airfoil, shown below.
 You can navigate [to this webpage](https://mflowcode.github.io/documentation/md_getting-started.html) to get started using MFC!
 It's rather straightforward.
 We'll give a brief intro. here for MacOS.
-Using [brew](https://brew.sh), install MFC's modest set of dependencies:
+Using [brew](https://brew.sh), install MFC's dependencies:
 ```shell
 brew install wget python cmake gcc@14 mpich
 ```
@@ -82,15 +82,15 @@ You can learn more about MFC's capabilities [via its documentation](https://mflo
 
 The shock-droplet interaction case above was run via
 ```shell
-./mfc.sh run ./examples/3d_shockdroplet/case.py -n 8
+./mfc.sh run -n $(nproc) ./examples/3d_shockdroplet/case.py 
 ```
 where `8` is the number of cores the example will run on.
-You can visualize the output data in `examples/3d_shockdroplet/silo_hdf5` via Paraview, Visit, or your other favorite software.
+You can visualize the output data in `examples/3d_shockdroplet/silo_hdf5` via Paraview, Visit, or your favorite software.
 
 ## Is this _really_ exascale?
 
 [OLCF Frontier](https://www.olcf.ornl.gov/frontier/) is the first exascale supercomputer.
-The weak scaling of MFC on this machine is below, showing near-ideal utilization. 
+The weak scaling of MFC on this machine shows near-ideal utilization. 
 
 <p align="center">
     <img src="docs/res/scaling.png" alt="Scaling" width="400"/>
@@ -158,7 +158,7 @@ They are organized below. Just click the drop-downs!
 * [Fypp](https://fypp.readthedocs.io/en/stable/fypp.html) metaprogramming for code readability, performance, and portability
 * Continuous Integration (CI)
 	* \>100 Regression tests with each PR.
- 		* Performed with GNU, Intel, and NVIDIA compilers on NVIDIA and AMD GPUs.
+ 		* Performed with GNU (GCC), Intel, Cray (CCE), and NVIDIA (NVHPC) compilers on NVIDIA and AMD GPUs.
 		* Line-level test coverage reports via [Codecov](https://app.codecov.io/gh/MFlowCode/MFC) and `gcov`
 	* Benchmarking to avoid performance regressions and identify speed-ups
 * Continuous Deployment (CD) of [website](https://mflowcode.github.io) and [API documentation](https://mflowcode.github.io/documentation/index.html)
@@ -201,7 +201,7 @@ If you use MFC, consider citing it as:
 
 ## License
 
-Copyright 2021-2024 Spencer Bryngelson and Tim Colonius.
+Copyright 2021 Spencer Bryngelson and Tim Colonius.
 MFC is under the MIT license (see [LICENSE](LICENSE) for full text).
 
 ## Acknowledgements
@@ -210,6 +210,7 @@ Multiple federal sponsors have supported MFC development, including the US Depar
 
 MFC computations have used many supercomputing systems. A partial list is below
   * OLCF Frontier and Summit, and testbed systems Wombat, Crusher, and Spock (allocation CFD154, PI Bryngelson)
+  * LLNL Lassen and El Capitan testbed system, Tioga
   * PSC Bridges(1/2), NCSA Delta, SDSC Comet and Expanse, Purdue Anvil, TACC Stampede(1-3), and TAMU ACES via ACCESS-CI (allocations TG-CTS120005 (PI Colonius) and TG-PHY210084 (PI Bryngelson))
   * DOD systems Onyx, Carpenter, and Nautilus via the DOD HPCMP program
   * Sandia National Labs systems Doom and Attaway and testbed systems Weaver and Vortex