Merge remote-tracking branch 'upstream/master' into nvidia

Author: wilfonba

.github/workflows/bench.yml

@@ -24,10 +24,7 @@ jobs:
 
   self:
     name: "${{ matrix.name }} (${{ matrix.device }})"
-    if: ${{ github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' && (
-             (github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
-             (github.event_name == 'pull_request' && github.event.pull_request.user.login == 'sbryngelson')
-           ) }}
+    if: ${{ github.repository=='MFlowCode/MFC' && needs.file-changes.outputs.checkall=='true' && ((github.event_name=='pull_request_review' && github.event.review.state=='approved') || (github.event_name=='pull_request' && (github.event.pull_request.user.login=='sbryngelson' || github.event.pull_request.user.login=='wilfonba'))) }}
     needs: file-changes
     strategy:
       fail-fast: false

.typos.toml

@@ -18,6 +18,7 @@ Gam = "Gam"
 strang = "strang"
 Strang = "Strang"
 TKE = "TKE"
+HSA = "HSA"
 
 [files]
 extend-exclude = ["docs/documentation/references*", "tests/", "toolchain/cce_simulation_workgroup_256.sh"]

CMakeLists.txt

@@ -391,9 +391,10 @@ HANDLE_SOURCES(syscheck OFF)
 #  * FFTW     (optional) Should be linked with an FFTW-like library (fftw/cufftw),
 #             depending on whether OpenACC is enabled and which compiler is
 #             being used.
+#  * LAPACK   (optional) Should be linked with LAPACK
 
 function(MFC_SETUP_TARGET)
-    cmake_parse_arguments(ARGS "OpenACC;MPI;SILO;HDF5;FFTW" "TARGET" "SOURCES" ${ARGN})
+    cmake_parse_arguments(ARGS "OpenACC;MPI;SILO;HDF5;FFTW;LAPACK" "TARGET" "SOURCES" ${ARGN})
 
     add_executable(${ARGS_TARGET} ${ARGS_SOURCES})
     set(IPO_TARGETS ${ARGS_TARGET})
@@ -461,6 +462,11 @@ function(MFC_SETUP_TARGET)
             endif()
         endif()
 
+				if (ARGS_LAPACK)
+					find_package(LAPACK REQUIRED)
+					target_link_libraries(${a_target} PRIVATE LAPACK::LAPACK)
+				endif()
+
         if (MFC_OpenACC AND ARGS_OpenACC)
             find_package(OpenACC)
 
@@ -506,6 +512,12 @@ function(MFC_SETUP_TARGET)
                     )
                 endif()
             elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
+                # Frontier Unified Memory Support
+                if (MFC_Unified)
+                    target_compile_options(${ARGS_TARGET}
+                        PRIVATE -DFRONTIER_UNIFIED)
+                endif()
+
                 find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
                 target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn)
             endif()
@@ -541,7 +553,7 @@ endif()
 if (MFC_POST_PROCESS)
     MFC_SETUP_TARGET(TARGET  post_process
                      SOURCES "${post_process_SRCs}"
-                     MPI SILO HDF5 FFTW)
+                     MPI SILO HDF5 FFTW LAPACK)
 
     # -O0 is in response to https://github.com/MFlowCode/MFC-develop/issues/95
     target_compile_options(post_process PRIVATE -O0)

docs/documentation/case.md

@@ -589,6 +589,7 @@ To restart the simulation from $k$-th time step, see [Restarting Cases](running.
 | `omega_wrt(i)`       | Logical | Add the $i$-direction vorticity to the database	 |
 | `schlieren_wrt`      | Logical | Add the numerical schlieren to the database|
 | `qm_wrt`             | Logical | Add the Q-criterion to the database|
+| `liutex_wrt`         | Logical | Add the Liutex to the database|
 | `tau_wrt`            | Logical | Add the elastic stress components to the database|
 | `fd_order`           | Integer | Order of finite differences for computing the vorticity and the numerical Schlieren function [1,2,4] |
 | `schlieren_alpha(i)` | Real    | Intensity of the numerical Schlieren computed via `alpha(i)` |
@@ -628,7 +629,7 @@ If `file_per_process` is true, then pre_process, simulation, and post_process mu
 - `output_partial_domain` activates the output of part of the domain specified by `[x,y,z]_output%beg` and `[x,y,z]_output%end`.
 This is useful for large domains where only a portion of the domain is of interest.
 It is not supported when `precision = 1` and `format = 1`. 
-It also cannot be enabled with `flux_wrt`, `heat_ratio_wrt`, `pres_inf_wrt`, `c_wrt`, `omega_wrt`, `ib`, `schlieren_wrt`, or `qm_wrt`.
+It also cannot be enabled with `flux_wrt`, `heat_ratio_wrt`, `pres_inf_wrt`, `c_wrt`, `omega_wrt`, `ib`, `schlieren_wrt`, `qm_wrt`, or 'liutex_wrt'.
 
 ### 8. Acoustic Source {#acoustic-source}
 

docs/documentation/references.md

@@ -69,3 +69,5 @@
 - <a id="Powell94">Powell, K. G. (1994). An approximate Riemann solver for magnetohydrodynamics: (That works in more than one dimension). In Upwind and high-resolution schemes (pp. 570-583). Springer.</a>
 
 - <a id="Cao19">Cao, S., Zhang, Y., Liao, D., Zhong, P., and Wang, K. G. (2019). Shock-induced damage and dynamic fracture in cylindrical bodies submerged in liquid. International Journal of Solids and Structures, 169:55–71. Elsevier.</a>
+
+- <a id="Xu2019">Xu, W., Gao, Y., Deng, Y., Liu, J., and Liu, C. (2019). An explicit expression for the calculation of the Rortex vector. Physics of Fluids, 31(9)..</a>

examples/3D_turb_mixing/README.md

@@ -0,0 +1,4 @@
+# 3D Turbulent Mixing layer (3D)
+
+## Liutex visualization at transitional state
+<img src='result.png' height='MAX_HEIGHT'/>

examples/3D_turb_mixing/case.py

@@ -96,6 +96,7 @@
             "omega_wrt(2)": "T",
             "omega_wrt(3)": "T",
             "qm_wrt": "T",
+            "liutex_wrt": "T",
             # Patch 1
             "patch_icpp(1)%geometry": 9,
             "patch_icpp(1)%x_centroid": Lx / 2.0,

examples/3D_turb_mixing/result.png

@@ -1626,7 +1626,7 @@ contains
 
     impure subroutine s_create_mpi_types(bc_type)
 
-        type(integer_field), dimension(1:num_dims, -1:1) :: bc_type
+        type(integer_field), dimension(1:num_dims, -1:1), intent(in) :: bc_type
 
 #ifdef MFC_MPI
         integer :: dir, loc
@@ -1659,9 +1659,9 @@ contains
 
     subroutine s_write_serial_boundary_condition_files(q_prim_vf, bc_type, step_dirpath, old_grid_in)
 
-        type(scalar_field), dimension(sys_size) :: q_prim_vf
-        type(integer_field), dimension(1:num_dims, -1:1) :: bc_type
-        logical :: old_grid_in
+        type(scalar_field), dimension(sys_size), intent(in) :: q_prim_vf
+        type(integer_field), dimension(1:num_dims, -1:1), intent(in) :: bc_type
+        logical, intent(in) :: old_grid_in
 
         character(LEN=*), intent(in) :: step_dirpath
 
@@ -1700,8 +1700,8 @@ contains
 
     subroutine s_write_parallel_boundary_condition_files(q_prim_vf, bc_type)
 
-        type(scalar_field), dimension(sys_size) :: q_prim_vf
-        type(integer_field), dimension(1:num_dims, -1:1) :: bc_type
+        type(scalar_field), dimension(sys_size), intent(in) :: q_prim_vf
+        type(integer_field), dimension(1:num_dims, -1:1), intent(in) :: bc_type
 
         integer :: dir, loc
         character(len=path_len) :: file_loc, file_path
@@ -1870,7 +1870,7 @@ contains
 
     subroutine s_pack_boundary_condition_buffers(q_prim_vf)
 
-        type(scalar_field), dimension(sys_size) :: q_prim_vf
+        type(scalar_field), dimension(sys_size), intent(in) :: q_prim_vf
         integer :: i, j, k
 
         do k = 0, p

src/common/m_boundary_common.fpp

@@ -37,7 +37,9 @@ module m_helper
               double_factorial, &
               factorial, &
               f_cut_on, &
-              f_cut_off
+              f_cut_off, &
+              s_downsample_data, &
+              s_upsample_data
 
 contains
 
@@ -625,4 +627,87 @@ contains
 
     end function f_gx
 
+    subroutine s_downsample_data(q_cons_vf, q_cons_temp, m_ds, n_ds, p_ds, m_glb_ds, n_glb_ds, p_glb_ds)
+
+        type(scalar_field), dimension(sys_size), intent(inout) :: q_cons_vf, q_cons_temp
+
+        ! Down sampling variables
+        integer :: i, j, k, l
+        integer :: ix, iy, iz, x_id, y_id, z_id
+        integer, intent(inout) :: m_ds, n_ds, p_ds, m_glb_ds, n_glb_ds, p_glb_ds
+
+        m_ds = int((m + 1)/3) - 1
+        n_ds = int((n + 1)/3) - 1
+        p_ds = int((p + 1)/3) - 1
+
+        m_glb_ds = int((m_glb + 1)/3) - 1
+        n_glb_ds = int((n_glb + 1)/3) - 1
+        p_glb_ds = int((p_glb + 1)/3) - 1
+
+        do i = 1, sys_size
+            $:GPU_UPDATE(host='[q_cons_vf(i)%sf]')
+        end do
+
+        do l = -1, p_ds + 1
+            do k = -1, n_ds + 1
+                do j = -1, m_ds + 1
+                    x_id = 3*j + 1
+                    y_id = 3*k + 1
+                    z_id = 3*l + 1
+                    do i = 1, sys_size
+                        q_cons_temp(i)%sf(j, k, l) = 0
+
+                        do iz = -1, 1
+                            do iy = -1, 1
+                                do ix = -1, 1
+                                    q_cons_temp(i)%sf(j, k, l) = q_cons_temp(i)%sf(j, k, l) &
+                                                                 + (1._wp/27._wp)*q_cons_vf(i)%sf(x_id + ix, y_id + iy, z_id + iz)
+                                end do
+                            end do
+                        end do
+                    end do
+                end do
+            end do
+        end do
+
+    end subroutine s_downsample_data
+
+    subroutine s_upsample_data(q_cons_vf, q_cons_temp)
+
+        type(scalar_field), intent(inout), dimension(sys_size) :: q_cons_vf, q_cons_temp
+        integer :: i, j, k, l
+        integer :: ix, iy, iz
+        integer :: x_id, y_id, z_id
+        real(wp), dimension(4) :: temp
+
+        do l = 0, p
+            do k = 0, n
+                do j = 0, m
+                    do i = 1, sys_size
+
+                        ix = int(j/3._wp)
+                        iy = int(k/3._wp)
+                        iz = int(l/3._wp)
+
+                        x_id = j - int(3*ix) - 1
+                        y_id = k - int(3*iy) - 1
+                        z_id = l - int(3*iz) - 1
+
+                        temp(1) = (2._wp/3._wp)*q_cons_temp(i)%sf(ix, iy, iz) + (1._wp/3._wp)*q_cons_temp(i)%sf(ix + x_id, iy, iz)
+                        temp(2) = (2._wp/3._wp)*q_cons_temp(i)%sf(ix, iy + y_id, iz) + (1._wp/3._wp)*q_cons_temp(i)%sf(ix + x_id, iy + y_id, iz)
+                        temp(3) = (2._wp/3._wp)*temp(1) + (1._wp/3._wp)*temp(2)
+
+                        temp(1) = (2._wp/3._wp)*q_cons_temp(i)%sf(ix, iy, iz + z_id) + (1._wp/3._wp)*q_cons_temp(i)%sf(ix + x_id, iy, iz + z_id)
+                        temp(2) = (2._wp/3._wp)*q_cons_temp(i)%sf(ix, iy + y_id, iz + z_id) + (1._wp/3._wp)*q_cons_temp(i)%sf(ix + x_id, iy + y_id, iz + z_id)
+                        temp(4) = (2._wp/3._wp)*temp(1) + (1._wp/3._wp)*temp(2)
+
+                        q_cons_vf(i)%sf(j, k, l) = (2._wp/3._wp)*temp(3) + (1._wp/3._wp)*temp(4)
+
+                    end do
+                end do
+            end do
+        end do
+
+    end subroutine s_upsample_data
+
 end module m_helper