Merge branch 'master' into move-nvtx

Author: sbryngelson

.github/workflows/phoenix/submit-bench.sh

@@ -21,6 +21,7 @@ sbatch_cpu_opts="\
 
 sbatch_gpu_opts="\
 #SBATCH -CV100
+#SBATCH --ntasks-per-node=4       # Number of cores per node required
 #SBATCH -G2\
 "
 

.github/workflows/phoenix/submit.sh

@@ -21,6 +21,7 @@ sbatch_cpu_opts="\
 
 sbatch_gpu_opts="\
 #SBATCH -p gpu-v100,gpu-a100,gpu-h100,gpu-l40s
+#SBATCH --ntasks-per-node=4       # Number of cores per node required
 #SBATCH -G2\
 "
 

src/common/m_constants.fpp

@@ -24,6 +24,7 @@ module m_constants
     integer, parameter :: num_patches_max = 10
     integer, parameter :: pathlen_max = 400
     integer, parameter :: nnode = 4    !< Number of QBMM nodes
+    integer, parameter :: gp_layers = 3 !< Number of ghost point layers for IBM
     real(wp), parameter :: capillary_cutoff = 1e-6 !< color function gradient magnitude at which to apply the surface tension fluxes
     real(wp), parameter :: acoustic_spatial_support_width = 2.5_wp !< Spatial support width of acoustic source, used in s_source_spatial
     real(wp), parameter :: dflt_vcfl_dt = 100._wp !< value of vcfl_dt when viscosity is off for computing adaptive timestep size

src/common/m_derived_types.fpp

@@ -367,15 +367,13 @@ module m_derived_types
 
     !> Ghost Point for Immersed Boundaries
     type ghost_point
-
-        real(wp), dimension(3) :: loc !< Physical location of the ghost point
+        integer, dimension(3) :: loc !< Physical location of the ghost point
         real(wp), dimension(3) :: ip_loc !< Physical location of the image point
         integer, dimension(3) :: ip_grid !< Top left grid point of IP
         real(wp), dimension(2, 2, 2) :: interp_coeffs !< Interpolation Coefficients of image point
         integer :: ib_patch_id !< ID of the IB Patch the ghost point is part of
         logical :: slip
         integer, dimension(3) :: DB
-
     end type ghost_point
 
     !> Species parameters

src/simulation/m_ibm.fpp

@@ -18,6 +18,8 @@ module m_ibm
 
     use m_helper
 
+    use m_constants
+
     implicit none
 
     private :: s_compute_image_points, &
@@ -39,7 +41,6 @@ module m_ibm
     type(ghost_point), dimension(:), allocatable :: inner_points
     !$acc declare create(ghost_points, inner_points)
 
-    integer :: gp_layers !< Number of ghost point layers
     integer :: num_gps !< Number of ghost points
     integer :: num_inner_gps !< Number of ghost points
     !$acc declare create(gp_layers, num_gps, num_inner_gps)
@@ -49,8 +50,6 @@ contains
     !>  Allocates memory for the variables in the IBM module
     subroutine s_initialize_ibm_module()
 
-        gp_layers = 3
-
         if (p > 0) then
             @:ALLOCATE(ib_markers%sf(-gp_layers:m+gp_layers, &
                 -gp_layers:n+gp_layers, -gp_layers:p+gp_layers))
@@ -69,9 +68,8 @@ contains
 
         @:ACC_SETUP_SFs(ib_markers)
         @:ACC_SETUP_SFs(levelset)
-        ! @:ALLOCATE(ib_markers%sf(0:m, 0:n, 0:p))
 
-        !$acc enter data copyin(gp_layers, num_gps, num_inner_gps)
+        !$acc enter data copyin(num_gps, num_inner_gps)
 
     end subroutine s_initialize_ibm_module
 
@@ -209,7 +207,9 @@ contains
 
             ! Calculate velocity of ghost cell
             if (gp%slip) then
-                norm = levelset_norm%sf(j, k, l, patch_id, :)
+                norm = levelset_norm%sf(gp%loc(1), gp%loc(2), gp%loc(3), gp%ib_patch_id, :)
+                buf = sqrt(sum(norm**2))
+                norm = norm/buf
                 vel_norm_IP = sum(vel_IP*norm)*norm
                 vel_g = vel_IP - vel_norm_IP
             else
@@ -417,6 +417,7 @@ contains
         integer :: i, j, k, l, q !< Iterator variables
 
         num_gps = 0
+        num_inner_gps = 0
 
         do i = 0, m
             do j = 0, n

src/simulation/m_weno.fpp

@@ -40,7 +40,6 @@ module m_weno
     !! of the characteristic decomposition are stored in custom-constructed WENO-
     !! stencils (WS) that are annexed to each position of a given scalar field.
     !> @{
-
     real(wp), allocatable, dimension(:, :, :, :) :: v_rs_ws_x, v_rs_ws_y, v_rs_ws_z
     !> @}
 
@@ -52,52 +51,41 @@ module m_weno
     !! second dimension identifies the position of its coefficients and the last
     !! dimension denotes the cell-location in the relevant coordinate direction.
     !> @{
-
     real(wp), target, allocatable, dimension(:, :, :) :: poly_coef_cbL_x
     real(wp), target, allocatable, dimension(:, :, :) :: poly_coef_cbL_y
     real(wp), target, allocatable, dimension(:, :, :) :: poly_coef_cbL_z
-
     real(wp), target, allocatable, dimension(:, :, :) :: poly_coef_cbR_x
     real(wp), target, allocatable, dimension(:, :, :) :: poly_coef_cbR_y
     real(wp), target, allocatable, dimension(:, :, :) :: poly_coef_cbR_z
-
-    !    real(wp), pointer, dimension(:, :, :) :: poly_coef_L => null()
-    !    real(wp), pointer, dimension(:, :, :) :: poly_coef_R => null()
     !> @}
 
     !> @name The ideal weights at the left and the right cell-boundaries and at the
     !! left and the right quadrature points, in x-, y- and z-directions. Note
     !! that the first dimension of the array identifies the weight, while the
     !! last denotes the cell-location in the relevant coordinate direction.
     !> @{
-
     real(wp), target, allocatable, dimension(:, :) :: d_cbL_x
     real(wp), target, allocatable, dimension(:, :) :: d_cbL_y
     real(wp), target, allocatable, dimension(:, :) :: d_cbL_z
 
     real(wp), target, allocatable, dimension(:, :) :: d_cbR_x
     real(wp), target, allocatable, dimension(:, :) :: d_cbR_y
     real(wp), target, allocatable, dimension(:, :) :: d_cbR_z
-!    real(wp), pointer, dimension(:, :) :: d_L => null()
-!    real(wp), pointer, dimension(:, :) :: d_R => null()
     !> @}
 
     !> @name Smoothness indicator coefficients in the x-, y-, and z-directions. Note
     !! that the first array dimension identifies the smoothness indicator, the
     !! second identifies the position of its coefficients and the last denotes
     !! the cell-location in the relevant coordinate direction.
     !> @{
-
     real(wp), target, allocatable, dimension(:, :, :) :: beta_coef_x
     real(wp), target, allocatable, dimension(:, :, :) :: beta_coef_y
     real(wp), target, allocatable, dimension(:, :, :) :: beta_coef_z
-!    real(wp), pointer, dimension(:, :, :) :: beta_coef => null()
     !> @}
 
     ! END: WENO Coefficients
 
     integer :: v_size !< Number of WENO-reconstructed cell-average variables
-
     !$acc declare create(v_size)
 
     !> @name Indical bounds in the s1-, s2- and s3-directions