almost working on GPUs

Author: Benjamin Wilfong

src/simulation/m_rhs.f90

@@ -197,7 +197,8 @@ module m_rhs
 !$acc   tau_Re_vf,qL_prim, qR_prim, iv,ix, iy, iz,is1,is2,is3,bub_adv_src,bub_r_src,bub_v_src, bub_p_src, bub_m_src, &
 !$acc   bub_mom_src, myflux_vf, myflux_src_vf,alf_sum, momxb, momxe, contxb, contxe, advxb, advxe, bubxb, bubxe, strxb, strxe, &
 !$acc   blkmod1, blkmod2, alpha1, alpha2, Kterm, divu, qL_rsx_vf_flat, qL_rsy_vf_flat, qL_rsz_vf_flat, qR_rsx_vf_flat, qR_rsy_vf_flat, qR_rsz_vf_flat, &
-!$acc     dqL_rsx_vf_flat, dqL_rsy_vf_flat, dqL_rsz_vf_flat, dqR_rsx_vf_flat, dqR_rsy_vf_flat, dqR_rsz_vf_flat)
+!$acc   dqL_rsx_vf_flat, dqL_rsy_vf_flat, dqL_rsz_vf_flat, dqR_rsx_vf_flat, dqR_rsy_vf_flat, dqR_rsz_vf_flat, &
+!$acc   ixt, iyt, izt)
 
     real(kind(0d0)), allocatable, dimension(:, :, :) :: nbub !< Bubble number density
     integer, allocatable, dimension(:) :: rs, vs, ps, ms
@@ -239,10 +240,8 @@ subroutine s_initialize_rhs_module() ! ---------------------------------
 !$acc                                        iy%beg:iy%end, &
 !$acc                                        iz%beg:iz%end))
         end if
-
-        ixt = ix
-        iyt = iy
-        izt = iz
+        print*, ix, iy, iz
+        ixt = ix; iyt = iy; izt = iz
 
         allocate (q_cons_qp%vf(1:sys_size))
         allocate (q_prim_qp%vf(1:sys_size))
@@ -570,6 +569,7 @@ subroutine s_initialize_rhs_module() ! ---------------------------------
            allocate (mono_mass_src(0:m, 0:n, 0:p))
            allocate (mono_mom_src(1:num_dims, 0:m, 0:n, 0:p))
            allocate (mono_E_src(0:m, 0:n, 0:p))
+!$acc enter data create(mono_mass_src(0:m, 0:n, 0:p), mono_mom_src(1:num_dims, 0:m, 0:n, 0:p), mono_E_src(0:m, 0:n, 0:p))
         end if
 
         allocate (divu%sf( &
@@ -796,7 +796,7 @@ subroutine s_compute_rhs(q_cons_vf, q_prim_vf, rhs_vf, t_step) ! -------
         type(scalar_field), dimension(sys_size), intent(INOUT) :: q_prim_vf
         type(scalar_field), dimension(sys_size), intent(INOUT) :: rhs_vf
         integer, intent(IN) :: t_step
-
+        
         real(kind(0d0)) :: top, bottom  !< Numerator and denominator when evaluating flux limiter function
         real(kind(0d0)), dimension(num_fluids) :: myalpha_rho, myalpha
 

src/simulation/m_time_steppers.f90

@@ -124,7 +124,7 @@ subroutine s_initialize_time_steppers_module() ! -----------------------
 
         ! Allocating the cell-average primitive variables
         allocate (q_prim_vf(1:sys_size))
-
+        
         do i = 1, adv_idx%end
             allocate (q_prim_vf(i)%sf(ix%beg:ix%end, &
                                       iy%beg:iy%end, &

src/simulation/m_viscous.f90

@@ -31,6 +31,7 @@ module m_viscous
 
     integer :: momxb, momxe
     integer :: contxb, contxe
+!$acc declare create(momxb, momxe, contxb, contxe)
 
     contains
 
@@ -58,6 +59,7 @@ subroutine s_initialize_viscous_module()
         momxe = mom_idx%end
         contxb = cont_idx%beg
         contxe = cont_idx%end
+!$acc update device(momxb, momxe, contxb, contxe)
 
     end subroutine s_initialize_viscous_module
 
@@ -100,13 +102,11 @@ subroutine s_compute_viscous_stress_tensor(q_prim_vf, grad_x_vf, grad_y_vf, grad
                 end do
             end do
         end do
-
         if (Re_size(1) > 0) then    ! Shear stresses
     !$acc parallel loop collapse(3) gang vector default(present) private(alpha_visc, alpha_rho_visc, Re_visc, tau_Re )
             do l = iz%beg, iz%end
                 do k = -1, 1
                     do j = ix%beg, ix%end
-    
     !$acc loop seq
                         do i = 1, num_fluids
                             alpha_rho_visc(i) = q_prim_vf(i)%sf(j, k, l)
@@ -180,15 +180,15 @@ subroutine s_compute_viscous_stress_tensor(q_prim_vf, grad_x_vf, grad_y_vf, grad
                                 end do
                             end if
                         end if
-                        
+
                         tau_Re(2, 1) = (grad_y_vf(1)%sf(j, k, l) + &
                                         grad_x_vf(2)%sf(j, k, l))/ &
                                     Re_visc(1)
 
                         tau_Re(2, 2) = (4d0*grad_y_vf(2)%sf(j, k, l) &
                                         - 2d0*grad_x_vf(1)%sf(j, k, l) &
                                         - 2d0*q_prim_vf(momxb + 1)%sf(j, k, l)/y_cc(k))/ &
-                                    (3d0*Re_visc(1))       
+                                    (3d0*Re_visc(1))      
     !$acc loop seq
                         do i = 1, 2
                             tau_Re_vf(contxe + i)%sf(j, k, l) = &
@@ -715,7 +715,7 @@ subroutine s_get_viscous(qL_prim_rsx_vf_flat, qL_prim_rsy_vf_flat, qL_prim_rsz_v
 
         type(vector_field) :: q_prim_qp
 
-        type(vector_field), dimension(sys_size), &
+        type(vector_field), dimension(1:num_dims), &
             intent(INOUT) :: dqL_prim_dx_n, dqR_prim_dx_n, &
                              dqL_prim_dy_n, dqR_prim_dy_n, &
                              dqL_prim_dz_n, dqR_prim_dz_n
@@ -1374,7 +1374,7 @@ end subroutine s_reconstruct_cell_boundary_values_visc ! --------------------
 
     subroutine s_finalize_viscous_module()
         deallocate (gammas, pi_infs)
-        allocate (Res(1:2, 1:maxval(Re_size)))
+        deallocate (Res(1:2, 1:maxval(Re_size)))
     end subroutine s_finalize_viscous_module
 
 end module m_viscous