bug fixes for indexing and GPU communication

Author: wilfonba

src/simulation/m_bubbles_EL.fpp

@@ -114,12 +114,15 @@ contains
 
         pcomm_coords(1)%beg = x_cb(buff_size - fd_number - 1)
         pcomm_coords(1)%end = x_cb(m - buff_size + fd_number)
+        $:GPU_UPDATE(device='[pcomm_coords(1)]')
         if (n > 0) then
             pcomm_coords(2)%beg = y_cb(buff_size - fd_number - 1)
             pcomm_coords(2)%end = y_cb(n - buff_size + fd_number)
+            $:GPU_UPDATE(device='[pcomm_coords(2)]')
             if (p > 0) then
                 pcomm_coords(3)%beg = z_cb(buff_size - fd_number - 1)
                 pcomm_coords(3)%end = z_cb(p - buff_size + fd_number)
+                $:GPU_UPDATE(device='[pcomm_coords(3)]')
             end if
         end if
 
@@ -1482,8 +1485,8 @@ contains
         $:GPU_UPDATE(host='[bub_R0, Rmax_stats, Rmin_stats, gas_mg, gas_betaT, &
             & gas_betaC, bub_dphidt, lag_id, gas_p, gas_mv, intfc_rad, intfc_vel, &
             & mtn_pos, mtn_posPrev, mtn_vel, mtn_s, intfc_draddt, intfc_dveldt, &
-            & gas_dpdt, gas_dmvdt, mtn_dposdt, mtn_dveldt, keep_bubble, &
-            & nBubs]')
+            & gas_dpdt, gas_dmvdt, mtn_dposdt, mtn_dveldt, keep_bubble, nBubs, &
+            & wrap_bubble_dir, wrap_bubble_loc]')
         call nvtxEndRange
 
         if (nBubs > 0) then

src/simulation/m_bubbles_EL_kernels.fpp

@@ -593,11 +593,11 @@ contains
                 eta(1) = (q_prim_vf(E_idx)%sf(cell(1), cell(2), cell(3)) - &
                           q_prim_vf(E_idx)%sf(cell(1), cell(2), cell(3) - 2))/ &
                          (z_cc(cell(3)) - z_cc(cell(3) - 2))
-                xi(2) = y_cc(cell(3))
+                xi(2) = z_cc(cell(3))
                 eta(2) = (q_prim_vf(E_idx)%sf(cell(1), cell(2), cell(3) + 1) - &
                           q_prim_vf(E_idx)%sf(cell(1), cell(2), cell(3) - 1))/ &
                          (z_cc(cell(3) + 1) - z_cc(cell(3) - 1))
-                xi(3) = y_cc(cell(3) + 1)
+                xi(3) = z_cc(cell(3) + 1)
                 eta(3) = (q_prim_vf(E_idx)%sf(cell(1), cell(2), cell(3) + 2) - &
                           q_prim_vf(E_idx)%sf(cell(1), cell(2), cell(3)))/ &
                          (z_cc(cell(3) + 2) - z_cc(cell(3)))

src/simulation/m_global_parameters.fpp

@@ -250,6 +250,7 @@ module m_global_parameters
     !! Processor coordinates in MPI_CART_COMM
 
     type(bounds_info), allocatable, dimension(:) :: pcomm_coords
+    $:GPU_DECLARE(create='[pcomm_coords]')
     !! Coordinates for EL particle transfer
 
     type(int_bounds_info), dimension(3) :: nidx !< Indices for neighboring processors
@@ -1389,7 +1390,7 @@ contains
         #:endif
 
         allocate (proc_coords(1:num_dims))
-        allocate (pcomm_coords(1:num_dims))
+        @:ALLOCATE(pcomm_coords(1:num_dims))
 
         if (parallel_io .neqv. .true.) return
 
@@ -1426,7 +1427,7 @@ contains
         end if
 
         deallocate (proc_coords)
-        deallocate (pcomm_coords)
+        @:DEALLOCATE(pcomm_coords)
         if (parallel_io) then
             deallocate (start_idx)