Skip to content

Commit d7dfc0d

Browse files
wilfonbawilfonba
committed
compiles and runs with OpenMP on CCE but fails all tests
1 parent 16c9ce3 commit d7dfc0d

21 files changed

+319
-306
lines changed

‎CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -492,8 +492,8 @@ function(MFC_SETUP_TARGET)
492492
target_compile_options(${a_target} PRIVATE -fopenmp -fopenmp-targets=spir64)
493493
target_link_options(${a_target} PRIVATE -fopenmp -fopenmp-targets=spir64)
494494
elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
495-
target_compile_options(${a_target} PRIVATE -fopenmp -fopenmp-targets=spir64 )
496-
target_link_options(${a_target} PRIVATE -fopenmp -fopenmp-targets=spir64)
495+
target_compile_options(${a_target} PRIVATE -fopenmp)
496+
target_link_options(${a_target} PRIVATE -fopenmp)
497497
endif()
498498
endif()
499499

‎src/common/include/omp_macros.fpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -160,9 +160,12 @@
160160
& no_create_val.strip('\n') + present_val.strip('\n') + &
161161
& deviceptr_val.strip('\n') + attach_val.strip('\n')
162162
#! Hardcoding the parallelism for now
163-
#:set omp_directive = '!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) ' + &
163+
!#:set omp_directive = '!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) ' + &
164+
!& clause_val + extraOmpArgs_val.strip('\n')
165+
!#:set omp_end_directive = '!$omp end target teams loop'
166+
#:set omp_directive = '!$omp target teams distribute parallel do simd defaultmap(firstprivate:scalar) ' + &
164167
& clause_val + extraOmpArgs_val.strip('\n')
165-
#:set omp_end_directive = '!$omp end target teams loop'
168+
#:set omp_end_directive = '!$omp end target teams distribute parallel do simd'
166169
$:omp_directive
167170
$:code
168171
$:omp_end_directive
@@ -201,7 +204,7 @@
201204
#! Not implemented yet
202205
#:def OMP_LOOP(collapse=None, parallelism=None, data_dependency=None, reduction=None, reductionOp=None, private=None, extraOmpArgs=None)
203206
#! loop is going to be ignored since all loops right now are seq
204-
#:set temp = '!$omp loop bind(thread)'
207+
#:set temp = ''
205208
$:temp
206209
#:enddef
207210

‎src/common/m_derived_types.fpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -449,4 +449,5 @@ module m_derived_types
449449
integer :: mn_min, np_min, mp_min, mnp_min
450450
end type cell_num_bounds
451451
452-
end module m_derived_types
452+
end module m_derived_types
453+

‎src/common/m_helper.fpp‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ module m_helper
1414

1515
implicit none
1616

17-
private;
17+
private;
1818
public :: s_comp_n_from_prim, &
1919
s_comp_n_from_cons, &
2020
s_initialize_nonpoly, &
@@ -292,7 +292,7 @@ contains
292292
!! @param a First vector.
293293
!! @param b Second vector.
294294
!! @return The cross product of the two vectors.
295-
function f_cross(a, b) result(c)
295+
pure function f_cross(a, b) result(c)
296296
297297
real(wp), dimension(3), intent(in) :: a, b
298298
real(wp), dimension(3) :: c
@@ -513,11 +513,11 @@ contains
513513
real(wp) :: Y, prefactor, local_pi
514514
515515
local_pi = acos(-1._wp)
516-
prefactor = sqrt((2*l + 1)/(4*local_pi)*factorial(l - m_order)/factorial(l + m_order));
516+
prefactor = sqrt((2*l + 1)/(4*local_pi)*factorial(l - m_order)/factorial(l + m_order));
517517
if (m_order == 0) then
518-
Y = prefactor*associated_legendre(x, l, m_order);
518+
Y = prefactor*associated_legendre(x, l, m_order);
519519
elseif (m_order > 0) then
520-
Y = (-1._wp)**m_order*sqrt(2._wp)*prefactor*associated_legendre(x, l, m_order)*cos(m_order*phi);
520+
Y = (-1._wp)**m_order*sqrt(2._wp)*prefactor*associated_legendre(x, l, m_order)*cos(m_order*phi);
521521
end if
522522
523523
end function spherical_harmonic_func
@@ -535,17 +535,17 @@ contains
535535
real(wp) :: result_P
536536
537537
if (m_order <= 0 .and. l <= 0) then
538-
result_P = 1;
538+
result_P = 1;
539539
elseif (l == 1 .and. m_order <= 0) then
540-
result_P = x;
540+
result_P = x;
541541
elseif (l == 1 .and. m_order == 1) then
542-
result_P = -(1 - x**2)**(1._wp/2._wp);
542+
result_P = -(1 - x**2)**(1._wp/2._wp);
543543
elseif (m_order == l) then
544-
result_P = (-1)**l*double_factorial(2*l - 1)*(1 - x**2)**(l/2);
544+
result_P = (-1)**l*double_factorial(2*l - 1)*(1 - x**2)**(l/2);
545545
elseif (m_order == l - 1) then
546-
result_P = x*(2*l - 1)*associated_legendre(x, l - 1, l - 1);
546+
result_P = x*(2*l - 1)*associated_legendre(x, l - 1, l - 1);
547547
else
548-
result_P = ((2*l - 1)*x*associated_legendre(x, l - 1, m_order) - (l + m_order - 1)*associated_legendre(x, l - 2, m_order))/(l - m_order);
548+
result_P = ((2*l - 1)*x*associated_legendre(x, l - 1, m_order) - (l + m_order - 1)*associated_legendre(x, l - 2, m_order))/(l - m_order);
549549
end if
550550
551551
end function associated_legendre

‎src/common/m_helper_basic.fpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,11 +89,12 @@ contains
8989
!! @param var_array Array to check.
9090
logical function f_all_default(var_array) result(res)
9191
real(wp), intent(in) :: var_array(:)
92-
! logical :: res_array(size(var_array))
93-
! integer :: i
9492

9593
res = all(f_is_default(var_array))
9694

95+
!logical :: res_array(size(var_array))
96+
!integer :: i
97+
9798
! do i = 1, size(var_array)
9899
! res_array(i) = f_is_default(var_array(i))
99100
! end do
@@ -161,7 +162,7 @@ contains
161162
!! @param m Number of cells in x-axis
162163
!! @param n Number of cells in y-axis
163164
!! @param p Number of cells in z-axis
164-
elemental subroutine s_update_cell_bounds(bounds, m, n, p)
165+
elemental subroutine s_update_cell_bounds(bounds, m, n, p)
165166
type(cell_num_bounds), intent(out) :: bounds
166167
integer, intent(in) :: m, n, p
167168

‎src/common/m_phase_change.fpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ module m_phase_change
4242
real(wp) :: A, B, C, D
4343
!> @}
4444

45-
$:GPU_DECLARE(create='[max_iter,pCr,TCr,mixM,lp,vp,A,B,C,D]')
45+
$:GPU_DECLARE(create='[A,B,C,D]')
4646

4747
contains
4848

‎src/common/m_variables_conversion.fpp‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,10 @@ module m_variables_conversion
5353
$:GPU_DECLARE(create='[gammas,gs_min,pi_infs,ps_inf,cvs,qvs,qvps]')
5454
#endif
5555

56-
real(wp), allocatable, dimension(:) :: Gs
57-
integer, allocatable, dimension(:) :: bubrs
58-
real(wp), allocatable, dimension(:, :) :: Res
59-
$:GPU_DECLARE(create='[bubrs,Gs,Res]')
56+
real(wp), allocatable, dimension(:) :: Gs_vc
57+
integer, allocatable, dimension(:) :: bubrs_vc
58+
real(wp), allocatable, dimension(:, :) :: Res_vc
59+
$:GPU_DECLARE(create='[bubrs_vc,Gs_vc,Res_vc]')
6060

6161
integer :: is1b, is2b, is3b, is1e, is2e, is3e
6262
$:GPU_DECLARE(create='[is1b,is2b,is3b,is1e,is2e,is3e]')
@@ -516,7 +516,7 @@ contains
516516
if (present(G_K)) then
517517
G_K = 0._wp
518518
do i = 1, num_fluids
519-
!TODO: change to use Gs directly here?
519+
!TODO: change to use Gs_vc directly here?
520520
!TODO: Make this changes as well for GPUs
521521
G_K = G_K + alpha_K(i)*G(i)
522522
end do
@@ -531,7 +531,7 @@ contains
531531
if (Re_size(i) > 0) Re_K(i) = 0._wp
532532

533533
do j = 1, Re_size(i)
534-
Re_K(i) = alpha_K(Re_idx(i, j))/Res(i, j) &
534+
Re_K(i) = alpha_K(Re_idx(i, j))/Res_vc(i, j) &
535535
+ Re_K(i)
536536
end do
537537

@@ -594,7 +594,7 @@ contains
594594
if (Re_size(i) > 0) Re_K(i) = 0._wp
595595

596596
do j = 1, Re_size(i)
597-
Re_K(i) = (1._wp - alpha_K(Re_idx(i, j)))/Res(i, j) &
597+
Re_K(i) = (1._wp - alpha_K(Re_idx(i, j)))/Res_vc(i, j) &
598598
+ Re_K(i)
599599
end do
600600

@@ -624,7 +624,7 @@ contains
624624
@:ALLOCATE(cvs (1:num_fluids))
625625
@:ALLOCATE(qvs (1:num_fluids))
626626
@:ALLOCATE(qvps (1:num_fluids))
627-
@:ALLOCATE(Gs (1:num_fluids))
627+
@:ALLOCATE(Gs_vc (1:num_fluids))
628628
#else
629629
@:ALLOCATE(gammas (1:num_fluids))
630630
@:ALLOCATE(gs_min (1:num_fluids))
@@ -633,46 +633,46 @@ contains
633633
@:ALLOCATE(cvs (1:num_fluids))
634634
@:ALLOCATE(qvs (1:num_fluids))
635635
@:ALLOCATE(qvps (1:num_fluids))
636-
@:ALLOCATE(Gs (1:num_fluids))
636+
@:ALLOCATE(Gs_vc (1:num_fluids))
637637
#endif
638638

639639
do i = 1, num_fluids
640640
gammas(i) = fluid_pp(i)%gamma
641641
gs_min(i) = 1.0_wp/gammas(i) + 1.0_wp
642642
pi_infs(i) = fluid_pp(i)%pi_inf
643-
Gs(i) = fluid_pp(i)%G
643+
Gs_vc(i) = fluid_pp(i)%G
644644
ps_inf(i) = pi_infs(i)/(1.0_wp + gammas(i))
645645
cvs(i) = fluid_pp(i)%cv
646646
qvs(i) = fluid_pp(i)%qv
647647
qvps(i) = fluid_pp(i)%qvp
648648
end do
649-
$:GPU_UPDATE(device='[gammas,gs_min,pi_infs,ps_inf,cvs,qvs,qvps,Gs]')
649+
$:GPU_UPDATE(device='[gammas,gs_min,pi_infs,ps_inf,cvs,qvs,qvps,Gs_vc]')
650650

651651
#ifdef MFC_SIMULATION
652652

653653
if (viscous) then
654-
@:ALLOCATE(Res(1:2, 1:Re_size_max))
654+
@:ALLOCATE(Res_vc(1:2, 1:Re_size_max))
655655
do i = 1, 2
656656
do j = 1, Re_size(i)
657-
Res(i, j) = fluid_pp(Re_idx(i, j))%Re(i)
657+
Res_vc(i, j) = fluid_pp(Re_idx(i, j))%Re(i)
658658
end do
659659
end do
660660

661-
$:GPU_UPDATE(device='[Res,Re_idx,Re_size]')
661+
$:GPU_UPDATE(device='[Res_vc,Re_idx,Re_size]')
662662
end if
663663
#endif
664664

665665
if (bubbles_euler) then
666666
#ifdef MFC_SIMULATION
667-
@:ALLOCATE(bubrs(1:nb))
667+
@:ALLOCATE(bubrs_vc(1:nb))
668668
#else
669-
@:ALLOCATE(bubrs(1:nb))
669+
@:ALLOCATE(bubrs_vc(1:nb))
670670
#endif
671671

672672
do i = 1, nb
673-
bubrs(i) = bub_idx%rs(i)
673+
bubrs_vc(i) = bub_idx%rs(i)
674674
end do
675-
$:GPU_UPDATE(device='[bubrs]')
675+
$:GPU_UPDATE(device='[bubrs_vc]')
676676
end if
677677

678678
#ifdef MFC_POST_PROCESS
@@ -906,7 +906,7 @@ contains
906906
! If in simulation, use acc mixture subroutines
907907
if (elasticity) then
908908
call s_convert_species_to_mixture_variables_acc(rho_K, gamma_K, pi_inf_K, qv_K, alpha_K, &
909-
alpha_rho_K, Re_K, G_K, Gs)
909+
alpha_rho_K, Re_K, G_K, Gs_vc)
910910
else if (bubbles_euler) then
911911
call s_convert_species_to_mixture_variables_bubbles_acc(rho_K, gamma_K, pi_inf_K, qv_K, &
912912
alpha_K, alpha_rho_K, Re_K)
@@ -1073,7 +1073,7 @@ contains
10731073
if (bubbles_euler) then
10741074
$:GPU_LOOP(parallelism='[seq]')
10751075
do i = 1, nb
1076-
nRtmp(i) = qK_cons_vf(bubrs(i))%sf(j, k, l)
1076+
nRtmp(i) = qK_cons_vf(bubrs_vc(i))%sf(j, k, l)
10771077
end do
10781078

10791079
vftmp = qK_cons_vf(alf_idx)%sf(j, k, l)
@@ -1517,7 +1517,7 @@ contains
15171517
if (elasticity) then
15181518
call s_convert_species_to_mixture_variables_acc(rho_K, gamma_K, pi_inf_K, qv_K, &
15191519
alpha_K, alpha_rho_K, Re_K, &
1520-
G_K, Gs)
1520+
G_K, Gs_vc)
15211521
else if (bubbles_euler) then
15221522
call s_convert_species_to_mixture_variables_bubbles_acc(rho_K, gamma_K, &
15231523
pi_inf_K, qv_K, alpha_K, alpha_rho_K, Re_K)
@@ -1607,14 +1607,14 @@ contains
16071607
#endif
16081608

16091609
#ifdef MFC_SIMULATION
1610-
@:DEALLOCATE(gammas, gs_min, pi_infs, ps_inf, cvs, qvs, qvps, Gs)
1610+
@:DEALLOCATE(gammas, gs_min, pi_infs, ps_inf, cvs, qvs, qvps, Gs_vc)
16111611
if (bubbles_euler) then
1612-
@:DEALLOCATE(bubrs)
1612+
@:DEALLOCATE(bubrs_vc)
16131613
end if
16141614
#else
1615-
@:DEALLOCATE(gammas, gs_min, pi_infs, ps_inf, cvs, qvs, qvps, Gs)
1615+
@:DEALLOCATE(gammas, gs_min, pi_infs, ps_inf, cvs, qvs, qvps, Gs_vc)
16161616
if (bubbles_euler) then
1617-
@:DEALLOCATE(bubrs)
1617+
@:DEALLOCATE(bubrs_vc)
16181618
end if
16191619
#endif
16201620

‎src/pre_process/m_assign_variables.fpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ contains
102102
!! @param eta pseudo volume fraction
103103
!! @param q_prim_vf Primitive variables
104104
!! @param patch_id_fp Array to track patch ids
105-
subroutine s_assign_patch_mixture_primitive_variables(patch_id, j, k, l, &
105+
subroutine s_assign_patch_mixture_primitive_variables(patch_id, j, k, l, &
106106
eta, q_prim_vf, patch_id_fp)
107107
$:GPU_ROUTINE(parallelism='[seq]')
108108
@@ -191,7 +191,7 @@ contains
191191
!! @param k the y-dir node index
192192
!! @param l the z-dir node index
193193
!! @param q_prim_vf Primitive variables
194-
subroutine s_perturb_primitive(j, k, l, q_prim_vf)
194+
subroutine s_perturb_primitive(j, k, l, q_prim_vf)
195195
196196
integer, intent(in) :: j, k, l
197197
type(scalar_field), dimension(1:sys_size), intent(inout) :: q_prim_vf

‎src/pre_process/m_compute_levelset.fpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ module m_compute_levelset
2727

2828
contains
2929

30-
subroutine s_circle_levelset(levelset, levelset_norm, ib_patch_id)
30+
subroutine s_circle_levelset(levelset, levelset_norm, ib_patch_id)
3131

3232
type(levelset_field), intent(INOUT) :: levelset
3333
type(levelset_norm_field), intent(INOUT) :: levelset_norm
@@ -63,7 +63,7 @@ contains
6363

6464
end subroutine s_circle_levelset
6565

66-
subroutine s_airfoil_levelset(levelset, levelset_norm, ib_patch_id)
66+
subroutine s_airfoil_levelset(levelset, levelset_norm, ib_patch_id)
6767

6868
type(levelset_field), intent(INOUT) :: levelset
6969
type(levelset_norm_field), intent(INOUT) :: levelset_norm
@@ -146,7 +146,7 @@ contains
146146

147147
end subroutine s_airfoil_levelset
148148

149-
subroutine s_3D_airfoil_levelset(levelset, levelset_norm, ib_patch_id)
149+
subroutine s_3D_airfoil_levelset(levelset, levelset_norm, ib_patch_id)
150150

151151
type(levelset_field), intent(INOUT) :: levelset
152152
type(levelset_norm_field), intent(INOUT) :: levelset_norm
@@ -250,7 +250,7 @@ contains
250250
end subroutine s_3D_airfoil_levelset
251251

252252
!> Initialize IBM module
253-
subroutine s_rectangle_levelset(levelset, levelset_norm, ib_patch_id)
253+
subroutine s_rectangle_levelset(levelset, levelset_norm, ib_patch_id)
254254

255255
type(levelset_field), intent(INOUT) :: levelset
256256
type(levelset_norm_field), intent(INOUT) :: levelset_norm
@@ -347,7 +347,7 @@ contains
347347

348348
end subroutine s_rectangle_levelset
349349

350-
subroutine s_cuboid_levelset(levelset, levelset_norm, ib_patch_id)
350+
subroutine s_cuboid_levelset(levelset, levelset_norm, ib_patch_id)
351351

352352
type(levelset_field), intent(INOUT) :: levelset
353353
type(levelset_norm_field), intent(INOUT) :: levelset_norm
@@ -464,7 +464,7 @@ contains
464464

465465
end subroutine s_cuboid_levelset
466466

467-
subroutine s_sphere_levelset(levelset, levelset_norm, ib_patch_id)
467+
subroutine s_sphere_levelset(levelset, levelset_norm, ib_patch_id)
468468

469469
type(levelset_field), intent(INOUT) :: levelset
470470
type(levelset_norm_field), intent(INOUT) :: levelset_norm
@@ -501,7 +501,7 @@ contains
501501

502502
end subroutine s_sphere_levelset
503503

504-
subroutine s_cylinder_levelset(levelset, levelset_norm, ib_patch_id)
504+
subroutine s_cylinder_levelset(levelset, levelset_norm, ib_patch_id)
505505

506506
type(levelset_field), intent(INOUT) :: levelset
507507
type(levelset_norm_field), intent(INOUT) :: levelset_norm

0 commit comments

Comments
 (0)