Correct line markers have been recovered

Author: danieljvickers

src/common/include/acc_macros.fpp

@@ -161,6 +161,35 @@
     $:acc_end_directive
 #:enddef
 
+#:def NEW_ACC_PARALLEL_LOOP(collapse=None, private=None, parallelism='[gang, vector]', &
+    & default='present', firstprivate=None, reduction=None, reductionOp=None, &
+    & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
+    & no_create=None, present=None, deviceptr=None, attach=None, extraAccArgs=None)
+    #:set collapse_val = GEN_COLLAPSE_STR(collapse)
+    #:set parallelism_val = GEN_PARALLELISM_STR(parallelism)
+    #:set default_val = GEN_DEFAULT_STR(default)
+    #:set private_val = GEN_PRIVATE_STR(private, False).strip('\n') + GEN_PRIVATE_STR(firstprivate, True).strip('\n')
+    #:set reduction_val = GEN_REDUCTION_STR(reduction, reductionOp)
+    #:set copy_val = GEN_COPY_STR(copy)
+    #:set copyin_val = GEN_COPYIN_STR(copyin, False).strip('\n') + GEN_COPYIN_STR(copyinReadOnly, True).strip('\n')
+    #:set copyout_val = GEN_COPYOUT_STR(copyout)
+    #:set create_val = GEN_CREATE_STR(create)
+    #:set no_create_val = GEN_NOCREATE_STR(no_create)
+    #:set present_val = GEN_PRESENT_STR(present)
+    #:set deviceptr_val = GEN_DEVICEPTR_STR(deviceptr)
+    #:set attach_val = GEN_ATTACH_STR(attach)
+    #:set extraAccArgs_val = GEN_EXTRA_ARGS_STR(extraAccArgs)
+    #:set clause_val = collapse_val.strip('\n') + parallelism_val.strip('\n') + &
+        & default_val.strip('\n') + private_val.strip('\n') + reduction_val.strip('\n') + &
+        & copy_val.strip('\n') + copyin_val.strip('\n') + &
+        & copyout_val.strip('\n') + create_val.strip('\n') + &
+        & no_create_val.strip('\n') + present_val.strip('\n') + &
+        & deviceptr_val.strip('\n') + attach_val.strip('\n')
+    #:set acc_directive = '!$acc parallel loop ' + &
+        & clause_val + extraAccArgs_val.strip('\n')
+    $:acc_directive
+#:enddef
+
 #:def ACC_ROUTINE(function_name=None, parallelism=None, nohost=False, extraAccArgs=None)
     #:set parallelism_val = GEN_PARALLELISM_STR(parallelism)
     #:assert isinstance(nohost, bool)

src/common/include/omp_macros.fpp

@@ -196,6 +196,62 @@
     $:omp_end_directive
 #:enddef
 
+#:def NEW_OMP_PARALLEL_LOOP(collapse=None, private=None, parallelism='[gang, vector]', &
+    & default='present', firstprivate=None, reduction=None, reductionOp=None, &
+    & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
+    & no_create=None, present=None, deviceptr=None, attach=None, extraOmpArgs=None)
+    
+    #:set collapse_val = GEN_COLLAPSE_STR(collapse)
+    #:set parallelism_val = OMP_PARALLELISM_STR(parallelism)
+    #:set default_val = OMP_DEFAULT_STR(default)
+    #:set private_val = GEN_PRIVATE_STR(private, False).strip('\n') + GEN_PRIVATE_STR(firstprivate, True).strip('\n')
+    #:set reduction_val = GEN_REDUCTION_STR(reduction, reductionOp)
+    #:set copy_val = OMP_COPY_STR(copy)
+    #:set copyin_val = OMP_COPYIN_STR(copyin).strip('\n') + OMP_COPYIN_STR(copyinReadOnly).strip('\n')
+    #:set copyout_val = OMP_COPYOUT_STR(copyout)
+    #:set create_val = OMP_CREATE_STR(create)
+    #:set no_create_val = OMP_NOCREATE_STR(no_create)
+    #:set present_val = OMP_PRESENT_STR(present)
+    #:set deviceptr_val = OMP_DEVICEPTR_STR(deviceptr)
+    #:set attach_val = OMP_MAP_STR('always,tofrom', attach)
+    #:set extraOmpArgs_val = GEN_EXTRA_ARGS_STR(extraOmpArgs)
+    #:set clause_val = collapse_val.strip('\n') + parallelism_val.strip('\n') + &
+        & default_val.strip('\n') + private_val.strip('\n') + reduction_val.strip('\n') + &
+        & copy_val.strip('\n') + copyin_val.strip('\n') + &
+        & copyout_val.strip('\n') + create_val.strip('\n') + &
+        & no_create_val.strip('\n') + present_val.strip('\n') + &
+        & deviceptr_val.strip('\n') + attach_val.strip('\n')
+    #! Hardcoding the parallelism for now
+
+    #:if MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID
+        #:set omp_start_directive = '!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) '
+    #:elif MFC_COMPILER == CCE_COMPILER_ID
+        #:set omp_start_directive = '!$omp target teams distribute parallel do simd defaultmap(firstprivate:scalar) '
+    #:elif MFC_COMPILER == AMD_COMPILER_ID
+        #:set omp_start_directive = '!$omp target teams distribute parallel do '
+    #:else
+        #:set omp_start_directive = '!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) '
+    #:endif
+
+    #:set omp_directive = omp_start_directive + clause_val + extraOmpArgs_val.strip('\n')
+    $:omp_directive
+#:enddef
+
+#:def END_OMP_PARALLEL_LOOP()
+    
+    #:if MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID
+        #:set omp_end_directive = '!$omp end target teams loop'
+    #:elif MFC_COMPILER == CCE_COMPILER_ID
+        #:set omp_end_directive = '!$omp end target teams distribute parallel do simd'
+    #:elif MFC_COMPILER == AMD_COMPILER_ID
+        #:set omp_end_directive = '!$omp end target teams distribute parallel do'
+    #:else
+        #:set omp_end_directive = '!$omp end target teams loop'
+    #:endif
+
+    $:omp_end_directive
+#:enddef
+
 #:def OMP_ROUTINE(function_name, nohost, extraOmpArgs)
     #:assert isinstance(nohost, bool)
     #:if nohost == True

src/common/include/parallel_macros.fpp

@@ -36,6 +36,33 @@
 #endif
 #:enddef
 
+#:def NEW_GPU_PARALLEL_LOOP(collapse=None, private=None, parallelism='[gang, vector]', &
+    & default='present', firstprivate=None, reduction=None, reductionOp=None, &
+    & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
+    & no_create=None, present=None, deviceptr=None, attach=None, extraAccArgs=None, extraOmpArgs=None)
+
+#if defined(MFC_OpenACC)
+    #:set directive = NEW_ACC_PARALLEL_LOOP(collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraAccArgs)
+#elif defined(MFC_OpenMP)
+    #:set directive = NEW_OMP_PARALLEL_LOOP(collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraOmpArgs)
+#endif
+
+    $:directive
+
+#:enddef
+
+#:def END_GPU_PARALLEL_LOOP()
+
+    #:set acc_end_directive = '!$acc end parallel loop'
+    #:set omp_code = END_OMP_PARALLEL_LOOP(code, collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraOmpArgs)
+
+#if defined(MFC_OpenACC)
+    $:acc_end_directive
+#elif defined(MFC_OpenMP)
+    $:omp_code
+#endif
+#:enddef
+
 #:def GPU_ROUTINE(function_name=None, parallelism=None, nohost=False, cray_inline=False, extraAccArgs=None, extraOmpArgs=None)
     #:assert isinstance(cray_inline, bool)
     #:set acc_directive = ACC_ROUTINE(function_name=function_name, parallelism=parallelism, nohost=nohost, extraAccArgs=extraAccArgs)

src/simulation/m_ibm.fpp

@@ -197,7 +197,7 @@ contains
         type(ghost_point) :: gp
         type(ghost_point) :: innerp
         if (num_gps > 0) then
-            #:call GPU_PARALLEL_LOOP(private='[physical_loc,dyn_pres,alpha_rho_IP, alpha_IP,pres_IP,vel_IP,vel_g,vel_norm_IP,r_IP, v_IP,pb_IP,mv_IP,nmom_IP,presb_IP,massv_IP,rho, gamma,pi_inf,Re_K,G_K,Gs,gp,innerp,norm,buf, radial_vector, rotation_velocity, j,k,l,q]')
+            $:NEW_GPU_PARALLEL_LOOP(private='[i,physical_loc,dyn_pres,alpha_rho_IP, alpha_IP,pres_IP,vel_IP,vel_g,vel_norm_IP,r_IP, v_IP,pb_IP,mv_IP,nmom_IP,presb_IP,massv_IP,rho, gamma,pi_inf,Re_K,G_K,Gs,gp,innerp,norm,buf, radial_vector, rotation_velocity, j,k,l,q]')
                 do i = 1, num_gps
 
                     gp = ghost_points(i)
@@ -365,7 +365,7 @@ contains
                         end do
                     end if
                 end do
-            #:endcall GPU_PARALLEL_LOOP
+            ! $:END_GPU_PARALLEL_LOOP
         end if
 
         !Correct the state of the inner points in IBs