Add line numbering to gpu loops (#1029)

Co-authored-by: Daniel Vickers <[email protected]>
Co-authored-by: Daniel Vickers <[email protected]>

Author: 3 people

docs/documentation/gpuParallelization.md

@@ -44,14 +44,16 @@ Note: Ordering is not guaranteed or stable, so use key-value pairing when using
 
 **Macro Invocation**
 
-Uses FYPP eval directive using `#:call`
+In order to parallelize a loop, simply place two macro calls on either end of the loop:
 
 ```C
-#:call GPU_PARALLEL_LOOP(...)
+$:$GPU_PARALLEL_LOOP(...)
    {code}
-#:endcall GPU_PARALLEL_LOOP
+$:END_GPU_PARALLEL_LOOP()
 ```
 
+This wraps the lines in `code` with parallelization calls to openACC or openMP, depending on environment and compiler settings.
+
 **Parameters**
 
 | name             | data type           | Default Value     | description                                                                               |

src/common/include/acc_macros.fpp

@@ -129,7 +129,7 @@
     $:end_acc_directive
 #:enddef
 
-#:def ACC_PARALLEL_LOOP(code, collapse=None, private=None, parallelism='[gang, vector]', &
+#:def ACC_PARALLEL_LOOP_OLD(code, collapse=None, private=None, parallelism='[gang, vector]', &
     & default='present', firstprivate=None, reduction=None, reductionOp=None, &
     & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
     & no_create=None, present=None, deviceptr=None, attach=None, extraAccArgs=None)
@@ -161,6 +161,35 @@
     $:acc_end_directive
 #:enddef
 
+#:def ACC_PARALLEL_LOOP(collapse=None, private=None, parallelism='[gang, vector]', &
+    & default='present', firstprivate=None, reduction=None, reductionOp=None, &
+    & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
+    & no_create=None, present=None, deviceptr=None, attach=None, extraAccArgs=None)
+    #:set collapse_val = GEN_COLLAPSE_STR(collapse)
+    #:set parallelism_val = GEN_PARALLELISM_STR(parallelism)
+    #:set default_val = GEN_DEFAULT_STR(default)
+    #:set private_val = GEN_PRIVATE_STR(private, False).strip('\n') + GEN_PRIVATE_STR(firstprivate, True).strip('\n')
+    #:set reduction_val = GEN_REDUCTION_STR(reduction, reductionOp)
+    #:set copy_val = GEN_COPY_STR(copy)
+    #:set copyin_val = GEN_COPYIN_STR(copyin, False).strip('\n') + GEN_COPYIN_STR(copyinReadOnly, True).strip('\n')
+    #:set copyout_val = GEN_COPYOUT_STR(copyout)
+    #:set create_val = GEN_CREATE_STR(create)
+    #:set no_create_val = GEN_NOCREATE_STR(no_create)
+    #:set present_val = GEN_PRESENT_STR(present)
+    #:set deviceptr_val = GEN_DEVICEPTR_STR(deviceptr)
+    #:set attach_val = GEN_ATTACH_STR(attach)
+    #:set extraAccArgs_val = GEN_EXTRA_ARGS_STR(extraAccArgs)
+    #:set clause_val = collapse_val.strip('\n') + parallelism_val.strip('\n') + &
+        & default_val.strip('\n') + private_val.strip('\n') + reduction_val.strip('\n') + &
+        & copy_val.strip('\n') + copyin_val.strip('\n') + &
+        & copyout_val.strip('\n') + create_val.strip('\n') + &
+        & no_create_val.strip('\n') + present_val.strip('\n') + &
+        & deviceptr_val.strip('\n') + attach_val.strip('\n')
+    #:set acc_directive = '!$acc parallel loop ' + &
+        & clause_val + extraAccArgs_val.strip('\n')
+    $:acc_directive
+#:enddef
+
 #:def ACC_ROUTINE(function_name=None, parallelism=None, nohost=False, extraAccArgs=None)
     #:set parallelism_val = GEN_PARALLELISM_STR(parallelism)
     #:assert isinstance(nohost, bool)
@@ -308,4 +337,4 @@
     #:set acc_directive = '!$acc wait ' + clause_val + extraAccArgs_val.strip('\n')
     $:acc_directive
 #:enddef
-! New line at end of file is required for FYPP
+! New line at end of file is required for FYPP

src/common/include/omp_macros.fpp

@@ -149,7 +149,7 @@
     $:omp_end_directive
 #:enddef
 
-#:def OMP_PARALLEL_LOOP(code, collapse=None, private=None, parallelism='[gang, vector]', &
+#:def OMP_PARALLEL_LOOP_OLD(code, collapse=None, private=None, parallelism='[gang, vector]', &
     & default='present', firstprivate=None, reduction=None, reductionOp=None, &
     & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
     & no_create=None, present=None, deviceptr=None, attach=None, extraOmpArgs=None)
@@ -196,6 +196,62 @@
     $:omp_end_directive
 #:enddef
 
+#:def OMP_PARALLEL_LOOP(collapse=None, private=None, parallelism='[gang, vector]', &
+    & default='present', firstprivate=None, reduction=None, reductionOp=None, &
+    & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
+    & no_create=None, present=None, deviceptr=None, attach=None, extraOmpArgs=None)
+    
+    #:set collapse_val = GEN_COLLAPSE_STR(collapse)
+    #:set parallelism_val = OMP_PARALLELISM_STR(parallelism)
+    #:set default_val = OMP_DEFAULT_STR(default)
+    #:set private_val = GEN_PRIVATE_STR(private, False).strip('\n') + GEN_PRIVATE_STR(firstprivate, True).strip('\n')
+    #:set reduction_val = GEN_REDUCTION_STR(reduction, reductionOp)
+    #:set copy_val = OMP_COPY_STR(copy)
+    #:set copyin_val = OMP_COPYIN_STR(copyin).strip('\n') + OMP_COPYIN_STR(copyinReadOnly).strip('\n')
+    #:set copyout_val = OMP_COPYOUT_STR(copyout)
+    #:set create_val = OMP_CREATE_STR(create)
+    #:set no_create_val = OMP_NOCREATE_STR(no_create)
+    #:set present_val = OMP_PRESENT_STR(present)
+    #:set deviceptr_val = OMP_DEVICEPTR_STR(deviceptr)
+    #:set attach_val = OMP_MAP_STR('always,tofrom', attach)
+    #:set extraOmpArgs_val = GEN_EXTRA_ARGS_STR(extraOmpArgs)
+    #:set clause_val = collapse_val.strip('\n') + parallelism_val.strip('\n') + &
+        & default_val.strip('\n') + private_val.strip('\n') + reduction_val.strip('\n') + &
+        & copy_val.strip('\n') + copyin_val.strip('\n') + &
+        & copyout_val.strip('\n') + create_val.strip('\n') + &
+        & no_create_val.strip('\n') + present_val.strip('\n') + &
+        & deviceptr_val.strip('\n') + attach_val.strip('\n')
+    #! Hardcoding the parallelism for now
+
+    #:if MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID
+        #:set omp_start_directive = '!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) '
+    #:elif MFC_COMPILER == CCE_COMPILER_ID
+        #:set omp_start_directive = '!$omp target teams distribute parallel do simd defaultmap(firstprivate:scalar) '
+    #:elif MFC_COMPILER == AMD_COMPILER_ID
+        #:set omp_start_directive = '!$omp target teams distribute parallel do '
+    #:else
+        #:set omp_start_directive = '!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) '
+    #:endif
+
+    #:set omp_directive = omp_start_directive + clause_val + extraOmpArgs_val.strip('\n')
+    $:omp_directive
+#:enddef
+
+#:def END_OMP_PARALLEL_LOOP()
+    
+    #:if MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID
+        #:set omp_end_directive = '!$omp end target teams loop'
+    #:elif MFC_COMPILER == CCE_COMPILER_ID
+        #:set omp_end_directive = '!$omp end target teams distribute parallel do simd'
+    #:elif MFC_COMPILER == AMD_COMPILER_ID
+        #:set omp_end_directive = '!$omp end target teams distribute parallel do'
+    #:else
+        #:set omp_end_directive = '!$omp end target teams loop'
+    #:endif
+
+    $:omp_end_directive
+#:enddef
+
 #:def OMP_ROUTINE(function_name, nohost, extraOmpArgs)
     #:assert isinstance(nohost, bool)
     #:if nohost == True

src/common/include/parallel_macros.fpp

@@ -19,13 +19,13 @@
 
 #:enddef
 
-#:def GPU_PARALLEL_LOOP(code, collapse=None, private=None, parallelism='[gang, vector]', &
+#:def GPU_PARALLEL_LOOP_OLD(code, collapse=None, private=None, parallelism='[gang, vector]', &
     & default='present', firstprivate=None, reduction=None, reductionOp=None, &
     & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
     & no_create=None, present=None, deviceptr=None, attach=None, extraAccArgs=None, extraOmpArgs=None)
 
-    #:set acc_code = ACC_PARALLEL_LOOP(code, collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraAccArgs)
-    #:set omp_code = OMP_PARALLEL_LOOP(code, collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraOmpArgs)
+    #:set acc_code = ACC_PARALLEL_LOOP_OLD(code, collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraAccArgs)
+    #:set omp_code = OMP_PARALLEL_LOOP_OLD(code, collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraOmpArgs)
 
 #if defined(MFC_OpenACC)
     $:acc_code
@@ -36,6 +36,35 @@
 #endif
 #:enddef
 
+#:def GPU_PARALLEL_LOOP(collapse=None, private=None, parallelism='[gang, vector]', &
+    & default='present', firstprivate=None, reduction=None, reductionOp=None, &
+    & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
+    & no_create=None, present=None, deviceptr=None, attach=None, extraAccArgs=None, extraOmpArgs=None)
+
+    #:set acc_directive = ACC_PARALLEL_LOOP(collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraAccArgs)
+    #:set omp_directive = OMP_PARALLEL_LOOP(collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraOmpArgs)
+
+#if defined(MFC_OpenACC)
+    $:acc_directive
+#elif defined(MFC_OpenMP)
+    $:omp_directive
+#endif
+
+#:enddef
+
+#:def END_GPU_PARALLEL_LOOP()
+
+    #:set acc_end_directive = '!$acc end parallel loop'
+    #:set omp_end_directive = END_OMP_PARALLEL_LOOP()
+
+#if defined(MFC_OpenACC)
+    $:acc_end_directive
+#elif defined(MFC_OpenMP)
+    $:omp_end_directive
+#endif
+
+#:enddef
+
 #:def GPU_ROUTINE(function_name=None, parallelism=None, nohost=False, cray_inline=False, extraAccArgs=None, extraOmpArgs=None)
     #:assert isinstance(cray_inline, bool)
     #:set acc_directive = ACC_ROUTINE(function_name=function_name, parallelism=parallelism, nohost=nohost, extraAccArgs=extraAccArgs)

src/common/include/shared_parallel_macros.fpp

@@ -107,4 +107,4 @@
     #:endif
     $:extraArgs_val
 #:enddef
-! New line at end of file is required for FYPP
+! New line at end of file is required for FYPP