Merge branch 'master' into cantera-pyro

Author: sbryngelson

.github/workflows/phoenix/submit-bench.sh

@@ -42,7 +42,7 @@ sbatch <<EOT
 #SBATCH --account=gts-sbryngelson3 # charge account
 #SBATCH -N1                        # Number of nodes required
 $sbatch_device_opts
-#SBATCH -t 02:00:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -t 03:00:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -q embers                  # QOS Name
 #SBATCH -o$job_slug.out            # Combined output and error messages file
 #SBATCH -W                         # Do not exit until the submitted job terminates.

docs/documentation/case.md

@@ -745,6 +745,7 @@ Details of the transducer acoustic source model can be found in [Maeda and Colon
 | `mu_v` †	        | Real 		|	Viscosity |
 | `k_v` †	          | Real 		|	Thermal conductivity |
 | `cp_v` †	        | Real 		|	Specific heat capacity |
+| `D_v` †           | Real    | Vapor diffusivity in the gas |
 
 These options work only for gas-liquid two-component flows.
 Component indexes are required to be 1 for liquid and 2 for gas.
@@ -836,7 +837,6 @@ When ``polytropic = 'F'``, the gas compression is modeled as non-polytropic due
 | `T0`                  | Real    | Reference temperature                                     |
 | `x0`                  | Real    | Reference length                                          |
 | `Thost`               | Real    | Temperature of the surrounding liquid (host)              |
-| `diffcoefvap`         | Real    | Vapor diffusivity in the gas                              |
 
 - `nBubs_glb` Total number of bubbles. Their initial conditions need to be specified in the ./input/lag_bubbles.dat file. See the example cases for additional information.
 

docs/documentation/gpuParallelization.md

@@ -44,14 +44,16 @@ Note: Ordering is not guaranteed or stable, so use key-value pairing when using
 
 **Macro Invocation**
 
-Uses FYPP eval directive using `#:call`
+In order to parallelize a loop, simply place two macro calls on either end of the loop:
 
 ```C
-#:call GPU_PARALLEL_LOOP(...)
+$:$GPU_PARALLEL_LOOP(...)
    {code}
-#:endcall GPU_PARALLEL_LOOP
+$:END_GPU_PARALLEL_LOOP()
 ```
 
+This wraps the lines in `code` with parallelization calls to openACC or openMP, depending on environment and compiler settings.
+
 **Parameters**
 
 | name             | data type           | Default Value     | description                                                                               |

examples/1D_qbmm/case.py

@@ -25,6 +25,7 @@
 M_n = 28.97
 mu_n = 1.8e-05
 k_n = 0.02556
+D = 0.242e-4
 
 # air props
 # gamma_gas = gamma_n
@@ -163,6 +164,7 @@
             "fluid_pp(2)%M_v": M_n,
             "fluid_pp(2)%mu_v": mu_n,
             "fluid_pp(2)%k_v": k_n,
+            "fluid_pp(2)%D_v": D,
             # Non-polytropic gas compression model AND/OR Tait EOS
             "pref": p0,
             "rhoref": rho0,

examples/3D_lagrange_bubblescreen/case.py

@@ -155,7 +155,6 @@
             "lag_params%rho0": rho0,
             "lag_params%T0": T0,
             "lag_params%x0": x0,
-            "lag_params%diffcoefvap": diffVapor,
             "lag_params%Thost": T_host,
             # Fluids Physical Parameters
             # Host medium
@@ -177,6 +176,7 @@
             "fluid_pp(2)%M_v": MW_g,
             "fluid_pp(2)%k_v": k_g,
             "fluid_pp(2)%cp_v": cp_g,
+            "fluid_pp(2)%D_v": diffVapor,
         }
     )
 )

examples/3D_lagrange_shbubcollapse/case.py

@@ -157,7 +157,6 @@
             "lag_params%rho0": rho0,
             "lag_params%T0": T0,
             "lag_params%x0": x0,
-            "lag_params%diffcoefvap": diffVapor,
             "lag_params%Thost": T_host,
             # Fluids Physical Parameters
             # Host medium
@@ -179,6 +178,7 @@
             "fluid_pp(2)%M_v": MW_g,
             "fluid_pp(2)%k_v": k_g,
             "fluid_pp(2)%cp_v": cp_g,
+            "fluid_pp(2)%D_v": diffVapor,
         }
     )
 )

src/common/include/acc_macros.fpp

@@ -129,7 +129,7 @@
     $:end_acc_directive
 #:enddef
 
-#:def ACC_PARALLEL_LOOP(code, collapse=None, private=None, parallelism='[gang, vector]', &
+#:def ACC_PARALLEL_LOOP_OLD(code, collapse=None, private=None, parallelism='[gang, vector]', &
     & default='present', firstprivate=None, reduction=None, reductionOp=None, &
     & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
     & no_create=None, present=None, deviceptr=None, attach=None, extraAccArgs=None)
@@ -161,6 +161,35 @@
     $:acc_end_directive
 #:enddef
 
+#:def ACC_PARALLEL_LOOP(collapse=None, private=None, parallelism='[gang, vector]', &
+    & default='present', firstprivate=None, reduction=None, reductionOp=None, &
+    & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
+    & no_create=None, present=None, deviceptr=None, attach=None, extraAccArgs=None)
+    #:set collapse_val = GEN_COLLAPSE_STR(collapse)
+    #:set parallelism_val = GEN_PARALLELISM_STR(parallelism)
+    #:set default_val = GEN_DEFAULT_STR(default)
+    #:set private_val = GEN_PRIVATE_STR(private, False).strip('\n') + GEN_PRIVATE_STR(firstprivate, True).strip('\n')
+    #:set reduction_val = GEN_REDUCTION_STR(reduction, reductionOp)
+    #:set copy_val = GEN_COPY_STR(copy)
+    #:set copyin_val = GEN_COPYIN_STR(copyin, False).strip('\n') + GEN_COPYIN_STR(copyinReadOnly, True).strip('\n')
+    #:set copyout_val = GEN_COPYOUT_STR(copyout)
+    #:set create_val = GEN_CREATE_STR(create)
+    #:set no_create_val = GEN_NOCREATE_STR(no_create)
+    #:set present_val = GEN_PRESENT_STR(present)
+    #:set deviceptr_val = GEN_DEVICEPTR_STR(deviceptr)
+    #:set attach_val = GEN_ATTACH_STR(attach)
+    #:set extraAccArgs_val = GEN_EXTRA_ARGS_STR(extraAccArgs)
+    #:set clause_val = collapse_val.strip('\n') + parallelism_val.strip('\n') + &
+        & default_val.strip('\n') + private_val.strip('\n') + reduction_val.strip('\n') + &
+        & copy_val.strip('\n') + copyin_val.strip('\n') + &
+        & copyout_val.strip('\n') + create_val.strip('\n') + &
+        & no_create_val.strip('\n') + present_val.strip('\n') + &
+        & deviceptr_val.strip('\n') + attach_val.strip('\n')
+    #:set acc_directive = '!$acc parallel loop ' + &
+        & clause_val + extraAccArgs_val.strip('\n')
+    $:acc_directive
+#:enddef
+
 #:def ACC_ROUTINE(function_name=None, parallelism=None, nohost=False, extraAccArgs=None)
     #:set parallelism_val = GEN_PARALLELISM_STR(parallelism)
     #:assert isinstance(nohost, bool)
@@ -308,4 +337,4 @@
     #:set acc_directive = '!$acc wait ' + clause_val + extraAccArgs_val.strip('\n')
     $:acc_directive
 #:enddef
-! New line at end of file is required for FYPP
+! New line at end of file is required for FYPP

src/common/include/omp_macros.fpp

@@ -149,7 +149,7 @@
     $:omp_end_directive
 #:enddef
 
-#:def OMP_PARALLEL_LOOP(code, collapse=None, private=None, parallelism='[gang, vector]', &
+#:def OMP_PARALLEL_LOOP_OLD(code, collapse=None, private=None, parallelism='[gang, vector]', &
     & default='present', firstprivate=None, reduction=None, reductionOp=None, &
     & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
     & no_create=None, present=None, deviceptr=None, attach=None, extraOmpArgs=None)
@@ -196,6 +196,62 @@
     $:omp_end_directive
 #:enddef
 
+#:def OMP_PARALLEL_LOOP(collapse=None, private=None, parallelism='[gang, vector]', &
+    & default='present', firstprivate=None, reduction=None, reductionOp=None, &
+    & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
+    & no_create=None, present=None, deviceptr=None, attach=None, extraOmpArgs=None)
+    
+    #:set collapse_val = GEN_COLLAPSE_STR(collapse)
+    #:set parallelism_val = OMP_PARALLELISM_STR(parallelism)
+    #:set default_val = OMP_DEFAULT_STR(default)
+    #:set private_val = GEN_PRIVATE_STR(private, False).strip('\n') + GEN_PRIVATE_STR(firstprivate, True).strip('\n')
+    #:set reduction_val = GEN_REDUCTION_STR(reduction, reductionOp)
+    #:set copy_val = OMP_COPY_STR(copy)
+    #:set copyin_val = OMP_COPYIN_STR(copyin).strip('\n') + OMP_COPYIN_STR(copyinReadOnly).strip('\n')
+    #:set copyout_val = OMP_COPYOUT_STR(copyout)
+    #:set create_val = OMP_CREATE_STR(create)
+    #:set no_create_val = OMP_NOCREATE_STR(no_create)
+    #:set present_val = OMP_PRESENT_STR(present)
+    #:set deviceptr_val = OMP_DEVICEPTR_STR(deviceptr)
+    #:set attach_val = OMP_MAP_STR('always,tofrom', attach)
+    #:set extraOmpArgs_val = GEN_EXTRA_ARGS_STR(extraOmpArgs)
+    #:set clause_val = collapse_val.strip('\n') + parallelism_val.strip('\n') + &
+        & default_val.strip('\n') + private_val.strip('\n') + reduction_val.strip('\n') + &
+        & copy_val.strip('\n') + copyin_val.strip('\n') + &
+        & copyout_val.strip('\n') + create_val.strip('\n') + &
+        & no_create_val.strip('\n') + present_val.strip('\n') + &
+        & deviceptr_val.strip('\n') + attach_val.strip('\n')
+    #! Hardcoding the parallelism for now
+
+    #:if MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID
+        #:set omp_start_directive = '!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) '
+    #:elif MFC_COMPILER == CCE_COMPILER_ID
+        #:set omp_start_directive = '!$omp target teams distribute parallel do simd defaultmap(firstprivate:scalar) '
+    #:elif MFC_COMPILER == AMD_COMPILER_ID
+        #:set omp_start_directive = '!$omp target teams distribute parallel do '
+    #:else
+        #:set omp_start_directive = '!$omp target teams loop defaultmap(firstprivate:scalar) bind(teams,parallel) '
+    #:endif
+
+    #:set omp_directive = omp_start_directive + clause_val + extraOmpArgs_val.strip('\n')
+    $:omp_directive
+#:enddef
+
+#:def END_OMP_PARALLEL_LOOP()
+    
+    #:if MFC_COMPILER == NVIDIA_COMPILER_ID or MFC_COMPILER == PGI_COMPILER_ID
+        #:set omp_end_directive = '!$omp end target teams loop'
+    #:elif MFC_COMPILER == CCE_COMPILER_ID
+        #:set omp_end_directive = '!$omp end target teams distribute parallel do simd'
+    #:elif MFC_COMPILER == AMD_COMPILER_ID
+        #:set omp_end_directive = '!$omp end target teams distribute parallel do'
+    #:else
+        #:set omp_end_directive = '!$omp end target teams loop'
+    #:endif
+
+    $:omp_end_directive
+#:enddef
+
 #:def OMP_ROUTINE(function_name, nohost, extraOmpArgs)
     #:assert isinstance(nohost, bool)
     #:if nohost == True

src/common/include/parallel_macros.fpp

@@ -19,13 +19,13 @@
 
 #:enddef
 
-#:def GPU_PARALLEL_LOOP(code, collapse=None, private=None, parallelism='[gang, vector]', &
+#:def GPU_PARALLEL_LOOP_OLD(code, collapse=None, private=None, parallelism='[gang, vector]', &
     & default='present', firstprivate=None, reduction=None, reductionOp=None, &
     & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
     & no_create=None, present=None, deviceptr=None, attach=None, extraAccArgs=None, extraOmpArgs=None)
 
-    #:set acc_code = ACC_PARALLEL_LOOP(code, collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraAccArgs)
-    #:set omp_code = OMP_PARALLEL_LOOP(code, collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraOmpArgs)
+    #:set acc_code = ACC_PARALLEL_LOOP_OLD(code, collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraAccArgs)
+    #:set omp_code = OMP_PARALLEL_LOOP_OLD(code, collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraOmpArgs)
 
 #if defined(MFC_OpenACC)
     $:acc_code
@@ -36,6 +36,35 @@
 #endif
 #:enddef
 
+#:def GPU_PARALLEL_LOOP(collapse=None, private=None, parallelism='[gang, vector]', &
+    & default='present', firstprivate=None, reduction=None, reductionOp=None, &
+    & copy=None, copyin=None, copyinReadOnly=None, copyout=None, create=None, &
+    & no_create=None, present=None, deviceptr=None, attach=None, extraAccArgs=None, extraOmpArgs=None)
+
+    #:set acc_directive = ACC_PARALLEL_LOOP(collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraAccArgs)
+    #:set omp_directive = OMP_PARALLEL_LOOP(collapse, private, parallelism, default, firstprivate, reduction, reductionOp, copy, copyin, copyinReadOnly, copyout, create, no_create, present, deviceptr, attach, extraOmpArgs)
+
+#if defined(MFC_OpenACC)
+    $:acc_directive
+#elif defined(MFC_OpenMP)
+    $:omp_directive
+#endif
+
+#:enddef
+
+#:def END_GPU_PARALLEL_LOOP()
+
+    #:set acc_end_directive = '!$acc end parallel loop'
+    #:set omp_end_directive = END_OMP_PARALLEL_LOOP()
+
+#if defined(MFC_OpenACC)
+    $:acc_end_directive
+#elif defined(MFC_OpenMP)
+    $:omp_end_directive
+#endif
+
+#:enddef
+
 #:def GPU_ROUTINE(function_name=None, parallelism=None, nohost=False, cray_inline=False, extraAccArgs=None, extraOmpArgs=None)
     #:assert isinstance(cray_inline, bool)
     #:set acc_directive = ACC_ROUTINE(function_name=function_name, parallelism=parallelism, nohost=nohost, extraAccArgs=extraAccArgs)

src/common/include/shared_parallel_macros.fpp

@@ -107,4 +107,4 @@
     #:endif
     $:extraArgs_val
 #:enddef
-! New line at end of file is required for FYPP
+! New line at end of file is required for FYPP