Merge remote-tracking branch 'upstream/master' into MPIRefactor

Author: wilfonba

.github/pull_request_template.md

@@ -54,5 +54,5 @@ To make sure the code is performing as expected on GPU devices, I have:
 - [ ] Ran the code on MI200+ GPUs and ensure the new features performed as expected (the GPU results match the CPU results)
 - [ ] Enclosed the new feature via `nvtx` ranges so that they can be identified in profiles
 - [ ] Ran a Nsight Systems profile using `./mfc.sh run XXXX --gpu -t simulation --nsys`, and have attached the output file (`.nsys-rep`) and plain text results to this PR
-- [ ] Ran an Omniperf profile using `./mfc.sh run XXXX --gpu -t simulation --omniperf`, and have attached the output file and plain text results to this PR.
+- [ ] Ran a Rocprof Systems profile using `./mfc.sh run XXXX --gpu -t simulation --rsys --hip-trace`, and have attached the output file and plain text results to this PR.
 - [ ] Ran my code using various numbers of different GPUs (1, 2, and 8, for example) in parallel and made sure that the results scale similarly to what happens if you run without the new code/feature

.github/workflows/frontier/build.sh

@@ -1,4 +1,9 @@
 #!/bin/bash
 
+build_opts=""
+if [ "$1" == "gpu" ]; then
+    build_opts="--gpu"
+fi
+
 . ./mfc.sh load -c f -m g
-./mfc.sh test --dry-run -j 8 --gpu
+./mfc.sh test --dry-run -j 8 $build_opts

.github/workflows/frontier/submit.sh

@@ -13,16 +13,29 @@ else
     exit 1
 fi
 
+if [ "$2" == "cpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 32                       # Number of cores required"
+elif [ "$2" == "gpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 8                       # Number of cores required"
+else
+    usage
+    exit 1
+fi
+
+
 job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
 
 sbatch <<EOT
 #!/bin/bash
 #SBATCH -JMFC-$job_slug            # Job name
 #SBATCH -A CFD154                  # charge account
 #SBATCH -N 1                       # Number of nodes required
-#SBATCH -n 8                       # Number of cores required
+$sbatch_device_opts
 #SBATCH -t 01:59:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -o$job_slug.out            # Combined output and error messages file
+#SBATCH -p extended                # Extended partition for shorter queues
 #SBATCH -q debug                   # Use debug QOS - only one job per user allowed in queue!
 #SBATCH -W                         # Do not exit until the submitted job terminates.
 

.github/workflows/frontier/test.sh

@@ -3,5 +3,8 @@
 gpus=`rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' '`
 ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
 
-./mfc.sh test --max-attempts 3 -j $ngpus -- -c frontier
-
+if [ "$job_device" == "gpu" ]; then
+    ./mfc.sh test --max-attempts 3 -j $ngpus -- -c frontier
+else
+    ./mfc.sh test --max-attempts 3 -j 32 -- -c frontier
+fi

.github/workflows/phoenix/bench.sh

@@ -9,7 +9,7 @@ if [ "$job_device" == "gpu" ]; then
 fi
 
 if ["$job_device" == "gpu"]; then
-    ./mfc.sh bench --mem 12 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix $device_opts -n $n_ranks
+    ./mfc.sh bench --mem 12 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks
 else
-    ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix $device_opts -n $n_ranks
+    ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks
 fi

.github/workflows/phoenix/submit.sh

@@ -42,7 +42,7 @@ sbatch <<EOT
 #SBATCH --account=gts-sbryngelson3 # charge account
 #SBATCH -N1                        # Number of nodes required
 $sbatch_device_opts
-#SBATCH -t 02:00:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -t 03:00:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -q embers                  # QOS Name
 #SBATCH -o$job_slug.out            # Combined output and error messages file
 #SBATCH -W                         # Do not exit until the submitted job terminates.

.github/workflows/test.yml

@@ -97,9 +97,6 @@ jobs:
       matrix:
         device: ['cpu', 'gpu']
         lbl: ['gt', 'frontier']
-        exclude:
-          - device:   cpu
-            lbl: frontier
     runs-on:
       group:  phoenix
       labels: ${{ matrix.lbl }}
@@ -116,7 +113,7 @@ jobs:
 
       - name: Build
         if:   matrix.lbl == 'frontier'
-        run:  bash .github/workflows/frontier/build.sh
+        run:  bash .github/workflows/frontier/build.sh ${{ matrix.device }}
 
       - name: Test
         if:   matrix.lbl == 'frontier'

docs/documentation/running.md

@@ -98,13 +98,13 @@ Learn more about NVIDIA Nsight Compute [here](https://docs.nvidia.com/nsight-com
 
 
 #### AMD GPUs
-- Rocprof (ROC): `./mfc.sh run ... -t simulation --roc --hip-trace [rocprof flags]` allows one to visualize MFC's system-wide performance with [Perfetto UI](https://ui.perfetto.dev/).
+- Rocprof Systems (RSYS): `./mfc.sh run ... -t simulation --rsys --hip-trace [rocprof flags]` allows one to visualize MFC's system-wide performance with [Perfetto UI](https://ui.perfetto.dev/).
 When used, `--roc` will run the simulation and generate files in the case directory for all targets.
 `results.json` can then be imported in [Perfetto's UI](https://ui.perfetto.dev/).
 Learn more about AMD Rocprof [here](https://rocm.docs.amd.com/projects/rocprofiler/en/docs-5.5.1/rocprof.html)
 It is best to run case files with few timesteps to keep the report file sizes manageable.
-- Omniperf (OMNI): `./mfc.sh run ... -t simulation --omni [omniperf flags]` allows one to conduct kernel-level profiling with [AMD's Omniperf](https://rocm.docs.amd.com/projects/omniperf/en/latest/index.html).
-When used, `--omni` will output profiling information for all subroutines, including rooflines, cache usage, register usage, and more, after the simulation is run.
+- Rocprof Compute (RCU): `./mfc.sh run ... -t simulation --rcu -n <name> [rocprof-compute flags]` allows one to conduct kernel-level profiling with [ROCm Compute Profiler](https://rocm.docs.amd.com/projects/rocprofiler-compute/en/latest/what-is-rocprof-compute.html).
+When used, `--rcu` will output profiling information for all subroutines, including rooflines, cache usage, register usage, and more, after the simulation is run.
 Adding this argument will moderately slow down the simulation and run the MFC executable several times.
 For this reason, it should only be used with case files with few timesteps.
 

misc/m_silo_proxy.f90

@@ -44,7 +44,7 @@ module m_silo_proxy
 
     !> @brief  Refer to page 28 of Silo's user guide (10/2007, v4.6) for
                 !!                           information about this subroutine
-    function DBCREATE(pathname, lpathname, mode, target, &
+   impure  function DBCREATE(pathname, lpathname, mode, target, &
                       fileinfo, lfileinfo, filetype, status)
 
         integer                                                   :: DBCREATE
@@ -66,7 +66,7 @@ end function DBCREATE
 
     !> @brief  Refer to page 235 of Silo's user guide (10/2007, v4.6)
                 !!                           for information about this subroutine
-    function DBGET2DSTRLEN()
+    impure function DBGET2DSTRLEN()
 
         integer :: DBGET2DSTRLEN
 
@@ -79,7 +79,7 @@ end function DBGET2DSTRLEN
 
     !> @brief  Refer to page 234 of Silo's user guide (10/2007, v4.6)
                 !!                           for information about this subroutine
-    function DBSET2DSTRLEN(len)
+    impure function DBSET2DSTRLEN(len)
 
         integer                                :: DBSET2DSTRLEN
         integer, intent(IN) :: len
@@ -93,7 +93,7 @@ end function DBSET2DSTRLEN
 
     !> @brief  Refer to page 185 of Silo's user guide (10/2007, v4.6)
                 !!                           for information about this subroutine
-    function DBMKOPTLIST(maxopts, optlist_id)
+    impure function DBMKOPTLIST(maxopts, optlist_id)
 
         integer                                :: DBMKOPTLIST
         integer, intent(IN) :: maxopts
@@ -107,7 +107,7 @@ end function DBMKOPTLIST
 
     !> @brief  Refer to page 186 of Silo's user guide (10/2007, v4.6)
                 !!                           for information about this subroutine
-    function DBADDIOPT(optlist_id, option, ivalue)
+    impure function DBADDIOPT(optlist_id, option, ivalue)
 
         integer                                :: DBADDIOPT
         integer, intent(IN) :: optlist_id
@@ -123,7 +123,7 @@ end function DBADDIOPT
 
     !> @brief  Refer to page 186 of Silo's user guide (10/2007, v4.6)
                 !!                           for information about this subroutine
-    function DBADDDOPT(optlist_id, option, dvalue)
+    impure function DBADDDOPT(optlist_id, option, dvalue)
 
         integer                                                                :: DBADDDOPT
         integer, intent(IN) :: optlist_id
@@ -139,7 +139,7 @@ end function DBADDDOPT
 
     !> @brief  Refer to page 121 of Silo's user guide (10/2007, v4.6)
                 !!                           for information about this subroutine
-    function DBPUTMMESH(dbid, name, lname, nmesh, meshnames, &
+    impure function DBPUTMMESH(dbid, name, lname, nmesh, meshnames, &
                         lmeshnames, meshtypes, optlist_id, status)
 
         integer                                                                                 :: DBPUTMMESH
@@ -162,7 +162,7 @@ end function DBPUTMMESH
 
     !> @brief  Refer to page 189 of Silo's user guide (10/2007, v4.6)
                 !!                           for information about this subroutine
-    function DBFREEOPTLIST(optlist_id)
+    impure function DBFREEOPTLIST(optlist_id)
 
         integer                                :: DBFREEOPTLIST
         integer, intent(IN) :: optlist_id
@@ -175,7 +175,7 @@ end function DBFREEOPTLIST
 
     !> @brief  Refer to page 57 of Silo's user guide (10/2007, v4.6) for
                 !!                           information about this subroutine
-    function DBPUTQM(dbid, name, lname, xname, lxname, yname, lyname, &
+    impure function DBPUTQM(dbid, name, lname, xname, lxname, yname, lyname, &
                      zname, lzname, x, y, z, dims, ndims, datatype, &
                      coordtype, optlist_id, status)
 
@@ -208,7 +208,7 @@ end function DBPUTQM
 
     !> @brief  Refer to page 46 of Silo's user guide (10/2007, v4.6) for
                 !!                           information about this subroutine
-    function DBPUTCURVE(dbid, curvename, lcurvename, xvals, yvals, &
+    impure function DBPUTCURVE(dbid, curvename, lcurvename, xvals, yvals, &
                         datatype, npoints, optlist_id, status)
 
         integer                                                                                 :: DBPUTCURVE
@@ -231,7 +231,7 @@ end function DBPUTCURVE
 
     !> @brief  Refer to page 130 of Silo's user guide (10/2007, v4.6)
                 !!                           for information about this subroutine
-    function DBPUTMVAR(dbid, name, lname, nvar, varnames, lvarnames, &
+    impure function DBPUTMVAR(dbid, name, lname, nvar, varnames, lvarnames, &
                        vartypes, optlist_id, status)
 
         integer                                                                                 :: DBPUTMVAR
@@ -254,7 +254,7 @@ end function DBPUTMVAR
 
     !> @brief  Refer to page 64 of Silo's user guide (10/2007, v4.6) for
                 !!                           information about this subroutine
-    function DBPUTQV1(dbid, name, lname, meshname, lmeshname, var, &
+    impure function DBPUTQV1(dbid, name, lname, meshname, lmeshname, var, &
                       dims, ndims, mixvar, mixlen, datatype, &
                       centering, optlist_id, status)
 
@@ -283,7 +283,7 @@ end function DBPUTQV1
 
     !> @brief  Refer to page 31 of Silo's user guide (10/2007, v4.6) for
                 !!                           information about this subroutine
-    function DBCLOSE(dbid)
+    impure function DBCLOSE(dbid)
 
         integer                                :: DBCLOSE
         integer, intent(IN) :: dbid