Merge branch 'master' into containerization

Author: sbryngelson

.fortitude.toml

@@ -1,6 +1,4 @@
 coverage:
-  strict_yaml_branch: default
-  max_report_age: off
   status:
     project:
       default:

.github/codecov.yml

@@ -30,7 +30,9 @@ jobs:
       - name: Setup Ubuntu
         run: |
             sudo apt update -y
-            sudo apt install -y tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev libblas-dev liblapack-dev
+            sudo apt install -y tar wget make cmake gcc g++ python3 \
+                python3-dev "openmpi-*" libopenmpi-dev hdf5-tools \
+                libfftw3-dev libhdf5-dev libblas-dev liblapack-dev
 
       - name: Build
         run: /bin/bash mfc.sh build -j $(nproc) --gcov

.github/workflows/coverage.yml

@@ -13,6 +13,6 @@ if [ "$2" == "bench" ]; then
         ./mfc.sh run "$dir/case.py" --case-optimization -j 8 --dry-run $build_opts
     done
 else
-    ./mfc.sh test -a --dry-run --rdma-mpi --generate -j 8 $build_opts
+    ./mfc.sh test -a --dry-run --rdma-mpi -j 8 $build_opts
 fi
 

.github/workflows/frontier/build.sh

@@ -29,11 +29,11 @@ job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
 
 sbatch <<EOT
 #!/bin/bash
-#SBATCH -JMFC-$job_slug            # Job name
+#SBATCH -J MFC-$job_slug            # Job name
 #SBATCH -A CFD154                  # charge account
 #SBATCH -N 1                       # Number of nodes required
 $sbatch_device_opts
-#SBATCH -t 03:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -t 04:59:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -o$job_slug.out            # Combined output and error messages file
 #SBATCH -p extended                # Extended partition for shorter queues
 #SBATCH -W                         # Do not exit until the submitted job terminates.

.github/workflows/frontier/submit.sh

@@ -6,5 +6,5 @@ ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
 if [ "$job_device" = "gpu" ]; then
     ./mfc.sh test -a --rdma-mpi --max-attempts 3 -j $ngpus -- -c frontier
 else
-    ./mfc.sh test -a --rdma-mpi --max-attempts 3 -j 32 -- -c frontier
+    ./mfc.sh test -a --max-attempts 3 -j 32 -- -c frontier
 fi

.github/workflows/frontier/test.sh

@@ -28,12 +28,6 @@ jobs:
     - name: Initialize MFC
       run: ./mfc.sh init
 
-    - name: Lint the full source
-      run: |
-        source build/venv/bin/activate
-        find ./src -type f -not -name '*nvtx*' -exec sh -c 'fortitude check "$1" | grep -v E001' _ {} \;
-        find ./src -type f -not -name '*nvtx*' -exec sh -c 'fortitude check "$1" | grep -v E001' _ {} \; | wc -l | xargs -I{} sh -c '[ {} -gt 0 ] && exit 1 || exit 0'
-
     - name: Looking for raw directives
       run: |
         ! grep -iR '!\$acc\|!\$omp' --exclude="parallel_macros.fpp" --exclude="syscheck.fpp" ./src/*

.github/workflows/lint-source.yml

@@ -2,8 +2,7 @@
 
 n_ranks=12
 
-echo "My benchmarking device is:" $device
-if [ "$device" = "gpu" ]; then
+if [ "$job_device" = "gpu" ]; then
     n_ranks=$(nvidia-smi -L | wc -l)        # number of GPUs on node
     gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1
     device_opts="--gpu -g $gpu_ids"
@@ -16,7 +15,7 @@ mkdir -p $currentdir
 
 export TMPDIR=$currentdir
 
-if [ "$device" = "gpu" ]; then
+if [ "$job_device" = "gpu" ]; then
     ./mfc.sh bench --mem 12 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks
 else
     ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks

.github/workflows/phoenix/bench.sh

@@ -1,107 +1,64 @@
-#!/usr/bin/env bash
-set -euo pipefail
+#!/bin/bash
+
+set -e
 
 usage() {
     echo "Usage: $0 [script.sh] [cpu|gpu]"
-    exit 1
 }
 
-[[ $# -eq 2 ]] || usage
-
-sbatch_script="$1"
-
-device="$2"
-job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
-
-# read the body of the user script
-sbatch_body=$(<"$sbatch_script")
-
-# common SBATCH directives
-sbatch_common_opts="\
-#SBATCH -J shb-${sbatch_script%%.sh}-$device    # job name
-#SBATCH --account=gts-sbryngelson3              # account
-#SBATCH -N1                                     # nodes
-#SBATCH -t 02:00:00                             # walltime
-#SBATCH -q embers                               # QOS
-#SBATCH -o $job_slug.out                        # stdout+stderr
-#SBATCH --mem-per-cpu=2G                        # default mem (overridden below)
-"
+if [ ! -z "$1" ]; then
+    sbatch_script_contents=`cat $1`
+else
+    usage
+    exit 1
+fi
 
-# CPU vs GPU overrides
-if [[ "$device" == "cpu" ]]; then
-  sbatch_device_opts="\
+sbatch_cpu_opts="\
 #SBATCH -p cpu-small               # partition
 #SBATCH --ntasks-per-node=24       # Number of cores per node required
 #SBATCH --mem-per-cpu=2G           # Memory per core\
 "
-elif [[ "$device" == "gpu" ]]; then
-  sbatch_device_opts="\
+
+sbatch_gpu_opts="\
 #SBATCH -CL40S
 #SBATCH --ntasks-per-node=4       # Number of cores per node required
 #SBATCH -G2\
 "
+
+if [ "$2" = "cpu" ]; then
+    sbatch_device_opts="$sbatch_cpu_opts"
+elif [ "$2" = "gpu" ]; then
+    sbatch_device_opts="$sbatch_gpu_opts"
 else
-  usage
+    usage
+    exit 1
 fi
 
-# submit and capture the JobID
-JOBID=$(sbatch <<-EOT | awk '{print $4}'
-	#!/usr/bin/env bash
-	${sbatch_common_opts}
-	${sbatch_device_opts}
- 
-	export job_slug="${job_slug}"
-	export device="${device}"
-
-	echo "Job slug is:" $job_slug
- 	echo "Device is:" $device
- 
-	set -e -x
-
-	cd "\$SLURM_SUBMIT_DIR"
-	echo "Running in \$(pwd):"
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
 
-	# load your modules & env
-	. ./mfc.sh load -c p -m $device
+sbatch <<EOT
+#!/bin/bash
+#SBATCH -Jshb-$job_slug            # Job name
+#SBATCH --account=gts-sbryngelson3 # charge account
+#SBATCH -N1                        # Number of nodes required
+$sbatch_device_opts
+#SBATCH -t 02:00:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -q embers                  # QOS Name
+#SBATCH -o$job_slug.out            # Combined output and error messages file
+#SBATCH -W                         # Do not exit until the submitted job terminates.
 
-	# user script contents
-	${sbatch_body}
-EOT
-)
+set -e
+set -x
 
-echo "Submitted: SLURM job $JOBID"
+cd "\$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
 
-# if this wrapper is killed/canceled, make sure SLURM job is cleaned up
-trap '[[ -n "${JOBID:-}" ]] && scancel "$JOBID" >/dev/null 2>&1 || :' EXIT
+job_slug="$job_slug"
+job_device="$2"
 
-# ────────── Poll until SLURM job finishes ──────────
-while :; do
-  # Try sacct first
-  STATE=$(sacct -j "$JOBID" --format=State --noheader --parsable2 | head -n1)
+. ./mfc.sh load -c p -m $2
 
-  # Fallback to squeue if sacct is empty
-  if [[ -z "$STATE" ]]; then
-    STATE=$(squeue -j "$JOBID" -h -o "%T" || echo "")
-  fi
+$sbatch_script_contents
 
-  # If it’s one of SLURM’s terminal states, break immediately
-  case "$STATE" in
-    COMPLETED|FAILED|CANCELLED|TIMEOUT|PREEMPTED)
-      echo "Completed: SLURM job $JOBID reached terminal state: $STATE"
-      break
-      ;;
-    "")
-      echo "Completed: SLURM job $JOBID no longer in queue; assuming finished"
-      break
-      ;;
-    *)
-      echo "Waiting: SLURM job $JOBID state: $STATE"
-      sleep 10
-      ;;
-  esac
-done
+EOT
 
-# Now retrieve the exit code and exit with it
-EXIT_CODE=$(sacct -j "$JOBID" --noheader --format=ExitCode | head -1 | cut -d: -f1)
-echo "Completed: SLURM job $JOBID exit code: $EXIT_CODE"
-exit "$EXIT_CODE"

.github/workflows/phoenix/submit-bench.sh

@@ -1,100 +1,64 @@
-#!/usr/bin/env bash
-set -euo pipefail
+#!/bin/bash
+
+set -e
 
 usage() {
     echo "Usage: $0 [script.sh] [cpu|gpu]"
-    exit 1
 }
 
-[[ $# -eq 2 ]] || usage
-
-sbatch_script="$1"
-device="$2"
-
-job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
-
-# read the body of the user script
-sbatch_body=$(<"$sbatch_script")
+if [ ! -z "$1" ]; then
+    sbatch_script_contents=`cat $1`
+else
+    usage
+    exit 1
+fi
 
-# common SBATCH directives
-sbatch_common_opts="\
-#SBATCH -J shb-${sbatch_script%%.sh}-$device    # job name
-#SBATCH --account=gts-sbryngelson3              # account
-#SBATCH -N1                                     # nodes
-#SBATCH -t 03:00:00                             # walltime
-#SBATCH -q embers                               # QOS
-#SBATCH -o $job_slug.out                        # stdout+stderr
-#SBATCH --mem-per-cpu=2G                        # default mem (overridden below)
+sbatch_cpu_opts="\
+#SBATCH -p cpu-small               # partition
+#SBATCH --ntasks-per-node=24       # Number of cores per node required
+#SBATCH --mem-per-cpu=2G           # Memory per core\
 "
 
-# CPU vs GPU overrides
-if [[ "$device" == "cpu" ]]; then
-  sbatch_device_opts="\
-#SBATCH -p cpu-small
-#SBATCH --ntasks-per-node=24
-"
-elif [[ "$device" == "gpu" ]]; then
-  sbatch_device_opts="\
+sbatch_gpu_opts="\
 #SBATCH -p gpu-v100,gpu-a100,gpu-h100,gpu-l40s
-#SBATCH --ntasks-per-node=4
-#SBATCH -G2
+#SBATCH --ntasks-per-node=4       # Number of cores per node required
+#SBATCH -G2\
 "
+
+if [ "$2" = "cpu" ]; then
+    sbatch_device_opts="$sbatch_cpu_opts"
+elif [ "$2" = "gpu" ]; then
+    sbatch_device_opts="$sbatch_gpu_opts"
 else
-  usage
+    usage
+    exit 1
 fi
 
-# submit and capture the JobID
-JOBID=$(sbatch <<-EOT | awk '{print $4}'
-	#!/usr/bin/env bash
-	${sbatch_common_opts}
-	${sbatch_device_opts}
-
-	set -e -x
-
-	cd "\$SLURM_SUBMIT_DIR"
-	echo "Running in \$(pwd):"
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
 
-	# load your modules & env
-	. ./mfc.sh load -c p -m $device
+sbatch <<EOT
+#!/bin/bash
+#SBATCH -Jshb-$job_slug            # Job name
+#SBATCH --account=gts-sbryngelson3 # charge account
+#SBATCH -N1                        # Number of nodes required
+$sbatch_device_opts
+#SBATCH -t 03:00:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -q embers                  # QOS Name
+#SBATCH -o$job_slug.out            # Combined output and error messages file
+#SBATCH -W                         # Do not exit until the submitted job terminates.
 
-	# user script contents
-	${sbatch_body}
-EOT
-)
+set -e
+set -x
 
-echo "Submitted: SLURM job $JOBID"
+cd "\$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
 
-# if this wrapper is killed/canceled, make sure SLURM job is cleaned up
-trap '[[ -n "${JOBID:-}" ]] && scancel "$JOBID" >/dev/null 2>&1 || :' EXIT
+job_slug="$job_slug"
+job_device="$2"
 
-# ────────── Poll until SLURM job finishes ──────────
-while :; do
-  # Try sacct first
-  STATE=$(sacct -j "$JOBID" --format=State --noheader --parsable2 | head -n1)
+. ./mfc.sh load -c p -m $2
 
-  # Fallback to squeue if sacct is empty
-  if [[ -z "$STATE" ]]; then
-    STATE=$(squeue -j "$JOBID" -h -o "%T" || echo "")
-  fi
+$sbatch_script_contents
 
-  # If it’s one of SLURM’s terminal states, break immediately
-  case "$STATE" in
-    COMPLETED|FAILED|CANCELLED|TIMEOUT|PREEMPTED)
-      echo "Completed: SLURM job $JOBID reached terminal state: $STATE"
-      break
-      ;;
-    "")
-      echo "Completed: SLURM job $JOBID no longer in queue; assuming finished"
-      break
-      ;;
-    *)
-      echo "Waiting: SLURM job $JOBID state: $STATE"
-      sleep 10
-      ;;
-  esac
-done
+EOT
 
-# Now retrieve the exit code and exit with it
-EXIT_CODE=$(sacct -j "$JOBID" --noheader --format=ExitCode | head -1 | cut -d: -f1)
-echo "Completed: SLURM job $JOBID exit code: $EXIT_CODE"
-exit "$EXIT_CODE"