Support ./mfc.sh test on multiple GPUs

Author: henryleberre

misc/run-phoenix-release-cpu.sh

@@ -1,14 +1,16 @@
 #!/bin/bash
-#SBATCH -Jshb-test-jobs                         # Job name
-#SBATCH --account=gts-sbryngelson3               # charge account
-#SBATCH -N1 --ntasks-per-node=12                 # Number of nodes and cores per node required
-#SBATCH --mem-per-cpu=2G                         # Memory per core
-#SBATCH -t 04:00:00                              # Duration of the job (Ex: 15 mins)
-#SBATCH -q embers                               # QOS Name
-#SBATCH -otest.out                               # Combined output and error messages file
-#SBATCH -W                                      # Do not exit until the submitted job terminates.
+#SBATCH -Jshb-test-jobs            # Job name
+#SBATCH --account=gts-sbryngelson3 # charge account
+#SBATCH -N1 --ntasks-per-node=12   # Number of nodes and cores per node required
+#SBATCH --mem-per-cpu=2G           # Memory per core
+#SBATCH -t 04:00:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -q embers                  # QOS Name
+#SBATCH -otest.out                 # Combined output and error messages file
+#SBATCH -W                         # Do not exit until the submitted job terminates.
+
+cd "$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
 
-cd $SLURM_SUBMIT_DIR                            # Change to working directory
-echo $(pwd)
 . ./mfc.sh load -c p -m gpu
 ./mfc.sh test -j 12 -b mpirun -a
+

misc/run-phoenix-release-gpu.sh

@@ -1,21 +1,21 @@
 #!/bin/bash
-#SBATCH -Jshb-test-jobs                         # Job name
-#SBATCH --account=gts-sbryngelson3               # charge account
-#SBATCH -N1                                     # Number of nodes and cores per node required
+#SBATCH -Jshb-test-jobs            # Job name
+#SBATCH --account=gts-sbryngelson3 # charge account
+#SBATCH -N1                        # Number of nodes and cores per node required
 #SBATCH -CV100-16GB
 #SBATCH -G2
-#SBATCH -t 02:00:00                              # Duration of the job (Ex: 15 mins)
-#SBATCH -q embers                               # QOS Name
-#SBATCH -otest.out                               # Combined output and error messages file
-#SBATCH -W                                      # Do not exit until the submitted job terminates.
+#SBATCH -t 02:00:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -q embers                  # QOS Name
+#SBATCH -otest.out                 # Combined output and error messages file
+#SBATCH -W                         # Do not exit until the submitted job terminates.
 
-cd $SLURM_SUBMIT_DIR                            # Change to working directory
-echo $(pwd)
+cd "$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
 
-. ./mfc.sh load -c p -m GPU
+set -x
 
-nvidia-smi
-echo $(nproc)
+. ./mfc.sh load -c p -m GPU
 
+./mfc.sh test -a -b mpirun -j $(nproc) \
+              --gpu -g $(seq -s ',' 0 $(($(nvidia-smi -L | wc -l)-1)))
 
-./mfc.sh test -b mpirun -a --gpu

toolchain/mfc/args.py

@@ -81,6 +81,7 @@ def add_common_arguments(p, mask = None):
     test.add_argument("-b", "--binary",      choices=binaries, type=str, default=None, help="(Serial) Override MPI execution binary")
     test.add_argument("-r", "--relentless",  action="store_true", default=False, help="Run all tests, even if multiple fail.")
     test.add_argument("-a", "--test-all",    action="store_true", default=False, help="Run the Post Process Tests too.")
+    test.add_argument("-g", "--gpus",        type=str, default="0", help="(GPU) Comma separated list of GPU #s to use.")
 
     test.add_argument("--case-optimization", action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.")
 
@@ -146,4 +147,8 @@ def append_defaults_to_data(name: str, parser):
         if args[e] is not None:
             args[e] = os.path.abspath(args[e])
 
+    # Turn GPU ID list into a comma separated string
+    if "gpus" in args:
+        args["gpus"] = [int(g) for g in args["gpus"].split(",")]
+
     return args

toolchain/mfc/case.py

@@ -1,4 +1,4 @@
-import json, copy, dataclasses
+import json, math, copy, dataclasses
 
 @dataclasses.dataclass(init=False)
 class Case:
@@ -10,6 +10,9 @@ def __init__(self, params: dict) -> None:
     def get_parameters(self) -> str:
         return self.params.keys()
 
+    def get_cell_count(self) -> int:
+        return math.prod([max(1, int(self.params.get(dir, 0))) for dir in {"m", "n", "p"}])
+
     def has_parameter(self, key: str)-> bool:
         return key in self.get_parameters()
 

toolchain/mfc/test/case.py

@@ -100,7 +100,8 @@ def __init__(self, trace: str, mods: dict, ppn: int = None) -> None:
         self.ppn   = ppn if ppn is not None else 1
         super().__init__({**BASE_CFG.copy(), **mods})
 
-    def run(self, targets: typing.List[str]) -> subprocess.CompletedProcess:
+    def run(self, targets: typing.List[str], gpu: int) -> subprocess.CompletedProcess:
+        gpu_select        = f"CUDA_VISIBLE_DEVICES={gpu}"
         filepath          = f'"{self.get_dirpath()}/case.py"'
         tasks             = f"-n {self.ppn}"
         jobs              = f"-j {ARG('jobs')}"    if ARG("case_optimization")  else ""
@@ -110,7 +111,7 @@ def run(self, targets: typing.List[str]) -> subprocess.CompletedProcess:
         mfc_script = ".\mfc.bat" if os.name == 'nt' else "./mfc.sh"
 
         command: str = f'''\
-            {mfc_script} run {filepath} {tasks} {binary_option} \
+            {gpu_select} {mfc_script} run {filepath} {tasks} {binary_option} \
             {case_optimization} {jobs} -t {' '.join(targets)} 2>&1\
             '''
 

toolchain/mfc/test/test.py

@@ -99,6 +99,9 @@ def test():
     cons.print(f" tests/[bold magenta]UUID[/bold magenta]    Summary")
     cons.print()
 
+    # Initialize GPU_LOAD to 0 for each GPU
+    handle_case.GPU_LOAD = { id: 0 for id in ARG("gpus") }
+
     # Select the correct number of threads to use to launch test CASES
     # We can't use ARG("jobs") when the --case-optimization option is set
     # because running a test case may cause it to rebuild, and thus
@@ -124,7 +127,7 @@ def test():
 
 def handle_case(test: TestCase):
     global nFAIL
-    
+ 
     try:
         if test.params.get("qbmm", 'F') == 'T':
             tol = 1e-10
@@ -137,8 +140,12 @@ def handle_case(test: TestCase):
 
         test.delete_output()
         test.create_directory()
-        
-        cmd = test.run(["pre_process", "simulation"])
+
+        load = test.get_cell_count()
+        gpu_id = min(handle_case.GPU_LOAD.items(), key=lambda x: x[1])[0]
+        handle_case.GPU_LOAD[gpu_id] += load
+       
+        cmd = test.run(["pre_process", "simulation"], gpu=gpu_id)
 
         out_filepath = os.path.join(test.get_dirpath(), "out_pre_sim.txt")
 
@@ -178,7 +185,7 @@ def handle_case(test: TestCase):
 
         if ARG("test_all"):
             test.delete_output()
-            cmd = test.run(["pre_process", "simulation", "post_process"])
+            cmd = test.run(["pre_process", "simulation", "post_process"], gpu=gpu_id)
             out_filepath = os.path.join(test.get_dirpath(), "out_post.txt")
             common.file_write(out_filepath, cmd.stdout)
 
@@ -210,6 +217,7 @@ def handle_case(test: TestCase):
 
         cons.print(f"  [bold magenta]{test.get_uuid()}[/bold magenta]    {test.trace}")
 
+        handle_case.GPU_LOAD[gpu_id] -= load
     except Exception as exc:
         nFAIL = nFAIL + 1
 
@@ -218,3 +226,5 @@ def handle_case(test: TestCase):
 
         cons.print(f"[bold red]Failed test {test}.[/bold red]")
         cons.print(f"{exc}")
+
+handle_case.GPU_LOAD = {}