Merge pull request #146 from MFlowCode/fix-143

Author: sbryngelson

mfc.sh

@@ -132,7 +132,7 @@ if [ "$1" == "load" ]; then
         MODULES=("${MODULES[@]}" "python/3.8.5")
     elif [ "$u_computer" == "p" ]; then # Phoenix
         if [ "$u_cg" == "c" ]; then
-            MODULES=("gcc/10.3.0-o57x6h" "mvapich2/2.3.6-ouywal")
+            MODULES=("gcc/10.3.0-o57x6h" "openmpi/4.1.4")
         elif [ "$u_cg" == "g" ]; then
             MODULES=("cuda/11.7.0-7sdye3" "nvhpc/22.11")
         fi

toolchain/mfc/args.py

@@ -95,6 +95,7 @@ def add_common_arguments(p, mask = None):
     run.add_argument(      "--dry-run",        action="store_true",                       default=False,      help="(Batch) Run without submitting batch file.")
     run.add_argument("--case-optimization",    action="store_true",                       default=False,      help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.")
     run.add_argument(      "--no-build",       action="store_true",                       default=False,      help="(Testing) Do not rebuild MFC.")
+    run.add_argument("--wait",                 action="store_true",                       default=False,      help="(Batch) Wait for the job to finish.")
 
     # === BENCH ===
     add_common_arguments(bench, "t")

toolchain/mfc/run/queues.py

@@ -1,7 +1,7 @@
 import os, typing, dataclasses
 
 from .. import common
-
+from ..state import ARG
 
 @dataclasses.dataclass
 class QueueSystem:
@@ -27,6 +27,9 @@ def is_active(self) -> bool:
         return common.does_command_exist("qsub")
 
     def gen_submit_cmd(self, filename: str) -> typing.List[str]:
+        if ARG("wait"):
+            raise common.MFCException("PBS Queue: Sorry, --wait is unimplemented.")
+
         return ["qsub", filename]
 
 
@@ -38,7 +41,12 @@ def is_active(self) -> bool:
         return common.does_command_exist("bsub") and common.does_command_exist("bqueues")
 
     def gen_submit_cmd(self, filename: str) -> None:
-        return ["bsub", filename]
+        cmd = ["bsub"]
+
+        if ARG("wait"):
+            cmd += ["--wait"]        
+
+        return cmd + [filename]
 
 
 class SLURMSystem(QueueSystem):
@@ -49,7 +57,12 @@ def is_active(self) -> bool:
         return common.does_command_exist("sbatch")
 
     def gen_submit_cmd(self, filename: str) -> None:
-        return ["sbatch", filename]
+        cmd = ["sbatch"]
+    
+        if ARG("wait"):
+            cmd += ["--wait"]
+
+        return cmd + [filename]
 
 
 QUEUE_SYSTEMS = [ LSFSystem(), SLURMSystem(), PBSSystem() ]

toolchain/templates/pbs.sh

@@ -65,19 +65,21 @@ for binpath in {MFC::BINARIES}; do
 
     echo -e ":) Running $binpath:"
 
-    srun                                    \
-         --nodes={nodes}                    \
-         --ntasks-per-node {tasks_per_node} \
-         {MFC::PROFILER} "$binpath"
-
-    #>
-    #> srun --mpi=pmix \
-    #>      {MFC::PROFILER} "$binpath"
-    #>
-    #> mpirun                            \
-    #>        -np {tasks_per_node*nodes} \
-    #>        {MFC::PROFILER} "$binpath"
-    #>
+    if command -v srun > /dev/null 2>&1; then
+        srun                                   \
+            --nodes           {nodes}          \
+            --ntasks-per-node {tasks_per_node} \
+            {MFC::PROFILER} "$binpath"
+        
+        #>
+        #> srun --mpi=pmix \
+        #>      {MFC::PROFILER} "$binpath"
+    else
+        mpirun                         \
+            -np {tasks_per_node*nodes} \
+            {MFC::PROFILER} "$binpath"
+    
+    fi
 
 done
 

toolchain/templates/slurm.sh

@@ -74,24 +74,21 @@ for binpath in {MFC::BINARIES}; do
 
     echo -e ":) Running $binpath:"
 
-#>
-#> Note: This MPI executable might not be well supported
-#>       on your system - if at all. {MFC::BIN} refers to
-#>       the path the MFC executable.
-#>
-#>srun                                   \
-#>     --nodes={nodes}                   \
-#>     --ntasks-per-node {cpus_per_node} \
-#>     --mpi=pmi2		                 \
-#>     {MFC::PROFILER} "{MFC::BIN}"
-#>
-#> srun --mpi=pmix                   \
-#>      {MFC::PROFILER} "{MFC::BIN}"
-#>
+    if command -v srun > /dev/null 2>&1; then
+        srun                                   \
+            --nodes           {nodes}          \
+            --ntasks-per-node {tasks_per_node} \
+            {MFC::PROFILER} "$binpath"
 
-    mpirun                         \
-        -np {nodes*tasks_per_node} \
-        {MFC::PROFILER} "$binpath"
+        #>
+        #> srun --mpi=pmix                 \
+        #>      {MFC::PROFILER} "$binpath"
+        #>
+    else
+        mpirun                         \
+            -np {nodes*tasks_per_node} \
+            {MFC::PROFILER} "$binpath"
+    fi
 
 done