Multi-target support to batch engine

Author: henryleberre

examples/1D_bubblescreen/fort.1

@@ -25,6 +25,17 @@ subroutine s_create_directory(dir_name)
 
     end subroutine s_create_directory
 
+    subroutine s_delete_file(filepath)
+        character(LEN=*), intent(IN) :: filepath
+
+#ifdef _WIN32
+        call system('del "'//filepath//'"')
+#else
+        call system('rm "'//filepath//'"')
+#endif
+
+    end subroutine s_delete_file
+
     subroutine s_delete_directory(dir_name)
         character(LEN=*), intent(IN) :: dir_name
 
@@ -70,7 +81,8 @@ subroutine s_get_basename(dirpath, basename)
         open (100, FILE='basename', FORM='formatted', STATUS='old')
         read (100, '(A)') basename; close (100)
 
-        call s_delete_directory("dir_name")
+        call s_delete_file("basename")
+
     end subroutine s_get_basename
 
 end module m_compile_specific

src/common/m_compile_specific.f90

@@ -175,7 +175,7 @@ def build_target(mfc, name: str, history: typing.List[str] = None):
 
             raise common.MFCException(f"Failed to configure the [bold magenta]{name}[/bold magenta] target.")
 
-    common.system(build, exception_text=f"Failed to build the [bold magenta]{name}[/bold magenta] target.")
+    common.system(build,   exception_text=f"Failed to build the [bold magenta]{name}[/bold magenta] target.")
     common.system(install, exception_text=f"Failed to install the [bold magenta]{name}[/bold magenta] target.")
     cons.print(no_indent=True)
 

toolchain/mfc/build.py

@@ -37,7 +37,7 @@ def _init(self) -> None:
     def get_args(self) -> typing.List[str]:
         raise common.MFCException(f"MFCEngine::get_args: not implemented for {self.name}.")
 
-    def run(self, target_name: str) -> None:
+    def run(self, names: typing.List[str]) -> None:
         raise common.MFCException(f"MFCEngine::run: not implemented for {self.name}.")
 
     def validate_job_options(self) -> None:
@@ -78,7 +78,7 @@ def get_exec_cmd(self, target_name: str) -> typing.List[str]:
         return [self.mpibin.bin] + self.mpibin.gen_params(self.mfc.args) + flags + [binpath]
 
 
-    def run(self, target_name: str) -> None:
+    def run(self, names: typing.List[str]) -> None:
         if not self.bWorks:
             # Fix MFlowCode/MFC#21: Check whether attempting to run a job will hang
             # forever. This can happen when using the wrong queue system.
@@ -116,19 +116,19 @@ def run(self, target_name: str) -> None:
                     + f"[bold magenta]Reason[/bold magenta]: {'Time limit.' if p.is_alive() else 'Exit code.'}"
                 )
 
+        for name in names:
+            cons.print(f"[bold]Running [magenta]{name}[/magenta][/bold]:")
+            cons.indent()
 
-        cons.print(f"Running [bold magenta]{target_name}[/bold magenta]:")
-        cons.indent()
+            if not self.mfc.args["dry_run"]:
+                start_time = time.monotonic()
+                common.system(self.get_exec_cmd(name), cwd=self.input.case_dirpath)
+                end_time   = time.monotonic()
+                cons.print(no_indent=True)
 
-        if not self.mfc.args["dry_run"]:
-            start_time = time.monotonic()
-            common.system(self.get_exec_cmd(target_name), cwd=self.input.case_dirpath)
-            end_time   = time.monotonic()
-            cons.print(no_indent=True)
+                cons.print(f"[bold green]Done[/bold green] (in {datetime.timedelta(seconds=end_time - start_time)})")
 
-            cons.print(f"[bold green]Done[/bold green] (in {datetime.timedelta(seconds=end_time - start_time)})")
-
-        cons.unindent()
+            cons.unindent()
 
     def validate_job_options(self, mfc) -> None:
         if mfc.args["nodes"] != 1:
@@ -150,19 +150,17 @@ def get_args(self) -> str:
 Email         (-@)  {self.mfc.args["email"]}
 """
 
-    def run(self, target_name: str) -> None:
-        cons.print(f"Running [bold magenta]{target_name}[/bold magenta]:")
-        cons.indent()
-
+    def run(self, names: typing.List[str]) -> None:
         system = queues.get_system()
         cons.print(f"Detected the [bold magenta]{system.name}[/bold magenta] queue system.")
 
-        self.__create_batch_file(system, target_name)
+        cons.print(f"Running [bold magenta]{common.format_list_to_string(names)}[/bold magenta]:")
+        cons.indent()
+
+        self.__create_batch_file(system, names)
 
         if not self.mfc.args["dry_run"]:
-            cons.print(no_indent=True)
-            self.__execute_batch_file(system, target_name)
-            cons.print(no_indent=True)
+            self.__execute_batch_file(system, names)
 
             cons.print("[bold yellow]INFO:[/bold yellow] Batch file submitted! Please check your queue system for the job status.")
             cons.print("[bold yellow]INFO:[/bold yellow] If an error occurs, please check the generated batch file and error logs for more information.")
@@ -173,16 +171,16 @@ def run(self, target_name: str) -> None:
     def __get_batch_dirpath(self) -> str:
         return copy.copy(self.input.case_dirpath)
 
-    def __get_batch_filename(self, target_name: str) -> str:
-        return f"{target_name}.sh"
+    def __get_batch_filename(self, names: typing.List[str]) -> str:
+        return f"{self.mfc.args['name']}.sh"
 
-    def __get_batch_filepath(self, target_name: str):
+    def __get_batch_filepath(self, names: typing.List[str]):
         return os.path.abspath(os.sep.join([
             self.__get_batch_dirpath(),
-            self.__get_batch_filename(target_name)
+            self.__get_batch_filename(names)
         ]))
 
-    def __generate_prologue(self, system: queues.QueueSystem,) -> str:
+    def __generate_prologue(self, system: queues.QueueSystem, names: typing.List[str]) -> str:
         modules = f""
 
         if common.does_system_use_modules():
@@ -270,11 +268,11 @@ def __evaluate_expression(self, expr: str) -> str:
         except Exception as exc:
             raise common.MFCException(f"BatchEngine: {expr_original} (interpreted as {expr}) is not a valid expression in the template file. Please check your spelling.")
 
-    def __batch_evaluate(self, s: str, system: queues.QueueSystem, target_name: str):
+    def __batch_evaluate(self, s: str, system: queues.QueueSystem, names: typing.List[str]):
         replace_list = [
-            ("{MFC::PROLOGUE}", self.__generate_prologue(system)),
+            ("{MFC::PROLOGUE}", self.__generate_prologue(system, names)),
             ("{MFC::EPILOGUE}", self.__generate_epilogue()),
-            ("{MFC::BIN}",      self.get_binpath(target_name))
+            ("{MFC::BINARIES}", ' '.join([f"'{self.get_binpath(x)}'" for x in names]))
         ]
 
         for (key, value) in replace_list:
@@ -298,27 +296,25 @@ def __batch_evaluate(self, s: str, system: queues.QueueSystem, target_name: str)
 
         return s
 
-    def __create_batch_file(self, system: queues.QueueSystem, target_name: str):
-        cons.print("> > Generating batch file...")
-        filepath = self.__get_batch_filepath(target_name)
-        cons.print("> > Evaluating template file...")
-        content = self.__batch_evaluate(system.template, system, target_name)
+    def __create_batch_file(self, system: queues.QueueSystem, names: typing.List[str]):
+        cons.print("> Generating batch file...")
+        filepath = self.__get_batch_filepath(names)
+        cons.print("> Evaluating template file...")
+        content = self.__batch_evaluate(system.template, system, names)
 
-        cons.print("> > Writing batch file...")
+        cons.print("> Writing batch file...")
         common.file_write(filepath, content)
 
-    def __execute_batch_file(self, system: queues.QueueSystem, target_name: str):
+    def __execute_batch_file(self, system: queues.QueueSystem, names: typing.List[str]):
         # We CD to the case directory before executing the batch file so that
         # any files the queue system generates (like .err and .out) are created
         # in the correct directory.
-        cmd = system.gen_submit_cmd(self.__get_batch_filename(target_name))
+        cmd = system.gen_submit_cmd(self.__get_batch_filename(names))
 
         if common.system(cmd, cwd=self.__get_batch_dirpath()) != 0:
             raise common.MFCException(f"Submitting batch file for {system.name} failed. It can be found here: {self.__get_batch_filepath(target_name)}. Please check the file for errors.")
 
     def validate_job_options(self, mfc) -> None:
-        if len(mfc.args["targets"]) != 1:
-            raise common.MFCException(f"The Batch engine requires a unique target (-t) to run.")
         pass
 
 
@@ -337,4 +333,3 @@ def get_engine(slug: str) -> Engine:
         raise common.MFCException(f"Unsupported engine {slug}.")
 
     return engine
-

toolchain/mfc/run/engines.py

@@ -33,6 +33,9 @@ def validate_job_options(self) -> None:
         engines.get_engine(self.mfc.args["engine"]).validate_job_options(self.mfc)
 
     def run(self) -> None:
+        for name in self.mfc.args["targets"]:
+            build.build_target(self.mfc, name)
+
         cons.print("[bold]Run[/bold]")
         cons.indent()
 
@@ -57,23 +60,9 @@ def run(self) -> None:
 
         self.validate_job_options()
 
-        for target_name in self.mfc.args["targets"]:
-            cons.print(no_indent=True)
-            cons.print(f"[bold]Running [magenta]{target_name}[/magenta][/bold]:")
-            cons.indent()
-
-            # We generate the /path/to/case/{target_name}.inp and
-            # src/{target_name}/case.fpp input files before building so that, in
-            # order for the target to be built with case optimization, if
-            # args["case_optimization"] is true. Doing so later, would require
-            # there be two builds of a target.
-            input_file.generate(target_name)
-
-            if not self.mfc.args["no_build"]:
-                build.build_target(self.mfc, target_name)
+        cons.print("Generating input files...")
+        for name in self.mfc.args["targets"]:
+            input_file.generate(name)
 
-            engine.run(target_name)
+        engine.run(self.mfc.args["targets"])
 
-            cons.unindent()
-
-        cons.unindent()

toolchain/mfc/run/run.py

@@ -37,7 +37,7 @@ class MFCException(Exception):
 
 def system(command: typing.List[str], no_exception: bool = False, exception_text=None, on_error=lambda: None, cwd=None, stdout=None, stderr=None) -> int:
     cmd = [ str(x) for x in command if not isspace(str(x)) ]
-    
+
     cons.print(no_indent=True)
     cons.print(f"$ {' '.join(cmd)}")
     cons.print(no_indent=True)

toolchain/mfc/util/common.py

@@ -64,12 +64,22 @@
 #>       on your system - if at all. {MFC::BIN} refers to
 #>       the path the MFC executable.
 #>
-jsrun {'--smpiargs="-gpu"' if gpus_per_node > 0 else ''} \
-      --nrs          {cpus_per_node*nodes}               \
-      --cpu_per_rs   1                                   \
-      --gpu_per_rs   {min(gpus_per_node, 1)}             \
-      --tasks_per_rs 1                                   \
-      "{MFC::BIN}"
+
+for binpath in {MFC::BINARIES}; do
+
+    echo -e ":) Running $binpath:"
+    echo ""
+
+    jsrun {'--smpiargs="-gpu"' if gpus_per_node > 0 else ''} \
+        --nrs          {cpus_per_node*nodes}               \
+        --cpu_per_rs   1                                   \
+        --gpu_per_rs   {min(gpus_per_node, 1)}             \
+        --tasks_per_rs 1                                   \
+        "$binpath"
+
+    echo ""
+
+done
 
 {MFC::EPILOGUE}
 #>

toolchain/templates/lsf.sh

@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 #>
-#>           - PBS Batch File Template -             
+#>           - PBS Batch File Template -
 #>
 #> This file is part of the ./mfc.sh run subsystem.
 #> For more information, please consult the README.
-#> 
+#>
 #> - You are invited to modify this file to suit your
 #>   needs, in order to get MFC running properly on
 #>   your system.
@@ -23,7 +23,7 @@
 #>
 #> - Statements of the form {MFC::expression} tell MFC
 #>   where to place the common code, across all batch
-#>   files that is required to run MFC. They are not 
+#>   files that is required to run MFC. They are not
 #>   intended to be modified by users.
 #>
 #PBS -N {name}
@@ -41,11 +41,11 @@
 
 
 
-#> 
+#>
 #> Note: If your system requires you to load environment
 #>       modules inside of your batch script, please load
 #>       them bellow.
-#> 
+#>
 
 
 
@@ -60,10 +60,18 @@
 #>       on your system - if at all. {MFC::BIN} refers to
 #>       the path the MFC executable.
 #>
-srun                                   \ 
-     --nodes={nodes}                   \
-     --ntasks-per-node {cpus_per_node} \
-     "{MFC::BIN}"
+
+for binpath in {MFC::BINARIES}; do
+
+    echo -e ":) Running $binpath:"
+
+    srun                                  \
+        --nodes={nodes}                   \
+        --ntasks-per-node {cpus_per_node} \
+        "$binpath"
+
+done
+
 #>
 #> srun --mpi=pmix   \
 #>      "{MFC::BIN}"
@@ -77,3 +85,4 @@ srun                                   \
 #>
 #> Note: Lines after the MFC Epilogue will not be executed.
 #>
+

toolchain/templates/pbs.sh

@@ -77,9 +77,16 @@
 #> srun --mpi=pmix   \
 #>      "{MFC::BIN}"
 #>
-mpirun                            \
+
+for binpath in {MFC::BINARIES}; do
+
+    echo -e ":) Running $binpath:"
+
+    mpirun                        \
         -np {cpus_per_node*nodes} \
-        "{MFC::BIN}"
+        "$binpath"
+
+done
 
 {MFC::EPILOGUE}