Fix h5py bug by using h5dump

Author: henryleberre

toolchain/dependencies/CMakeLists.txt

@@ -49,7 +49,7 @@ if (MFC_HDF5)
                            -DFORTRAN_LIBRARIES=ON
                            -DBUILD_TESTING=OFF
                            -DHDF5_BUILD_UTILS=OFF
-                           -DHDF5_BUILD_TOOLS=OFF
+                           -DHDF5_BUILD_TOOLS=ON
                            -DHDF5_BUILD_EXAMPLES=OFF
         )
     else()

toolchain/mfc.py

@@ -15,7 +15,7 @@ def __print_greeting():
     max_logo_line_length = max([ len(line) for line in MFC_LOGO_LINES ])
 
     host_line    = f"{getpass.getuser()}@{platform.node()} [{platform.system()}]"
-    targets_line = f"[bold]--targets {format_list_to_string([ f'[magenta]{target}[/magenta]' for target in ARG('targets')], 'None')}[/bold]"
+    targets_line = f"[bold]--targets {format_list_to_string(ARG('targets'), 'magenta', 'None')}[/bold]"
     help_line    = "$ ./mfc.sh \[build, run, test, clean] --help"
 
     MFC_SIDEBAR_LINES = [

toolchain/mfc/common.py

@@ -110,14 +110,20 @@ def delete_directory(dirpath: str) -> None:
         shutil.rmtree(dirpath)
 
 
-def get_py_program_output(filepath: str):
+def get_program_output(arguments: typing.List[str] = None, cwd=None):
+    if arguments is None:
+        arguments = []
+
+    proc = subprocess.Popen(arguments, cwd=cwd, stdout=subprocess.PIPE)
+
+    return (proc.communicate()[0].decode(), proc.returncode)
+
+
+def get_py_program_output(filepath: str, arguments: typing.List[str] = None):    
     dirpath  = os.path.abspath (os.path.dirname(filepath))
     filename = os.path.basename(filepath)
 
-    proc = subprocess.Popen(["python3", filename], cwd=dirpath,
-                            stdout=subprocess.PIPE)
-
-    return (proc.communicate()[0], proc.returncode)
+    return get_program_output(["python3", filename], cwd=dirpath)
 
 
 def isspace(s: str) -> bool:
@@ -145,18 +151,26 @@ def does_command_exist(s: str) -> bool:
     return 0 == os.system(f"command -v {s} > /dev/null 2>&1")
 
 
-def format_list_to_string(arr: list, empty = "nothing"):
+def format_list_to_string(arr: list, item_style=None, empty=None):
+    if empty is None:
+        empty = "nothing"
+    
+    pre, post = "", ""
+    if item_style is not None:
+        pre  = f"[{item_style}]"
+        post = f"[/{item_style}]"
+    
     if len(arr) == 0:
-        return empty
+        return f"{pre}{empty}{post}"
 
     if len(arr) == 1:
-        return arr[0]
+        return f"{pre}{arr[0]}{post}"
 
     if len(arr) == 2:
-        return f"{arr[0]} and {arr[1]}"
+        return f"{pre}{arr[0]}{post} and {pre}{arr[1]}{post}"
 
-    lhs = ', '.join(arr[:-1])
-    rhs = f", and {arr[-1]}"
+    lhs = ', '.join([ f"{pre}{e}{post}" for e in arr[:-1]])
+    rhs = f", and {pre}{arr[-1]}{post}"
 
     return lhs + rhs
 

toolchain/mfc/run/engines.py

@@ -132,7 +132,7 @@ def run(self, names: typing.List[str]) -> None:
         system = queues.get_system()
         cons.print(f"Detected the [bold magenta]{system.name}[/bold magenta] queue system.")
 
-        cons.print(f"Running [bold magenta]{common.format_list_to_string(names)}[/bold magenta]:")
+        cons.print(f"Running {common.format_list_to_string(names, 'bold magenta')}:")
         cons.indent()
 
         self.__create_batch_file(system, names)

toolchain/mfc/test/case.py

@@ -1,4 +1,4 @@
-import os, hashlib, binascii, subprocess, dataclasses
+import os, typing, hashlib, binascii, subprocess, dataclasses
 
 from ..      import case, common
 from ..state import ARG
@@ -101,25 +101,19 @@ def __init__(self, trace: str, mods: dict, ppn: int = None) -> None:
         self.ppn   = ppn if ppn is not None else 1
         super().__init__({**BASE_CFG.copy(), **mods})
 
-    def run(self, post_process_toggle) -> subprocess.CompletedProcess:
-        filepath          = f'"{self.get_dirpath()}/case.py"'
+    def run(self, filename: str, targets: typing.List[str]) -> subprocess.CompletedProcess:
+        filepath          = f'"{self.get_dirpath()}/{filename}.py"'
         tasks             = f"-n {self.ppn}"
         jobs              = f"-j {ARG('jobs')}"    if ARG("case_optimization")  else ""
         binary_option     = f"-b {ARG('binary')}"  if ARG("binary") is not None else ""
         case_optimization =  "--case-optimization" if ARG("case_optimization")  else "--no-build"
 
         mfc_script = ".\mfc.bat" if os.name == 'nt' else "./mfc.sh"
 
-        if post_process_toggle:
-            command: str = f'''\
-                {mfc_script} run {filepath} {tasks} {binary_option} {case_optimization} \
-                {jobs} 2>&1\
-                '''
-        else:
-            command: str = f'''\
-                {mfc_script} run {filepath} {tasks} {binary_option} {case_optimization} \
-                {jobs} -t pre_process simulation 2>&1\
-                '''
+        command: str = f'''\
+            {mfc_script} run {filepath} {tasks} {binary_option} {case_optimization} \
+            {jobs} -t {' '.join(targets)} 2>&1\
+            '''
 
         return subprocess.run(command, stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE, universal_newlines=True,
@@ -131,7 +125,7 @@ def get_uuid(self) -> str:
     def get_dirpath(self):
         return os.path.join(common.MFC_TESTDIR, self.get_uuid())
 
-    def create_directory(self):
+    def create_directory(self, filename: str):
         dirpath = self.get_dirpath()
 
         content = f"""\
@@ -144,7 +138,7 @@ def create_directory(self):
 
         common.create_directory(dirpath)
 
-        common.file_write(f"{dirpath}/case.py", content)
+        common.file_write(f"{dirpath}/{filename}.py", content)
 
     def __str__(self) -> str:
         return f"tests/[bold magenta]{self.get_uuid()}[/bold magenta]: {self.trace}"

toolchain/mfc/test/test.py

@@ -1,18 +1,16 @@
-import os
+import os, math, shutil
 
 from ..printer import cons
 from ..        import common
 from ..state   import ARG
 from .case     import TestCase
 from .cases    import generate_cases
 from ..        import sched
-from ..common  import MFCException
-from ..build   import build_targets
+from ..common  import MFCException, does_command_exist, format_list_to_string, get_program_output
+from ..build   import build_targets, get_install_dirpath
 from .         import pack as packer
 
 import rich, rich.table
-import h5py
-import numpy as np
 
 
 CASES = generate_cases()
@@ -82,21 +80,18 @@ def test():
 
         return
 
+    codes = ["pre_process", "simulation"] + (["post_process"] if ARG('test_all') else [])
     if not ARG("case_optimization"):
-        build_targets(["pre_process", "simulation", "post_process"])
+        build_targets(codes)
 
     range_str = f"from [bold magenta]{ARG('from')}[/bold magenta] to [bold magenta]{ARG('to')}[/bold magenta]"
 
     if len(ARG("only")) > 0:
-        range_str = "Only " + common.format_list_to_string([
-            f"[bold magenta]{uuid}[/bold magenta]" for uuid in ARG("only")
-        ], "Nothing to run")
-
-
-    cons.print(f"[bold]Test[/bold] | {range_str} ({len(CASES)} test{'s' if len(CASES) != 1 else ''})")
+        range_str = "Only " + format_list_to_string(ARG("only"), "bold magenta", "Nothing to run")
+    
+    cons.print(f"[bold]Test {format_list_to_string(codes, 'magenta')}[/bold] | {range_str} ({len(CASES)} test{'s' if len(CASES) != 1 else ''})")
     cons.indent()
 
-
     # Run CASES with multiple threads (if available)
     cons.print()
     cons.print(f" tests/[bold magenta]UUID[/bold magenta]    Summary")
@@ -122,31 +117,13 @@ def test():
         else:
             raise MFCException(f"Testing: There were [bold red]{nFAIL}[/bold red] failures.")
 
-    if ARG("test_all"):
-        cons.print(f"Now testing post process...")
-        cons.print()
-        nFAIL = 0
-        tasks = [
-            sched.Task(ppn=case.ppn, func=handle_case_post_process, args=[ case ]) for case in CASES
-        ]
-        sched.sched(tasks, nThreads)
-        cons.print()
-        if nFAIL == 0:
-            cons.print(f"Tested Post Process [bold green]✓[/bold green]")
-        else:
-            if nFAIL == 1:
-                raise MFCException(f"Testing: There was [bold red]1[/bold red] failure.")
-            else:
-                raise MFCException(f"Testing: There were [bold red]{nFAIL}[/bold red] failures.")
     cons.unindent()
 
 
 def handle_case(test: TestCase):
     global nFAIL
 
     try:
-        test.create_directory()
-
         if test.params.get("qbmm", 'F') == 'T':
             tol = 1e-10
         elif test.params.get("bubbles", 'F') == 'T':
@@ -156,9 +133,10 @@ def handle_case(test: TestCase):
         else:
             tol = 1e-12
 
-        cmd = test.run(False)
+        test.create_directory("case_pre_sim")
+        cmd = test.run("case_pre_sim", ["pre_process", "simulation"])
 
-        out_filepath = os.path.join(test.get_dirpath(), "out.txt")
+        out_filepath = os.path.join(test.get_dirpath(), "out_pre_sim.txt")
 
         common.file_write(out_filepath, cmd.stdout)
 
@@ -179,63 +157,56 @@ def handle_case(test: TestCase):
 
             packer.check_tolerance(test, pack, packer.load(golden_filepath), tol)
 
-        cons.print(f"  [bold magenta]{test.get_uuid()}[/bold magenta]    {test.trace}")
+        if ARG("test_all"):
+            test.params.update({
+                'parallel_io'       : 'T',
+                'cons_vars_wrt'     : 'T',
+                'prim_vars_wrt'     : 'T',
+                'alpha_rho_wrt(1)'  : 'T',
+                'rho_wrt'           : 'T',
+                'mom_wrt(1)'        : 'T',
+                'vel_wrt(1)'        : 'T',
+                'E_wrt'             : 'T',
+                'pres_wrt'          : 'T',
+                'alpha_wrt(1)'      : 'T',
+                'gamma_wrt'         : 'T',
+                'heat_ratio_wrt'    : 'T',
+                'pi_inf_wrt'        : 'T',
+                'pres_inf_wrt'      : 'T',
+                'c_wrt'             : 'T',
+            })
+            
+            if test.params['p'] != 0:
+                test.params['omega_wrt'] = 'T'
+                test.params['fd_order'] = 1            
+            
+            test.create_directory("case_post")
+            cmd = test.run("case_post", ["pre_process", "simulation", "post_process"])
+            out_filepath = os.path.join(test.get_dirpath(), "out_post.txt")
+            common.file_write(out_filepath, cmd.stdout)
+
+            for t_step in [ i*test["t_step_save"] for i in range(0, math.floor(test["t_step_stop"] / test["t_step_save"]) + 1) ]:
+                silo_filepath = os.path.join(test.get_dirpath(), 'silo_hdf5', 'p0', f'{t_step}.silo')
+        
+                h5dump = f"{get_install_dirpath()}/bin/h5dump"
+            
+                if ARG("no_hdf5"):
+                    if not does_command_exist("h5dump"):
+                        raise MFCException("--no-hdf5 was specified and h5dump couldn't be found.")
+                    
+                    h5dump = shutil.which("h5dump")
 
-    except Exception as exc:
-        nFAIL = nFAIL + 1
+                output, err = get_program_output([h5dump, silo_filepath])
 
-        if not ARG("relentless"):
-            raise exc
+                if err != 0:
+                    raise MFCException(f"""Test {test}: Failed to run h5dump. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
 
-        cons.print(f"[bold red]Failed test {test}.[/bold red]")
-        cons.print(f"{exc}")
+                if "nan," in output:
+                    raise MFCException(f"""Test {test}: Post Process has detected a NaN. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
 
-def handle_case_post_process(test: TestCase):
-    global nFAIL
-    
-    try:
-        test.params.update({
-            'parallel_io'       : 'T',
-            'cons_vars_wrt'     : 'T',
-            'prim_vars_wrt'     : 'T',
-            'alpha_rho_wrt(1)'  : 'T',
-            'rho_wrt'           : 'T',
-            'mom_wrt(1)'        : 'T',
-            'vel_wrt(1)'        : 'T',
-            'E_wrt'             : 'T',
-            'pres_wrt'          : 'T',
-            'alpha_wrt(1)'      : 'T',
-            'gamma_wrt'         : 'T',
-            'heat_ratio_wrt'    : 'T',
-            'pi_inf_wrt'        : 'T',
-            'pres_inf_wrt'      : 'T',
-            'c_wrt'             : 'T',
-        })
-
-        if test.params['p'] != 0:
-            test.params['omega_wrt'] = 'T'
-            test.params['fd_order'] = 1
-
-        test.create_directory()
-
-        cmd = test.run(True)
-
-        out_filepath = os.path.join(test.get_dirpath(), "out_pp.txt")
+                if "inf," in output:
+                    raise MFCException(f"""Test {test}: Post Process has detected an Infinity. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
 
-        common.file_write(out_filepath, cmd.stdout)
-
-        if cmd.returncode != 0:
-            cons.print(cmd.stdout)
-            raise MFCException(f"""Test {test}: Failed to execute MFC. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
-
-        silo_filepath = os.path.join(test.get_dirpath(), 'silo_hdf5', 'p0', '50.silo')
-        f = h5py.File(silo_filepath, 'r')
-        silo = f['.silo']
-        for key in silo.keys():
-            dataset = silo[key]
-            for data in dataset:
-                check_data(data)
-        
         cons.print(f"  [bold magenta]{test.get_uuid()}[/bold magenta]    {test.trace}")
 
     except Exception as exc:
@@ -246,14 +217,3 @@ def handle_case_post_process(test: TestCase):
 
         cons.print(f"[bold red]Failed test {test}.[/bold red]")
         cons.print(f"{exc}")
-
-def check_data(data):
-    if isinstance(data, np.ndarray):
-        for subdata in data:
-            check_data(subdata)
-    else:
-        if np.isnan(data):
-            raise MFCException(f"""Test {test}: Post Process has detected a NaN. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
-        if np.isinf(data):
-            raise MFCException(f"""Test {test}: Post Process has detected an Infinity. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
-

toolchain/requirements.txt

@@ -5,4 +5,3 @@ typing
 PyYAML
 argparse
 dataclasses
-h5py