Skip to content

Commit 7997a7e

Browse files
sbryngelsonsbryngelson
authored
Merge pull request #112 from MFlowCode/fix_h5py
2 parents 19e3a88 + 60734fd commit 7997a7e

File tree

7 files changed

+93
-126
lines changed

7 files changed

+93
-126
lines changed

‎toolchain/dependencies/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ if (MFC_HDF5)
4949
-DFORTRAN_LIBRARIES=ON
5050
-DBUILD_TESTING=OFF
5151
-DHDF5_BUILD_UTILS=OFF
52-
-DHDF5_BUILD_TOOLS=OFF
52+
-DHDF5_BUILD_TOOLS=ON
5353
-DHDF5_BUILD_EXAMPLES=OFF
5454
)
5555
else()

‎toolchain/mfc.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ def __print_greeting():
1515
max_logo_line_length = max([ len(line) for line in MFC_LOGO_LINES ])
1616

1717
host_line = f"{getpass.getuser()}@{platform.node()} [{platform.system()}]"
18-
targets_line = f"[bold]--targets {format_list_to_string([ f'[magenta]{target}[/magenta]' for target in ARG('targets')], 'None')}[/bold]"
18+
targets_line = f"[bold]--targets {format_list_to_string(ARG('targets'), 'magenta', 'None')}[/bold]"
1919
help_line = "$ ./mfc.sh \[build, run, test, clean] --help"
2020

2121
MFC_SIDEBAR_LINES = [

‎toolchain/mfc/common.py‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -110,14 +110,20 @@ def delete_directory(dirpath: str) -> None:
110110
shutil.rmtree(dirpath)
111111

112112

113-
def get_py_program_output(filepath: str):
113+
def get_program_output(arguments: typing.List[str] = None, cwd=None):
114+
if arguments is None:
115+
arguments = []
116+
117+
proc = subprocess.Popen(arguments, cwd=cwd, stdout=subprocess.PIPE)
118+
119+
return (proc.communicate()[0].decode(), proc.returncode)
120+
121+
122+
def get_py_program_output(filepath: str, arguments: typing.List[str] = None):
114123
dirpath = os.path.abspath (os.path.dirname(filepath))
115124
filename = os.path.basename(filepath)
116125

117-
proc = subprocess.Popen(["python3", filename], cwd=dirpath,
118-
stdout=subprocess.PIPE)
119-
120-
return (proc.communicate()[0], proc.returncode)
126+
return get_program_output(["python3", filename], cwd=dirpath)
121127

122128

123129
def isspace(s: str) -> bool:
@@ -145,18 +151,26 @@ def does_command_exist(s: str) -> bool:
145151
return 0 == os.system(f"command -v {s} > /dev/null 2>&1")
146152

147153

148-
def format_list_to_string(arr: list, empty = "nothing"):
154+
def format_list_to_string(arr: list, item_style=None, empty=None):
155+
if empty is None:
156+
empty = "nothing"
157+
158+
pre, post = "", ""
159+
if item_style is not None:
160+
pre = f"[{item_style}]"
161+
post = f"[/{item_style}]"
162+
149163
if len(arr) == 0:
150-
return empty
164+
return f"{pre}{empty}{post}"
151165

152166
if len(arr) == 1:
153-
return arr[0]
167+
return f"{pre}{arr[0]}{post}"
154168

155169
if len(arr) == 2:
156-
return f"{arr[0]} and {arr[1]}"
170+
return f"{pre}{arr[0]}{post} and {pre}{arr[1]}{post}"
157171

158-
lhs = ', '.join(arr[:-1])
159-
rhs = f", and {arr[-1]}"
172+
lhs = ', '.join([ f"{pre}{e}{post}" for e in arr[:-1]])
173+
rhs = f", and {pre}{arr[-1]}{post}"
160174

161175
return lhs + rhs
162176

‎toolchain/mfc/run/engines.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ def run(self, names: typing.List[str]) -> None:
132132
system = queues.get_system()
133133
cons.print(f"Detected the [bold magenta]{system.name}[/bold magenta] queue system.")
134134

135-
cons.print(f"Running [bold magenta]{common.format_list_to_string(names)}[/bold magenta]:")
135+
cons.print(f"Running {common.format_list_to_string(names, 'bold magenta')}:")
136136
cons.indent()
137137

138138
self.__create_batch_file(system, names)

‎toolchain/mfc/test/case.py‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import os, hashlib, binascii, subprocess, dataclasses
1+
import os, typing, hashlib, binascii, subprocess, dataclasses
22

33
from .. import case, common
44
from ..state import ARG
@@ -101,25 +101,19 @@ def __init__(self, trace: str, mods: dict, ppn: int = None) -> None:
101101
self.ppn = ppn if ppn is not None else 1
102102
super().__init__({**BASE_CFG.copy(), **mods})
103103

104-
def run(self, post_process_toggle) -> subprocess.CompletedProcess:
105-
filepath = f'"{self.get_dirpath()}/case.py"'
104+
def run(self, filename: str, targets: typing.List[str]) -> subprocess.CompletedProcess:
105+
filepath = f'"{self.get_dirpath()}/{filename}.py"'
106106
tasks = f"-n {self.ppn}"
107107
jobs = f"-j {ARG('jobs')}" if ARG("case_optimization") else ""
108108
binary_option = f"-b {ARG('binary')}" if ARG("binary") is not None else ""
109109
case_optimization = "--case-optimization" if ARG("case_optimization") else "--no-build"
110110

111111
mfc_script = ".\mfc.bat" if os.name == 'nt' else "./mfc.sh"
112112

113-
if post_process_toggle:
114-
command: str = f'''\
115-
{mfc_script} run {filepath} {tasks} {binary_option} {case_optimization} \
116-
{jobs} 2>&1\
117-
'''
118-
else:
119-
command: str = f'''\
120-
{mfc_script} run {filepath} {tasks} {binary_option} {case_optimization} \
121-
{jobs} -t pre_process simulation 2>&1\
122-
'''
113+
command: str = f'''\
114+
{mfc_script} run {filepath} {tasks} {binary_option} {case_optimization} \
115+
{jobs} -t {' '.join(targets)} 2>&1\
116+
'''
123117

124118
return subprocess.run(command, stdout=subprocess.PIPE,
125119
stderr=subprocess.PIPE, universal_newlines=True,
@@ -131,7 +125,7 @@ def get_uuid(self) -> str:
131125
def get_dirpath(self):
132126
return os.path.join(common.MFC_TESTDIR, self.get_uuid())
133127

134-
def create_directory(self):
128+
def create_directory(self, filename: str):
135129
dirpath = self.get_dirpath()
136130

137131
content = f"""\
@@ -144,7 +138,7 @@ def create_directory(self):
144138

145139
common.create_directory(dirpath)
146140

147-
common.file_write(f"{dirpath}/case.py", content)
141+
common.file_write(f"{dirpath}/{filename}.py", content)
148142

149143
def __str__(self) -> str:
150144
return f"tests/[bold magenta]{self.get_uuid()}[/bold magenta]: {self.trace}"

‎toolchain/mfc/test/test.py‎

Lines changed: 56 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,16 @@
1-
import os
1+
import os, math, shutil
22

33
from ..printer import cons
44
from .. import common
55
from ..state import ARG
66
from .case import TestCase
77
from .cases import generate_cases
88
from .. import sched
9-
from ..common import MFCException
10-
from ..build import build_targets
9+
from ..common import MFCException, does_command_exist, format_list_to_string, get_program_output
10+
from ..build import build_targets, get_install_dirpath
1111
from . import pack as packer
1212

1313
import rich, rich.table
14-
import h5py
15-
import numpy as np
1614

1715

1816
CASES = generate_cases()
@@ -82,21 +80,18 @@ def test():
8280

8381
return
8482

83+
codes = ["pre_process", "simulation"] + (["post_process"] if ARG('test_all') else [])
8584
if not ARG("case_optimization"):
86-
build_targets(["pre_process", "simulation", "post_process"])
85+
build_targets(codes)
8786

8887
range_str = f"from [bold magenta]{ARG('from')}[/bold magenta] to [bold magenta]{ARG('to')}[/bold magenta]"
8988

9089
if len(ARG("only")) > 0:
91-
range_str = "Only " + common.format_list_to_string([
92-
f"[bold magenta]{uuid}[/bold magenta]" for uuid in ARG("only")
93-
], "Nothing to run")
94-
95-
96-
cons.print(f"[bold]Test[/bold] | {range_str} ({len(CASES)} test{'s' if len(CASES) != 1 else ''})")
90+
range_str = "Only " + format_list_to_string(ARG("only"), "bold magenta", "Nothing to run")
91+
92+
cons.print(f"[bold]Test {format_list_to_string(codes, 'magenta')}[/bold] | {range_str} ({len(CASES)} test{'s' if len(CASES) != 1 else ''})")
9793
cons.indent()
9894

99-
10095
# Run CASES with multiple threads (if available)
10196
cons.print()
10297
cons.print(f" tests/[bold magenta]UUID[/bold magenta] Summary")
@@ -122,31 +117,13 @@ def test():
122117
else:
123118
raise MFCException(f"Testing: There were [bold red]{nFAIL}[/bold red] failures.")
124119

125-
if ARG("test_all"):
126-
cons.print(f"Now testing post process...")
127-
cons.print()
128-
nFAIL = 0
129-
tasks = [
130-
sched.Task(ppn=case.ppn, func=handle_case_post_process, args=[ case ]) for case in CASES
131-
]
132-
sched.sched(tasks, nThreads)
133-
cons.print()
134-
if nFAIL == 0:
135-
cons.print(f"Tested Post Process [bold green]✓[/bold green]")
136-
else:
137-
if nFAIL == 1:
138-
raise MFCException(f"Testing: There was [bold red]1[/bold red] failure.")
139-
else:
140-
raise MFCException(f"Testing: There were [bold red]{nFAIL}[/bold red] failures.")
141120
cons.unindent()
142121

143122

144123
def handle_case(test: TestCase):
145124
global nFAIL
146125

147126
try:
148-
test.create_directory()
149-
150127
if test.params.get("qbmm", 'F') == 'T':
151128
tol = 1e-10
152129
elif test.params.get("bubbles", 'F') == 'T':
@@ -156,9 +133,10 @@ def handle_case(test: TestCase):
156133
else:
157134
tol = 1e-12
158135

159-
cmd = test.run(False)
136+
test.create_directory("case_pre_sim")
137+
cmd = test.run("case_pre_sim", ["pre_process", "simulation"])
160138

161-
out_filepath = os.path.join(test.get_dirpath(), "out.txt")
139+
out_filepath = os.path.join(test.get_dirpath(), "out_pre_sim.txt")
162140

163141
common.file_write(out_filepath, cmd.stdout)
164142

@@ -179,63 +157,56 @@ def handle_case(test: TestCase):
179157

180158
packer.check_tolerance(test, pack, packer.load(golden_filepath), tol)
181159

182-
cons.print(f" [bold magenta]{test.get_uuid()}[/bold magenta] {test.trace}")
160+
if ARG("test_all"):
161+
test.params.update({
162+
'parallel_io' : 'T',
163+
'cons_vars_wrt' : 'T',
164+
'prim_vars_wrt' : 'T',
165+
'alpha_rho_wrt(1)' : 'T',
166+
'rho_wrt' : 'T',
167+
'mom_wrt(1)' : 'T',
168+
'vel_wrt(1)' : 'T',
169+
'E_wrt' : 'T',
170+
'pres_wrt' : 'T',
171+
'alpha_wrt(1)' : 'T',
172+
'gamma_wrt' : 'T',
173+
'heat_ratio_wrt' : 'T',
174+
'pi_inf_wrt' : 'T',
175+
'pres_inf_wrt' : 'T',
176+
'c_wrt' : 'T',
177+
})
178+
179+
if test.params['p'] != 0:
180+
test.params['omega_wrt'] = 'T'
181+
test.params['fd_order'] = 1
182+
183+
test.create_directory("case_post")
184+
cmd = test.run("case_post", ["pre_process", "simulation", "post_process"])
185+
out_filepath = os.path.join(test.get_dirpath(), "out_post.txt")
186+
common.file_write(out_filepath, cmd.stdout)
187+
188+
for t_step in [ i*test["t_step_save"] for i in range(0, math.floor(test["t_step_stop"] / test["t_step_save"]) + 1) ]:
189+
silo_filepath = os.path.join(test.get_dirpath(), 'silo_hdf5', 'p0', f'{t_step}.silo')
190+
191+
h5dump = f"{get_install_dirpath()}/bin/h5dump"
192+
193+
if ARG("no_hdf5"):
194+
if not does_command_exist("h5dump"):
195+
raise MFCException("--no-hdf5 was specified and h5dump couldn't be found.")
196+
197+
h5dump = shutil.which("h5dump")
183198

184-
except Exception as exc:
185-
nFAIL = nFAIL + 1
199+
output, err = get_program_output([h5dump, silo_filepath])
186200

187-
if not ARG("relentless"):
188-
raise exc
201+
if err != 0:
202+
raise MFCException(f"""Test {test}: Failed to run h5dump. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
189203

190-
cons.print(f"[bold red]Failed test {test}.[/bold red]")
191-
cons.print(f"{exc}")
204+
if "nan," in output:
205+
raise MFCException(f"""Test {test}: Post Process has detected a NaN. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
192206

193-
def handle_case_post_process(test: TestCase):
194-
global nFAIL
195-
196-
try:
197-
test.params.update({
198-
'parallel_io' : 'T',
199-
'cons_vars_wrt' : 'T',
200-
'prim_vars_wrt' : 'T',
201-
'alpha_rho_wrt(1)' : 'T',
202-
'rho_wrt' : 'T',
203-
'mom_wrt(1)' : 'T',
204-
'vel_wrt(1)' : 'T',
205-
'E_wrt' : 'T',
206-
'pres_wrt' : 'T',
207-
'alpha_wrt(1)' : 'T',
208-
'gamma_wrt' : 'T',
209-
'heat_ratio_wrt' : 'T',
210-
'pi_inf_wrt' : 'T',
211-
'pres_inf_wrt' : 'T',
212-
'c_wrt' : 'T',
213-
})
214-
215-
if test.params['p'] != 0:
216-
test.params['omega_wrt'] = 'T'
217-
test.params['fd_order'] = 1
218-
219-
test.create_directory()
220-
221-
cmd = test.run(True)
222-
223-
out_filepath = os.path.join(test.get_dirpath(), "out_pp.txt")
207+
if "inf," in output:
208+
raise MFCException(f"""Test {test}: Post Process has detected an Infinity. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
224209

225-
common.file_write(out_filepath, cmd.stdout)
226-
227-
if cmd.returncode != 0:
228-
cons.print(cmd.stdout)
229-
raise MFCException(f"""Test {test}: Failed to execute MFC. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
230-
231-
silo_filepath = os.path.join(test.get_dirpath(), 'silo_hdf5', 'p0', '50.silo')
232-
f = h5py.File(silo_filepath, 'r')
233-
silo = f['.silo']
234-
for key in silo.keys():
235-
dataset = silo[key]
236-
for data in dataset:
237-
check_data(data)
238-
239210
cons.print(f" [bold magenta]{test.get_uuid()}[/bold magenta] {test.trace}")
240211

241212
except Exception as exc:
@@ -246,14 +217,3 @@ def handle_case_post_process(test: TestCase):
246217

247218
cons.print(f"[bold red]Failed test {test}.[/bold red]")
248219
cons.print(f"{exc}")
249-
250-
def check_data(data):
251-
if isinstance(data, np.ndarray):
252-
for subdata in data:
253-
check_data(subdata)
254-
else:
255-
if np.isnan(data):
256-
raise MFCException(f"""Test {test}: Post Process has detected a NaN. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
257-
if np.isinf(data):
258-
raise MFCException(f"""Test {test}: Post Process has detected an Infinity. You can find the run's output in {out_filepath}, and the case dictionary in {os.path.join(test.get_dirpath(), "case.py")}.""")
259-

‎toolchain/requirements.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,3 @@ typing
55
PyYAML
66
argparse
77
dataclasses
8-
h5py

0 commit comments

Comments
 (0)