Add visualization & case utility libraries

Author: henryleberre

.github/file-filter.yml

@@ -7,7 +7,7 @@ fortran_src: &fortran_src
 
 python_src: &python_src
   - '**/*.py'
-  - 'toolchain/requirements.txt'
+  - 'toolchain/pyproject.toml'
 
 cmakelist: &cmakelist
   - 'CMakeLists.txt'

docs/documentation/case.md

@@ -41,17 +41,18 @@ parser = argparse.ArgumentParser(
     description="Weak- and strong-scaling benchmark case.",
     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-parser.add_argument("dict", type=str, metavar="DICT")
-parser.add_argument("-s", "--scaling",  type=str, metavar="SCALING",  choices=["weak", "strong"], help="Whether weak- or strong-scaling is being exercised.")
+parser.add_argument("--mfc",           type=str, metavar="DICT", default='{}')
+parser.add_argument("-s", "--scaling", type=str, metavar="SCALING", choices=["weak", "strong"],
+    help="Whether weak- or strong-scaling is being exercised.")
 
 # Your parsed arguments are here
 args = parser.parse_args()
 ```
 
-The first argument is always a JSON string representing `mfc.sh run`'s internal
-state.
+The `--mfc` argument is a JSON string representing `mfc.sh run`'s internal
+state, passed in when MFC runs your input file.
 It contains all the runtime information you might want from the build/run system.
-You can add as many additional arguments as you may need.
+You can add as many additional arguments and options as you may need.
 
 To run such a case, use the following format:
 

examples/scaling/case.py

@@ -7,7 +7,7 @@
     description="Weak- and strong-scaling benchmark case.",
     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-parser.add_argument("dict", type=str, metavar="DICT")
+parser.add_argument("--mfc", type=str, metavar="MFC")
 parser.add_argument("-s", "--scaling",       type=str, metavar="SCALING", choices=["weak", "strong"], help="Whether weak- or strong-scaling is being exercised.")
 parser.add_argument("-m", "--memory",        type=int, metavar="MEMORY",  help="Weak scaling: memory per rank in GB. Strong scaling: global memory in GB. Used to determine cell count.")
 parser.add_argument("-f", "--fidelity",      type=str, metavar="FIDELITY", choices=["ideal", "exact"], default="ideal")
@@ -20,7 +20,7 @@
     parser.print_help()
     sys.exit(1)
 
-DICT = json.loads(args.dict)
+DICT = json.loads(args.mfc)
 
 # \approx The number of cells per GB of memory. The exact value is not important.
 cpg    = 8000000 / 16.0

mfc.bat

@@ -31,11 +31,11 @@ if %errorlevel% neq 0 (
 	exit /b 1
 )
 
-fc /b "%cd%\build\requirements.txt" "%cd%\toolchain\requirements.txt" 2> NUL
+fc /b "%cd%\build\pyproject.toml" "%cd%\toolchain\pyproject.toml" 2> NUL
 if %errorlevel% neq 0 (
-    pip3 install -r toolchain/requirements.txt
+    pip3 install -e toolchain
 
-    copy "%cd%\toolchain\requirements.txt" "%cd%\build" 2> NUL
+    copy "%cd%\toolchain\pyproject.toml" "%cd%\build" 2> NUL
 )
 
 python "%cd%\toolchain\main.py" %*

misc/viz.py

@@ -2,6 +2,6 @@
 
 log "(venv) Running$MAGENTA pylint$COLOR_RESET on$MAGENTA MFC$COLOR_RESET's $MAGENTA""toolchain$COLOR_RESET."
 
-pylint -d R1722,W0718,C0301,C0116,C0115,C0114,C0410,W0622,W0640,C0103,W1309,C0411,W1514,R0401,W0511 "$(pwd)/toolchain/"
+pylint -d R1722,W0718,C0301,C0116,C0115,C0114,C0410,W0622,W0640,C0103,W1309,C0411,W1514,R0401,W0511,C0321,C3001 "$(pwd)/toolchain/"
 
 exit $?

toolchain/bootstrap/lint.sh

@@ -74,7 +74,7 @@ if [ ! -f "$(pwd)/build/venv/bin/activate" ]; then
 
     ok "Created a$MAGENTA Python$COLOR_RESET virtual environment (venv)."
 
-    rm "$(pwd)/build/requirements.txt" > /dev/null 2>&1 || true
+    rm "$(pwd)/build/pyproject.toml" > /dev/null 2>&1 || true
 fi
 
 
@@ -111,8 +111,8 @@ ok "(venv) Entered the $MAGENTA$(python3 --version)$COLOR_RESET virtual environm
 # Install Python dependencies if, either:
 # - This script is running for the first time
 # (or)
-# - The requirements.txt file has changed
-if ! cmp "$(pwd)/toolchain/requirements.txt" "$(pwd)/build/requirements.txt" > /dev/null 2>&1; then
+# - The pyproject.toml file has changed
+if ! cmp "$(pwd)/toolchain/pyproject.toml" "$(pwd)/build/pyproject.toml" > /dev/null 2>&1; then
     log "(venv) (Re)Installing mfc.sh's Python dependencies (via Pip)."
 
     next_arg=0
@@ -129,7 +129,7 @@ if ! cmp "$(pwd)/toolchain/requirements.txt" "$(pwd)/build/requirements.txt" > /
         fi
     done
 
-    if ! PIP_DISABLE_PIP_VERSION_CHECK=1 MAKEFLAGS=$nthreads pip3 install -r "$(pwd)/toolchain/requirements.txt"; then
+    if ! PIP_DISABLE_PIP_VERSION_CHECK=1 MAKEFLAGS=$nthreads pip3 install -e "$(pwd)/toolchain"; then
         error "(venv) Installation failed."
 
         log   "(venv) Exiting the$MAGENTA Python$COLOR_RESET virtual environment."
@@ -140,6 +140,6 @@ if ! cmp "$(pwd)/toolchain/requirements.txt" "$(pwd)/build/requirements.txt" > /
 
     ok "(venv) Installation succeeded."
 
-    # Save the new/current requirements.txt
-    cp "$(pwd)/toolchain/requirements.txt" "$(pwd)/build/"
+    # Save the new/current pyproject.toml
+    cp "$(pwd)/toolchain/pyproject.toml" "$(pwd)/build/"
 fi

toolchain/bootstrap/python.sh

@@ -0,0 +1,25 @@
+import re
+
+def remove_higher_dimensional_keys(case: dict, ndims: int) -> dict:
+    assert 1 <= ndims <= 3
+
+    rm_dims = [set(), set(['y', 'z']), set(['z'])][ndims]
+    dim_ids = {dim: i + 1 for i, dim in enumerate(['x', 'y', 'z'])}
+    dim_mnp = {'x': 'm', 'y': 'n', 'z': 'p'}
+
+    rm_keys = set()
+    for key in case.keys():
+        for dim in rm_dims:
+            if any([
+                re.match(f'.+_{dim}', key), re.match(f'{dim}_.+', key),
+                re.match(f'%{dim}', key),   f'%vel({dim_ids[dim]})' in key
+            ]):
+                rm_keys.add(key)
+                break
+
+    new_case = {k: v for k, v in case.items() if k not in rm_keys}
+
+    for dim in rm_dims:
+        new_case[dim_mnp[dim]] = 0
+
+    return new_case

toolchain/mfc/case_utils.py

@@ -144,7 +144,7 @@ def load(filepath: str = None, args: typing.List[str] = None, empty_data: dict =
         raise common.MFCException(f"Input file '{filename}' does not exist. Please check the path is valid.")
 
     if filename.endswith(".py"):
-        (json_str, err) = common.get_py_program_output(filename, [json.dumps(ARGS())] + (args or []))
+        (json_str, err) = common.get_py_program_output(filename, ["--mfc", json.dumps(ARGS())] + (args or []))
 
         if err != 0:
             raise common.MFCException(f"Input file {filename} terminated with a non-zero exit code. Please make sure running the file doesn't produce any errors.")

toolchain/mfc/run/input.py

@@ -179,16 +179,16 @@ def create_directory(self):
     description="{self.get_filepath()}: {self.trace}",
     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-parser.add_argument("dict", type=str, metavar="DICT", help=argparse.SUPPRESS)
+parser.add_argument("--mfc", type=str, metavar="MFC", help=argparse.SUPPRESS)
 
 ARGS = vars(parser.parse_args())
 
-ARGS["dict"] = json.loads(ARGS["dict"])
+ARGS["mfc"] = json.loads(ARGS["mfc"])
 
 case = {self.gen_json_dict_str()}
 mods = {{}}
 
-if "post_process" in ARGS["dict"]["targets"]:
+if "post_process" in ARGS["mfc"]["targets"]:
     mods = {{
         'parallel_io'  : 'T', 'cons_vars_wrt'   : 'T',
         'prim_vars_wrt': 'T', 'alpha_rho_wrt(1)': 'T',