mfc.sh test: --add-new-variables option

Author: henryleberre

docs/documentation/testing.md

@@ -17,13 +17,15 @@ To restrict to a given range, use the `--from` (`-f`) and `--to` (`-t`) options.
 
 ### Creating Tests
 
-To (re)generate *golden files*, append the `-g` (i.e `--generate`) option:
+To (re)generate *golden files*, append the `--generate` option:
 ```console
-$ ./mfc.sh test -g -j 8
+$ ./mfc.sh test --generate -j 8
 ```
 
 It is recommended that a range be specified when generating golden files for new test cases, as described in the previous section, in an effort not to regenerate the golden files of existing test cases.
 
+**Note:** If you output new variables and want to update the golden files to include these without modifying the original data, use the `--add-new-variables` option instead.
+
 Adding a new test case can be done by modifying [cases.py](https://github.com/MFlowCode/MFC/tree/master/toolchain/mfc/test/cases.py). The function `generate_cases` is responsible for generating the list of test cases. Loops and conditionals are used to vary parameters, whose defaults can be found in the `BASE_CFG` case object within [case.py](https://github.com/MFlowCode/MFC/tree/master/toolchain/mfc/test/case.py). The function operates on two variables:
 
 - `stack`: A stack that holds the variations to the default case parameters. By pushing and popping the stack inside loops and conditionals, it is easier to nest test case descriptions, as it holds the variations that are common to all future test cases within the same indentation level (in most scenarios).

toolchain/mfc/args.py

@@ -20,7 +20,7 @@ def parse(config):
     )
 
     parsers = parser.add_subparsers(dest="command")
-
+    
     run   = parsers.add_parser(name="run",   help="Run a case with MFC.",            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     test  = parsers.add_parser(name="test",  help="Run MFC's test suite.",           formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     build = parsers.add_parser(name="build", help="Build MFC and its dependencies.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -63,15 +63,19 @@ def add_common_arguments(p, mask = None):
 
     # === TEST ===
     add_common_arguments(test, "t")
-    test.add_argument(      "--generate",    action="store_true", help="Generate golden files.")
     test.add_argument("-l", "--list",        action="store_true", help="List all available tests.")
     test.add_argument("-f", "--from",        default=TEST_CASES[0].get_uuid(), type=str, help="First test UUID to run.")
     test.add_argument("-t", "--to",          default=TEST_CASES[-1].get_uuid(), type=str, help="Last test UUID to run.")
     test.add_argument("-o", "--only",        nargs="+", type=str, default=[], metavar="L", help="Only run tests with UUIDs or hashes L.")
     test.add_argument("-b", "--binary",      choices=binaries, type=str, default=None, help="(Serial) Override MPI execution binary")
     test.add_argument("-r", "--relentless",  action="store_true", default=False, help="Run all tests, even if multiple fail.")
     test.add_argument("-a", "--test-all",    action="store_true", default=False, help="Run the Post Process Tests too.")
+
     test.add_argument("--case-optimization", action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.")
+    
+    test_meg = test.add_mutually_exclusive_group()
+    test_meg.add_argument("--generate",          action="store_true", default=False, help="(Test Generation) Generate golden files.")
+    test_meg.add_argument("--add-new-variables", action="store_true", default=False, help="(Test Generation) If new variables are found in D/ when running tests, add them to the golden files.")
 
     # === RUN ===
     engines = [ e.slug for e in ENGINES ]

toolchain/mfc/test/pack.py

@@ -1,4 +1,4 @@
-import os, re, math, dataclasses
+import os, re, math, typing, dataclasses
 
 from pathlib import Path
 
@@ -60,25 +60,41 @@ def __repr__(self) -> str:
         return self.get().__repr__()
 
 
+# This class maps to the data contained in one file in D/
 @dataclasses.dataclass(repr=False)
 class PackEntry:
     filepath: str
-    doubles:  list
+    doubles:  typing.List[float]
 
     def __repr__(self) -> str:
         return f"{self.filepath} {' '.join([ str(d) for d in self.doubles ])}"
 
 
-@dataclasses.dataclass
+# This class maps to the data contained in the entirety of D/: it is tush a list
+# of PackEntry classes.
 class Pack:
-    entries: list
+    entries: typing.Dict[str, PackEntry]
+
+    def __init__(self):
+        self.entries = {}
+
+    def __init__(self, entries: typing.List[PackEntry]):
+        self.entries = {}
+        for entry in entries:
+            self.set(entry)
+
+    def find(self, filepath: str) -> PackEntry:
+        return self.entries.get(filepath, None)
+
+    def set(self, entry: PackEntry):
+        self.entries[entry.filepath] = entry
 
     def save(self, filepath: str):
-        common.file_write(filepath, '\n'.join([ str(e) for e in self.entries ]))
+        common.file_write(filepath, '\n'.join([ str(e) for e in sorted(self.entries.values(), key=lambda x: x.filepath) ]))
 
 
 def load(filepath: str) -> Pack:
-    entries: list = []
+    entries: typing.List[PackEntry] = []
 
     for line in common.file_read(filepath).splitlines():
         if common.isspace(line):
@@ -103,18 +119,14 @@ def generate(case: case.TestCase) -> Pack:
     for filepath in list(Path(D_dir).rglob("*.dat")):
         short_filepath = str(filepath).replace(f'{case_dir}', '')[1:].replace("\\", "/")
 
-        data_content = common.file_read(filepath)
-
-        # 2 or more (contiguous) spaces
-        pattern = r"([ ]{2,})"
-
-        numbers_str = re.sub(pattern, " ", data_content.replace('\n', '')).strip()
-
-        doubles: list = [ float(e) for e in numbers_str.split(' ') ]
+        try:
+            doubles = [ float(e) for e in re.sub(r"[\n\t\s]+", " ", common.file_read(filepath)).strip().split(' ') ]
+        except ValueError:
+            raise MFCException(f"Test {case}: Failed to interpret the content of [magenta]{filepath}[/magenta] as a list of floating point numbers.")
 
         for double in doubles:
             if math.isnan(double):
-                raise MFCException(f"Test {case}: A NaN was found while generating a pack file.")
+                raise MFCException(f"Test {case}: A NaN was found in {filepath} while generating a pack file.")
 
         entries.append(PackEntry(short_filepath,doubles))
 
@@ -144,21 +156,19 @@ def check_tolerance(case: case.TestCase, candidate: Pack, golden: Pack, tol: flo
 
     # Compare entry-count
     if len(candidate.entries) != len(golden.entries):
-        raise MFCException(f"Test {case}: Line count didn't match.")
+        raise MFCException(f"Test {case}: Line count does not match.")
 
     # For every entry in the golden's pack
-    for gIndex, gEntry in enumerate(golden.entries):
+    for gFilepath, gEntry in golden.entries.items():
         # Find the corresponding entry in the candidate's pack
-        cIndex, cEntry = common.find(lambda i, e: e.filepath == gEntry.filepath, candidate.entries)
-
-        if cIndex == None:
-            raise MFCException(f"Test {case}: No reference of {gEntry.filepath} in the candidate's pack.")
+        cEntry = candidate.find(gFilepath)
 
-        filepath: str = gEntry.filepath
+        if cEntry == None:
+            raise MFCException(f"Test {case}: No reference of {gFilepath} in the candidate's pack.")
 
         # Compare variable-count
         if len(gEntry.doubles) != len(cEntry.doubles):
-            raise MFCException(f"Test {case}: Variable count didn't match for {filepath}.")
+            raise MFCException(f"Test {case}: Variable count didn't match for {gFilepath}.")
 
         # Check if each variable is within tolerance
         for valIndex, (gVal, cVal) in enumerate(zip(gEntry.doubles, cEntry.doubles)):
@@ -168,7 +178,7 @@ def check_tolerance(case: case.TestCase, candidate: Pack, golden: Pack, tol: flo
 
             def raise_err(msg: str):
                 raise MFCException(f"""\
-Test {case}: Variable n°{valIndex+1} (1-indexed) in {filepath} {msg}:
+Test {case}: Variable n°{valIndex+1} (1-indexed) in {gFilepath} {msg}:
   - Description: {case.trace}
   - Candidate:   {cVal}
   - Golden:      {gVal}

toolchain/mfc/test/test.py

@@ -153,9 +153,18 @@ def handle_case(test: TestCase):
             pack.save(golden_filepath)
         else:
             if not os.path.isfile(golden_filepath):
-                raise MFCException(f"Test {test}: Golden file doesn't exist! To generate golden files, use the '-g' flag.")
+                raise MFCException(f"Test {test}: The golden file does not exist! To generate golden files, use the '--generate' flag.")
 
-            packer.check_tolerance(test, pack, packer.load(golden_filepath), tol)
+            golden = packer.load(golden_filepath)
+
+            if ARG("add_new_variables"):
+                for pfilepath, pentry in pack.entries.items():
+                    if golden.find(pfilepath) is None:
+                        golden.set(pentry)
+
+                golden.save(golden_filepath)
+            else:
+                packer.check_tolerance(test, pack, packer.load(golden_filepath), tol)
 
         if ARG("test_all"):
             test.params.update({